From 05561a6fc9e3bc785ba6956cb6b1284db8aaca0f Mon Sep 17 00:00:00 2001 From: Ondrej Smola Date: Mon, 13 Jul 2026 14:36:12 +0200 Subject: [PATCH 1/2] Add label-based Slack channel routing Route alerts to different Slack channels based on pod label rules. Each route lists label regexes (all must match) and one or more target channels; all matching routes apply with channels deduplicated. Events matching no route, and label-less messages (PVC alerts, startup ping), fall back to the default channel. Token mode only; webhook mode ignores routes with a warning. Co-Authored-By: Claude Fable 5 --- README.md | 25 ++++ alertmanager/slack/route.go | 76 ++++++++++ alertmanager/slack/slack.go | 79 +++++++++-- alertmanager/slack/slack_test.go | 232 +++++++++++++++++++++++++++++++ deploy/config.yaml | 8 ++ 5 files changed, 409 insertions(+), 11 deletions(-) create mode 100644 alertmanager/slack/route.go diff --git a/README.md b/README.md index 8ef9bfeb..76d1c56a 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,31 @@ If you want to enable Slack, provide either a webhook URL or a bot token with ch | `alert.slack.channel` | Required when using token. For webhooks, used by legacy webhooks to send messages to specific channel instead of default one | | `alert.slack.title` | Customized title in slack message | | `alert.slack.text` | Customized text in slack message | +| `alert.slack.routes` | Rules to route alerts to extra channels based on pod labels (token mode only) | + +With a bot token you can route alerts to different channels based on pod labels. +Each route lists label regexes (all must match; unanchored, use `^...$` to match +the full value) and the channels to notify. All matching routes apply. Events +matching no route, and messages without labels (e.g. PVC alerts), go to the +default `channel`. + +```yaml +alert: + slack: + token: xoxb-... + channel: "#alerts-default" + routes: + - labels: + app: "^payments-.*" + tier: "backend" + channels: + - "#payments-alerts" + - "#backend-oncall" + - labels: + team: "platform" + channels: + - "#platform-alerts" +``` #### Discord diff --git a/alertmanager/slack/route.go b/alertmanager/slack/route.go new file mode 100644 index 00000000..23d8eec2 --- /dev/null +++ b/alertmanager/slack/route.go @@ -0,0 +1,76 @@ +package slack + +import ( + "fmt" + "regexp" + + "gopkg.in/yaml.v3" +) + +// route sends events matching all label regexes to a set of channels +type route struct { + labels map[string]*regexp.Regexp + channels []string +} + +// matches reports whether every label regex in the route matches the +// corresponding event label. Regexes are unanchored; use ^...$ to match +// the full value. +func (r *route) matches(labels map[string]string) bool { + for key, re := range r.labels { + value, ok := labels[key] + if !ok || !re.MatchString(value) { + return false + } + } + return true +} + +type rawRoute struct { + Labels map[string]string `yaml:"labels"` + Channels []string `yaml:"channels"` +} + +// parseRoutes converts the untyped routes config into compiled routes +func parseRoutes(raw interface{}) ([]route, error) { + if raw == nil { + return nil, nil + } + + b, err := yaml.Marshal(raw) + if err != nil { + return nil, fmt.Errorf("invalid routes config: %w", err) + } + + var rawRoutes []rawRoute + if err := yaml.Unmarshal(b, &rawRoutes); err != nil { + return nil, fmt.Errorf("invalid routes config: %w", err) + } + + routes := make([]route, 0, len(rawRoutes)) + for i, rr := range rawRoutes { + if len(rr.Labels) == 0 { + return nil, fmt.Errorf("route %d has no labels", i) + } + if len(rr.Channels) == 0 { + return nil, fmt.Errorf("route %d has no channels", i) + } + + labels := make(map[string]*regexp.Regexp, len(rr.Labels)) + for key, pattern := range rr.Labels { + re, err := regexp.Compile(pattern) + if err != nil { + return nil, fmt.Errorf( + "route %d has invalid regex for label %q: %w", i, key, err) + } + labels[key] = re + } + + routes = append(routes, route{ + labels: labels, + channels: rr.Channels, + }) + } + + return routes, nil +} diff --git a/alertmanager/slack/slack.go b/alertmanager/slack/slack.go index 1f45c94c..bfc18c9f 100644 --- a/alertmanager/slack/slack.go +++ b/alertmanager/slack/slack.go @@ -1,6 +1,7 @@ package slack import ( + "errors" "fmt" "os" "strings" @@ -35,6 +36,15 @@ type Slack struct { // client is used for token-based messaging client *slackClient.Client + + // postMessage posts to a channel using the token-based client + postMessage func( + channelID string, + options ...slackClient.MsgOption) (string, string, error) + + // routes send events to additional channels based on their labels. + // Only supported with token-based authentication. + routes []route } // NewSlack returns new Slack instance @@ -62,14 +72,24 @@ func NewSlack(config map[string]interface{}, appCfg *config.App) *Slack { logrus.Warnf("initializing slack with token requires channel") return nil } + + routes, err := parseRoutes(config["routes"]) + if err != nil { + logrus.Errorf("initializing slack failed: %s", err) + return nil + } + logrus.Infof("initializing slack with token for channel: %s", channel) + client := slackClient.New(token) return &Slack{ - token: token, - channel: channel, - title: title, - text: text, - client: slackClient.New(token), - appCfg: appCfg, + token: token, + channel: channel, + title: title, + text: text, + client: client, + postMessage: client.PostMessage, + routes: routes, + appCfg: appCfg, } } @@ -79,6 +99,10 @@ func NewSlack(config map[string]interface{}, appCfg *config.App) *Slack { return nil } + if config["routes"] != nil { + logrus.Warnf("slack routes require token mode; ignoring routes") + } + logrus.Infof("initializing slack with webhook url: %s", webhook) return &Slack{ @@ -156,17 +180,41 @@ func (s *Slack) SendEvent(ev *event.Event) error { Blocks: &slackClient.Blocks{ BlockSet: append(blocks, markdownSection(constant.Footer)), }, - }) + }, s.channelsForEvent(ev)) } // SendMessage sends text message to the provider func (s *Slack) SendMessage(msg string) error { return s.sendAPI(&slackClient.WebhookMessage{ Text: msg, - }) + }, nil) } -func (s *Slack) sendAPI(msg *slackClient.WebhookMessage) error { +// channelsForEvent returns channels of all routes matching event labels, +// deduplicated in first-seen order, or the default channel if none match +func (s *Slack) channelsForEvent(ev *event.Event) []string { + var channels []string + seen := map[string]struct{}{} + for _, r := range s.routes { + if !r.matches(ev.Labels) { + continue + } + for _, ch := range r.channels { + if _, ok := seen[ch]; ok { + continue + } + seen[ch] = struct{}{} + channels = append(channels, ch) + } + } + if len(channels) == 0 { + return []string{s.channel} + } + return channels +} + +func (s *Slack) sendAPI( + msg *slackClient.WebhookMessage, channels []string) error { // Use token-based API if client is configured if s.client != nil { options := []slackClient.MsgOption{} @@ -176,8 +224,17 @@ func (s *Slack) sendAPI(msg *slackClient.WebhookMessage) error { if len(msg.Text) > 0 { options = append(options, slackClient.MsgOptionText(msg.Text, false)) } - _, _, err := s.client.PostMessage(s.channel, options...) - return err + + if len(channels) == 0 { + channels = []string{s.channel} + } + var errs []error + for _, ch := range channels { + if _, _, err := s.postMessage(ch, options...); err != nil { + errs = append(errs, fmt.Errorf("slack channel %q: %w", ch, err)) + } + } + return errors.Join(errs...) } // Fall back to webhook diff --git a/alertmanager/slack/slack_test.go b/alertmanager/slack/slack_test.go index baabc606..10aa1574 100644 --- a/alertmanager/slack/slack_test.go +++ b/alertmanager/slack/slack_test.go @@ -1,6 +1,7 @@ package slack import ( + "errors" "os" "testing" @@ -60,6 +61,237 @@ func TestSlackTokenPrecedence(t *testing.T) { assert.Empty(s.webhook) } +func TestSlackTokenWithRoutes(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{ + "app": "^payments-.*", + }, + "channels": []interface{}{"#payments-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + assert.Len(s.routes, 1) + assert.Equal([]string{"#payments-alerts"}, s.routes[0].channels) +} + +func TestSlackRoutesInvalidRegex(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"app": "[invalid"}, + "channels": []interface{}{"#payments-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.Nil(s) +} + +func TestSlackRoutesEmptyLabels(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "channels": []interface{}{"#payments-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.Nil(s) +} + +func TestSlackRoutesEmptyChannels(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"app": "payments"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.Nil(s) +} + +func TestSlackRoutesIgnoredInWebhookMode(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "webhook": "testtest", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"app": "payments"}, + "channels": []interface{}{"#payments-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + assert.Empty(s.routes) +} + +func TestChannelsForEvent(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{ + "app": "^payments-.*", + "tier": "backend", + }, + "channels": []interface{}{"#payments-alerts", "#backend-oncall"}, + }, + map[string]interface{}{ + "labels": map[string]interface{}{"tier": "backend"}, + "channels": []interface{}{"#backend-oncall", "#backend-all"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + + // all labels must match (AND); both rules match here, channels + // are deduplicated in first-seen order + assert.Equal( + []string{"#payments-alerts", "#backend-oncall", "#backend-all"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "app": "payments-api", + "tier": "backend", + }})) + + // only second rule matches when app label doesn't match + assert.Equal( + []string{"#backend-oncall", "#backend-all"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "app": "billing-api", + "tier": "backend", + }})) + + // missing label key means no match; fall back to default channel + assert.Equal( + []string{"#alerts"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "app": "payments-api", + }})) + + // no labels at all falls back to default channel + assert.Equal( + []string{"#alerts"}, + s.channelsForEvent(&event.Event{})) +} + +func TestSendEventRouting(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"team": "platform"}, + "channels": []interface{}{"#platform-alerts", "#platform-oncall"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + + var posted []string + s.postMessage = func( + ch string, opts ...slackClient.MsgOption) (string, string, error) { + posted = append(posted, ch) + return "", "", nil + } + + assert.Nil(s.SendEvent(&event.Event{ + PodName: "test-pod", + Labels: map[string]string{"team": "platform"}, + })) + assert.Equal([]string{"#platform-alerts", "#platform-oncall"}, posted) + + posted = nil + assert.Nil(s.SendEvent(&event.Event{ + PodName: "test-pod", + Labels: map[string]string{"team": "payments"}, + })) + assert.Equal([]string{"#alerts"}, posted) +} + +func TestSendMessageTokenDefaultChannel(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"team": "platform"}, + "channels": []interface{}{"#platform-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + + var posted []string + s.postMessage = func( + ch string, opts ...slackClient.MsgOption) (string, string, error) { + posted = append(posted, ch) + return "", "", nil + } + + assert.Nil(s.SendMessage("test")) + assert.Equal([]string{"#alerts"}, posted) +} + +func TestSendEventRoutingPostError(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"team": "platform"}, + "channels": []interface{}{"#failing", "#working"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + + var posted []string + s.postMessage = func( + ch string, opts ...slackClient.MsgOption) (string, string, error) { + posted = append(posted, ch) + if ch == "#failing" { + return "", "", errors.New("channel_not_found") + } + return "", "", nil + } + + err := s.SendEvent(&event.Event{ + PodName: "test-pod", + Labels: map[string]string{"team": "platform"}, + }) + assert.Error(err) + assert.Contains(err.Error(), "#failing") + // failing channel must not block the remaining channels + assert.Equal([]string{"#failing", "#working"}, posted) +} + func TestSlack(t *testing.T) { assert := assert.New(t) diff --git a/deploy/config.yaml b/deploy/config.yaml index b3dc7b8b..c12f854c 100644 --- a/deploy/config.yaml +++ b/deploy/config.yaml @@ -15,6 +15,14 @@ data: alert: slack: webhook: + # or use a bot token with optional label-based channel routing: + # token: + # channel: "#alerts-default" + # routes: + # - labels: + # app: "^payments-.*" + # channels: + # - "#payments-alerts" pagerduty: integrationKey: discord: From 1fe926c75c2576175b158ff1c76505baf8cbbb06 Mon Sep 17 00:00:00 2001 From: Ondrej Smola Date: Tue, 14 Jul 2026 16:15:50 +0200 Subject: [PATCH 2/2] Add includeDefault option to slack routes A route with includeDefault: true also sends matching events to the default channel (dual-send), without hardcoding its name in the route. Needed because the default channel may come from the SLACK_CHANNEL environment variable and vary per deployment. Co-Authored-By: Claude Fable 5 --- README.md | 8 +++--- alertmanager/slack/route.go | 13 +++++++--- alertmanager/slack/slack.go | 15 +++++++---- alertmanager/slack/slack_test.go | 43 ++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 76d1c56a..579f3a1a 100644 --- a/README.md +++ b/README.md @@ -151,9 +151,10 @@ If you want to enable Slack, provide either a webhook URL or a bot token with ch With a bot token you can route alerts to different channels based on pod labels. Each route lists label regexes (all must match; unanchored, use `^...$` to match -the full value) and the channels to notify. All matching routes apply. Events -matching no route, and messages without labels (e.g. PVC alerts), go to the -default `channel`. +the full value) and the channels to notify. All matching routes apply. Set +`includeDefault: true` on a route to also send matching events to the default +`channel`. Events matching no route, and messages without labels (e.g. PVC +alerts), go to the default `channel`. ```yaml alert: @@ -167,6 +168,7 @@ alert: channels: - "#payments-alerts" - "#backend-oncall" + includeDefault: true # also notify #alerts-default - labels: team: "platform" channels: diff --git a/alertmanager/slack/route.go b/alertmanager/slack/route.go index 23d8eec2..8105e347 100644 --- a/alertmanager/slack/route.go +++ b/alertmanager/slack/route.go @@ -11,6 +11,9 @@ import ( type route struct { labels map[string]*regexp.Regexp channels []string + + // includeDefault also sends matching events to the default channel + includeDefault bool } // matches reports whether every label regex in the route matches the @@ -27,8 +30,9 @@ func (r *route) matches(labels map[string]string) bool { } type rawRoute struct { - Labels map[string]string `yaml:"labels"` - Channels []string `yaml:"channels"` + Labels map[string]string `yaml:"labels"` + Channels []string `yaml:"channels"` + IncludeDefault bool `yaml:"includeDefault"` } // parseRoutes converts the untyped routes config into compiled routes @@ -67,8 +71,9 @@ func parseRoutes(raw interface{}) ([]route, error) { } routes = append(routes, route{ - labels: labels, - channels: rr.Channels, + labels: labels, + channels: rr.Channels, + includeDefault: rr.IncludeDefault, }) } diff --git a/alertmanager/slack/slack.go b/alertmanager/slack/slack.go index bfc18c9f..6e19d72b 100644 --- a/alertmanager/slack/slack.go +++ b/alertmanager/slack/slack.go @@ -195,16 +195,21 @@ func (s *Slack) SendMessage(msg string) error { func (s *Slack) channelsForEvent(ev *event.Event) []string { var channels []string seen := map[string]struct{}{} + add := func(ch string) { + if _, ok := seen[ch]; !ok { + seen[ch] = struct{}{} + channels = append(channels, ch) + } + } for _, r := range s.routes { if !r.matches(ev.Labels) { continue } for _, ch := range r.channels { - if _, ok := seen[ch]; ok { - continue - } - seen[ch] = struct{}{} - channels = append(channels, ch) + add(ch) + } + if r.includeDefault { + add(s.channel) } } if len(channels) == 0 { diff --git a/alertmanager/slack/slack_test.go b/alertmanager/slack/slack_test.go index 10aa1574..c15a25e6 100644 --- a/alertmanager/slack/slack_test.go +++ b/alertmanager/slack/slack_test.go @@ -195,6 +195,49 @@ func TestChannelsForEvent(t *testing.T) { s.channelsForEvent(&event.Event{})) } +func TestChannelsForEventIncludeDefault(t *testing.T) { + assert := assert.New(t) + + s := NewSlack(map[string]interface{}{ + "token": "xoxb-test-token", + "channel": "#alerts", + "routes": []interface{}{ + map[string]interface{}{ + "labels": map[string]interface{}{"team": "edr"}, + "channels": []interface{}{"#edr-alerts"}, + "includeDefault": true, + }, + map[string]interface{}{ + "labels": map[string]interface{}{"team": "platform"}, + "channels": []interface{}{"#platform-alerts"}, + }, + }, + }, &config.App{ClusterName: "dev"}) + assert.NotNil(s) + + // includeDefault adds the default channel after the route channels + assert.Equal( + []string{"#edr-alerts", "#alerts"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "team": "edr", + }})) + + // without includeDefault only the route channels are used + assert.Equal( + []string{"#platform-alerts"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "team": "platform", + }})) + + // default channel listed explicitly is not duplicated + s.routes[0].channels = []string{"#edr-alerts", "#alerts"} + assert.Equal( + []string{"#edr-alerts", "#alerts"}, + s.channelsForEvent(&event.Event{Labels: map[string]string{ + "team": "edr", + }})) +} + func TestSendEventRouting(t *testing.T) { assert := assert.New(t)