diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c99b295..a851efc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ + * [#158](https://github.com/linki/chaoskube/pull/158) Support for sending Slack notifications @GaruGaru + ## v0.16.0 - 2019-11-08 Features: diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index eaae805a..ce302d65 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -23,6 +23,7 @@ import ( "k8s.io/client-go/tools/reference" "github.com/linki/chaoskube/metrics" + "github.com/linki/chaoskube/notifier" "github.com/linki/chaoskube/terminator" "github.com/linki/chaoskube/util" ) @@ -67,6 +68,8 @@ type Chaoskube struct { Now func() time.Time MaxKill int + // chaos events notifier + Notifier notifier.Notifier } var ( @@ -90,7 +93,7 @@ var ( // * a logger implementing logrus.FieldLogger to send log output to // * what specific terminator to use to imbue chaos on victim pods // * whether to enable/disable dry-run mode -func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int) *Chaoskube { +func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)}) recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"}) @@ -114,6 +117,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces, namespace EventRecorder: recorder, Now: time.Now, MaxKill: maxKill, + Notifier: notifier, } } @@ -175,7 +179,6 @@ func (c *Chaoskube) TerminateVictims() error { for _, victim := range victims { err = c.DeletePod(victim) result = multierror.Append(result, err) - } return result.ErrorOrNil() @@ -257,6 +260,10 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error { c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.") + if err := c.Notifier.NotifyPodTermination(victim); err != nil { + c.Logger.WithField("err", err).Warn("failed to notify pod termination") + } + return nil } diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 970421fb..c0fda563 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -17,6 +17,7 @@ import ( "k8s.io/client-go/kubernetes/fake" "github.com/linki/chaoskube/internal/testutil" + "github.com/linki/chaoskube/notifier" "github.com/linki/chaoskube/terminator" "github.com/linki/chaoskube/util" @@ -35,6 +36,7 @@ type podInfo struct { var ( logger, logOutput = test.NewNullLogger() + testNotifier = ¬ifier.Noop{} ) func (suite *Suite) SetupTest() { @@ -59,6 +61,7 @@ func (suite *Suite) TestNew() { dryRun = true terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) maxKill = 1 + notifier = testNotifier ) chaoskube := New( @@ -78,6 +81,7 @@ func (suite *Suite) TestNew() { dryRun, terminator, maxKill, + notifier, ) suite.Require().NotNil(chaoskube) @@ -723,6 +727,10 @@ func (suite *Suite) assertVictim(chaoskube *Chaoskube, expected map[string]strin suite.assertVictims(chaoskube, []map[string]string{expected}) } +func (suite *Suite) assertNotified(notifier *notifier.Noop) { + suite.Assert().Greater(notifier.Calls, 0) +} + func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube { chaoskube := suite.setup( labelSelector, @@ -797,6 +805,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele dryRun, terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod), maxKill, + testNotifier, ) } @@ -971,3 +980,27 @@ func (suite *Suite) TestFilterByOwnerReference() { } } } + +func (suite *Suite) TestNotifierCall() { + chaoskube := suite.setupWithPods( + labels.Everything(), + labels.Everything(), + labels.Everything(), + labels.Everything(), + ®exp.Regexp{}, + ®exp.Regexp{}, + []time.Weekday{}, + []util.TimePeriod{}, + []time.Time{}, + time.UTC, + time.Duration(0), + false, + 10, + ) + + victim := util.NewPod("default", "foo", v1.PodRunning) + err := chaoskube.DeletePod(victim) + + suite.Require().NoError(err) + suite.assertNotified(testNotifier) +} diff --git a/main.go b/main.go index 0f63d279..20333c4e 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ import ( "k8s.io/klog" "github.com/linki/chaoskube/chaoskube" + "github.com/linki/chaoskube/notifier" "github.com/linki/chaoskube/terminator" "github.com/linki/chaoskube/util" ) @@ -56,6 +57,7 @@ var ( gracePeriod time.Duration logFormat string logCaller bool + slackWebhook string ) func init() { @@ -83,6 +85,7 @@ func init() { kingpin.Flag("grace-period", "Grace period to terminate Pods. Negative values will use the Pod's grace period.").Default("-1s").DurationVar(&gracePeriod) kingpin.Flag("log-format", "Specify the format of the log messages. Options are text and json. Defaults to text.").Default("text").EnumVar(&logFormat, "text", "json") kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").BoolVar(&logCaller) + kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").StringVar(&slackWebhook) } func main() { @@ -123,6 +126,7 @@ func main() { "metricsAddress": metricsAddress, "gracePeriod": gracePeriod, "logFormat": logFormat, + "slackWebhook": slackWebhook, }).Debug("reading config") log.WithFields(log.Fields{ @@ -191,6 +195,8 @@ func main() { "offset": offset / int(time.Hour/time.Second), }).Info("setting timezone") + notifiers := createNotifier() + chaoskube := chaoskube.New( client, labelSelector, @@ -208,6 +214,7 @@ func main() { dryRun, terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod), maxKill, + notifiers, ) if metricsAddress != "" { @@ -277,6 +284,15 @@ func parseSelector(str string) labels.Selector { return selector } +func createNotifier() notifier.Notifier { + notifiers := notifier.New() + if slackWebhook != "" { + notifiers.Add(notifier.NewSlackNotifier(slackWebhook)) + } + + return notifiers +} + func serveMetrics() { http.Handle("/metrics", promhttp.Handler()) http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { diff --git a/notifier/noop.go b/notifier/noop.go new file mode 100644 index 00000000..4438abdf --- /dev/null +++ b/notifier/noop.go @@ -0,0 +1,16 @@ +package notifier + +import ( + v1 "k8s.io/api/core/v1" +) + +const NotifierNoop = "noop" + +type Noop struct { + Calls int +} + +func (t *Noop) NotifyPodTermination(pod v1.Pod) error { + t.Calls++ + return nil +} diff --git a/notifier/notifier.go b/notifier/notifier.go new file mode 100644 index 00000000..015f9fb6 --- /dev/null +++ b/notifier/notifier.go @@ -0,0 +1,32 @@ +package notifier + +import ( + multierror "github.com/hashicorp/go-multierror" + v1 "k8s.io/api/core/v1" +) + +type Notifier interface { + NotifyPodTermination(pod v1.Pod) error +} + +type Notifiers struct { + notifiers []Notifier +} + +func New() *Notifiers { + return &Notifiers{notifiers: make([]Notifier, 0)} +} + +func (m *Notifiers) NotifyPodTermination(pod v1.Pod) error { + var result error + for _, n := range m.notifiers { + if err := n.NotifyPodTermination(pod); err != nil { + result = multierror.Append(result, err) + } + } + return result +} + +func (m *Notifiers) Add(notifier Notifier) { + m.notifiers = append(m.notifiers, notifier) +} diff --git a/notifier/notifier_test.go b/notifier/notifier_test.go new file mode 100644 index 00000000..c3db18dd --- /dev/null +++ b/notifier/notifier_test.go @@ -0,0 +1,87 @@ +package notifier + +import ( + "fmt" + "github.com/hashicorp/go-multierror" + "testing" + + v1 "k8s.io/api/core/v1" + + "github.com/linki/chaoskube/internal/testutil" + + "github.com/stretchr/testify/suite" +) + +type NotifierSuite struct { + testutil.TestSuite +} + +type FailingNotifier struct{} + +func (f FailingNotifier) NotifyPodTermination(pod v1.Pod) error { + return fmt.Errorf("notify error") +} + +func (suite *NotifierSuite) TestMultiNotifierWithoutNotifiers() { + manager := New() + err := manager.NotifyPodTermination(v1.Pod{}) + suite.NoError(err) +} + +func (suite *NotifierSuite) TestMultiNotifierWithNotifier() { + manager := New() + n := Noop{} + manager.Add(&n) + err := manager.NotifyPodTermination(v1.Pod{}) + suite.Require().NoError(err) + + suite.Equal(1, n.Calls) +} + +func (suite *NotifierSuite) TestMultiNotifierWithMultipleNotifier() { + manager := New() + n1 := Noop{} + n2 := Noop{} + manager.Add(&n1) + manager.Add(&n2) + + err := manager.NotifyPodTermination(v1.Pod{}) + suite.Require().NoError(err) + + suite.Equal(1, n1.Calls) + suite.Equal(1, n2.Calls) +} + +func (suite *NotifierSuite) TestMultiNotifierWithNotifierError() { + manager := New() + f := FailingNotifier{} + manager.Add(&f) + err := manager.NotifyPodTermination(v1.Pod{}) + suite.Require().Error(err) +} + +func (suite *NotifierSuite) TestMultiNotifierWithNotifierMultipleError() { + manager := New() + f0 := FailingNotifier{} + f1 := FailingNotifier{} + manager.Add(&f0) + manager.Add(&f1) + err := manager.NotifyPodTermination(v1.Pod{}).(*multierror.Error) + suite.Require().Error(err) + suite.Require().Len(err.Errors, 2) +} + +func (suite *NotifierSuite) TestMultiNotifierWithOneFailingNotifier() { + manager := New() + f := FailingNotifier{} + n := Noop{} + manager.Add(&n) + manager.Add(&f) + err := manager.NotifyPodTermination(v1.Pod{}).(*multierror.Error) + suite.Require().Error(err) + suite.Require().Len(err.Errors, 1) +} + +func TestNotifierSuite(t *testing.T) { + suite.Run(t, new(NotifierSuite)) +} diff --git a/notifier/slack.go b/notifier/slack.go new file mode 100644 index 00000000..76649540 --- /dev/null +++ b/notifier/slack.go @@ -0,0 +1,109 @@ +package notifier + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "time" + + v1 "k8s.io/api/core/v1" +) + +const NotifierSlack = "slack" + +var NotificationColor = "#F35A00" +var DefaultTimeout = 10 * time.Second + +type Slack struct { + Webhook string + Client *http.Client +} + +type slackMessage struct { + Message string `json:"text"` + Attachments []attachment `json:"attachments"` +} + +type slackField struct { + Title string `yaml:"title,omitempty" json:"title,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty"` + Short *bool `yaml:"short,omitempty" json:"short,omitempty"` +} + +type attachment struct { + Title string `json:"title,omitempty"` + TitleLink string `json:"title_link,omitempty"` + Pretext string `json:"pretext,omitempty"` + Text string `json:"text"` + Fallback string `json:"fallback"` + CallbackID string `json:"callback_id"` + Fields []slackField `json:"fields,omitempty"` + ImageURL string `json:"image_url,omitempty"` + ThumbURL string `json:"thumb_url,omitempty"` + Footer string `json:"footer"` + Color string `json:"color,omitempty"` + MrkdwnIn []string `json:"mrkdwn_in,omitempty"` +} + +func NewSlackNotifier(webhook string) *Slack { + return &Slack{ + Webhook: webhook, + Client: &http.Client{Timeout: DefaultTimeout}, + } +} + +func (s Slack) NotifyPodTermination(pod v1.Pod) error { + title := "Chaos event - Pod termination" + text := fmt.Sprintf("pod %s has been selected by chaos-kube for termination", pod.Name) + + short := len(pod.Namespace) < 20 && len(pod.Name) < 20 + fields := []slackField{ + { + Title: "namespace", + Value: pod.Namespace, + Short: &short, + }, + { + Title: "pod", + Value: pod.Name, + Short: &short, + }, + } + + message := createSlackRequest(title, text, fields) + return s.sendSlackMessage(message) +} + +func createSlackRequest(title string, text string, fields []slackField) slackMessage { + return slackMessage{ + Attachments: []attachment{{ + Title: title, + Text: text, + Footer: "chaos-kube", + Color: NotificationColor, + Fields: fields, + }}, + } +} + +func (s Slack) sendSlackMessage(message slackMessage) error { + messageBody, err := json.Marshal(message) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPost, s.Webhook, bytes.NewBuffer(messageBody)) + if err != nil { + return err + } + req.Header.Add("Content-Type", "application/json") + res, err := s.Client.Do(req) + if err != nil { + return err + } + if res.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code %d from slack webhook %s", res.StatusCode, s.Webhook) + } + + return nil +} diff --git a/notifier/slack_test.go b/notifier/slack_test.go new file mode 100644 index 00000000..a001b7bb --- /dev/null +++ b/notifier/slack_test.go @@ -0,0 +1,60 @@ +package notifier + +import ( + "net/http" + "net/http/httptest" + "testing" + + v1 "k8s.io/api/core/v1" + + "github.com/linki/chaoskube/internal/testutil" + "github.com/linki/chaoskube/util" + + "github.com/stretchr/testify/suite" +) + +type SlackSuite struct { + testutil.TestSuite +} + +func (suite *SlackSuite) TestSlackNotificationForTerminationStatusOk() { + webhookPath := "/services/T07M5HUDA/BQ1U5VDGA/yhpIczRK0cZ3jDLK1U8qD634" + + testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + suite.Require().Equal(webhookPath, req.URL.Path) + res.WriteHeader(200) + _, err := res.Write([]byte("ok")) + suite.Require().NoError(err) + })) + defer testServer.Close() + + testPod := util.NewPod("chaos", "chaos-57df4db6b-h9ktj", v1.PodRunning) + + slack := NewSlackNotifier(testServer.URL + webhookPath) + err := slack.NotifyPodTermination(testPod) + + suite.NoError(err) +} + +func (suite *SlackSuite) TestSlackNotificationForTerminationStatus500() { + webhookPath := "/services/T07M5HUDA/BQ1U5VDGA/yhpIczRK0cZ3jDLK1U8qD634" + + testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + suite.Require().Equal(webhookPath, req.URL.Path) + res.WriteHeader(500) + _, err := res.Write([]byte("ok")) + suite.Require().NoError(err) + })) + defer testServer.Close() + + testPod := util.NewPod("chaos", "chaos-57df4db6b-h9ktj", v1.PodRunning) + + slack := NewSlackNotifier(testServer.URL + webhookPath) + err := slack.NotifyPodTermination(testPod) + + suite.Error(err) +} + +func TestSlackSuite(t *testing.T) { + suite.Run(t, new(SlackSuite)) +}