Slack notification integration

CHANGELOG.md

@@ -1,3 +1,5 @@
+  * [#158](https://github.com/linki/chaoskube/pull/158) Support for sending Slack notifications @GaruGaru
+
 ## v0.16.0 - 2019-11-08
 
   Features:

chaoskube/chaoskube.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/client-go/tools/reference"
 
 	"github.com/linki/chaoskube/metrics"
+	"github.com/linki/chaoskube/notifier"
 	"github.com/linki/chaoskube/terminator"
 	"github.com/linki/chaoskube/util"
 )
@@ -67,6 +68,8 @@ type Chaoskube struct {
 	Now func() time.Time
 
 	MaxKill int
+	// chaos events notifier
+	Notifier notifier.Notifier
 }
 
 var (
@@ -90,7 +93,7 @@ var (
 // * a logger implementing logrus.FieldLogger to send log output to
 // * what specific terminator to use to imbue chaos on victim pods
 // * whether to enable/disable dry-run mode
-func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int) *Chaoskube {
+func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube {
 	broadcaster := record.NewBroadcaster()
 	broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)})
 	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"})
@@ -114,6 +117,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces, namespace
 		EventRecorder:      recorder,
 		Now:                time.Now,
 		MaxKill:            maxKill,
+		Notifier:           notifier,
 	}
 }
 
@@ -175,7 +179,6 @@ func (c *Chaoskube) TerminateVictims() error {
 	for _, victim := range victims {
 		err = c.DeletePod(victim)
 		result = multierror.Append(result, err)
-
 	}
 
 	return result.ErrorOrNil()
@@ -257,6 +260,10 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error {
 
 	c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.")
 
+	if err := c.Notifier.NotifyPodTermination(victim); err != nil {
+		c.Logger.WithField("err", err).Warn("failed to notify pod termination")
+	}
+
 	return nil
 }
 

chaoskube/chaoskube_test.go

@@ -17,6 +17,7 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 
 	"github.com/linki/chaoskube/internal/testutil"
+	"github.com/linki/chaoskube/notifier"
 	"github.com/linki/chaoskube/terminator"
 	"github.com/linki/chaoskube/util"
 
@@ -35,6 +36,7 @@ type podInfo struct {
 
 var (
 	logger, logOutput = test.NewNullLogger()
+	testNotifier      = &notifier.Noop{}
 )
 
 func (suite *Suite) SetupTest() {
@@ -59,6 +61,7 @@ func (suite *Suite) TestNew() {
 		dryRun             = true
 		terminator         = terminator.NewDeletePodTerminator(client, logger, 10*time.Second)
 		maxKill            = 1
+		notifier           = testNotifier
 	)
 
 	chaoskube := New(
@@ -78,6 +81,7 @@ func (suite *Suite) TestNew() {
 		dryRun,
 		terminator,
 		maxKill,
+		notifier,
 	)
 	suite.Require().NotNil(chaoskube)
 
@@ -723,6 +727,10 @@ func (suite *Suite) assertVictim(chaoskube *Chaoskube, expected map[string]strin
 	suite.assertVictims(chaoskube, []map[string]string{expected})
 }
 
+func (suite *Suite) assertNotified(notifier *notifier.Noop) {
+	suite.Assert().Greater(notifier.Calls, 0)
+}
+
 func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube {
 	chaoskube := suite.setup(
 		labelSelector,
@@ -797,6 +805,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele
 		dryRun,
 		terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod),
 		maxKill,
+		testNotifier,
 	)
 }
 
@@ -971,3 +980,27 @@ func (suite *Suite) TestFilterByOwnerReference() {
 		}
 	}
 }
+
+func (suite *Suite) TestNotifierCall() {
+	chaoskube := suite.setupWithPods(
+		labels.Everything(),
+		labels.Everything(),
+		labels.Everything(),
+		labels.Everything(),
+		&regexp.Regexp{},
+		&regexp.Regexp{},
+		[]time.Weekday{},
+		[]util.TimePeriod{},
+		[]time.Time{},
+		time.UTC,
+		time.Duration(0),
+		false,
+		10,
+	)
+
+	victim := util.NewPod("default", "foo", v1.PodRunning)
+	err := chaoskube.DeletePod(victim)
+
+	suite.Require().NoError(err)
+	suite.assertNotified(testNotifier)
+}

main.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/klog"
 
 	"github.com/linki/chaoskube/chaoskube"
+	"github.com/linki/chaoskube/notifier"
 	"github.com/linki/chaoskube/terminator"
 	"github.com/linki/chaoskube/util"
 )
@@ -56,6 +57,7 @@ var (
 	gracePeriod        time.Duration
 	logFormat          string
 	logCaller          bool
+	slackWebhook       string
 )
 
 func init() {
@@ -83,6 +85,7 @@ func init() {
 	kingpin.Flag("grace-period", "Grace period to terminate Pods. Negative values will use the Pod's grace period.").Default("-1s").DurationVar(&gracePeriod)
 	kingpin.Flag("log-format", "Specify the format of the log messages. Options are text and json. Defaults to text.").Default("text").EnumVar(&logFormat, "text", "json")
 	kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").BoolVar(&logCaller)
+	kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").StringVar(&slackWebhook)
 }
 
 func main() {
@@ -123,6 +126,7 @@ func main() {
 		"metricsAddress":     metricsAddress,
 		"gracePeriod":        gracePeriod,
 		"logFormat":          logFormat,
+		"slackWebhook":       slackWebhook,
 	}).Debug("reading config")
 
 	log.WithFields(log.Fields{
@@ -191,6 +195,8 @@ func main() {
 		"offset":   offset / int(time.Hour/time.Second),
 	}).Info("setting timezone")
 
+	notifiers := createNotifier()
+
 	chaoskube := chaoskube.New(
 		client,
 		labelSelector,
@@ -208,6 +214,7 @@ func main() {
 		dryRun,
 		terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod),
 		maxKill,
+		notifiers,
 	)
 
 	if metricsAddress != "" {
@@ -277,6 +284,15 @@ func parseSelector(str string) labels.Selector {
 	return selector
 }
 
+func createNotifier() notifier.Notifier {
+	notifiers := notifier.New()
+	if slackWebhook != "" {
+		notifiers.Add(notifier.NewSlackNotifier(slackWebhook))
+	}
+
+	return notifiers
+}
+
 func serveMetrics() {
 	http.Handle("/metrics", promhttp.Handler())
 	http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {

notifier/noop.go

@@ -0,0 +1,16 @@
+package notifier
+
+import (
+	v1 "k8s.io/api/core/v1"
+)
+
+const NotifierNoop = "noop"
+
+type Noop struct {
+	Calls int
+}
+
+func (t *Noop) NotifyPodTermination(pod v1.Pod) error {
+	t.Calls++
+	return nil
+}

notifier/notifier.go

@@ -0,0 +1,32 @@
+package notifier
+
+import (
+	multierror "github.com/hashicorp/go-multierror"
+	v1 "k8s.io/api/core/v1"
+)
+
+type Notifier interface {
+	NotifyPodTermination(pod v1.Pod) error
+}
+
+type Notifiers struct {
+	notifiers []Notifier
+}
+
+func New() *Notifiers {
+	return &Notifiers{notifiers: make([]Notifier, 0)}
+}
+
+func (m *Notifiers) NotifyPodTermination(pod v1.Pod) error {
+	var result error
+	for _, n := range m.notifiers {
+		if err := n.NotifyPodTermination(pod); err != nil {
+			result = multierror.Append(result, err)
+		}
+	}
+	return result
+}
+
+func (m *Notifiers) Add(notifier Notifier) {
+	m.notifiers = append(m.notifiers, notifier)
+}

notifier/notifier_test.go

@@ -0,0 +1,87 @@
+package notifier
+
+import (
+	"fmt"
+	"github.com/hashicorp/go-multierror"
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/linki/chaoskube/internal/testutil"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type NotifierSuite struct {
+	testutil.TestSuite
+}
+
+type FailingNotifier struct{}
+
+func (f FailingNotifier) NotifyPodTermination(pod v1.Pod) error {
+	return fmt.Errorf("notify error")
+}
+
+func (suite *NotifierSuite) TestMultiNotifierWithoutNotifiers() {
+	manager := New()
+	err := manager.NotifyPodTermination(v1.Pod{})
+	suite.NoError(err)
+}
+
+func (suite *NotifierSuite) TestMultiNotifierWithNotifier() {
+	manager := New()
+	n := Noop{}
+	manager.Add(&n)
+	err := manager.NotifyPodTermination(v1.Pod{})
+	suite.Require().NoError(err)
+
+	suite.Equal(1, n.Calls)
+}
+
+func (suite *NotifierSuite) TestMultiNotifierWithMultipleNotifier() {
+	manager := New()
+	n1 := Noop{}
+	n2 := Noop{}
+	manager.Add(&n1)
+	manager.Add(&n2)
+
+	err := manager.NotifyPodTermination(v1.Pod{})
+	suite.Require().NoError(err)
+
+	suite.Equal(1, n1.Calls)
+	suite.Equal(1, n2.Calls)
+}
+
+func (suite *NotifierSuite) TestMultiNotifierWithNotifierError() {
+	manager := New()
+	f := FailingNotifier{}
+	manager.Add(&f)
+	err := manager.NotifyPodTermination(v1.Pod{})
+	suite.Require().Error(err)
+}
+
+func (suite *NotifierSuite) TestMultiNotifierWithNotifierMultipleError() {
+	manager := New()
+	f0 := FailingNotifier{}
+	f1 := FailingNotifier{}
+	manager.Add(&f0)
+	manager.Add(&f1)
+	err := manager.NotifyPodTermination(v1.Pod{}).(*multierror.Error)
+	suite.Require().Error(err)
+	suite.Require().Len(err.Errors, 2)
+}
+
+func (suite *NotifierSuite) TestMultiNotifierWithOneFailingNotifier() {
+	manager := New()
+	f := FailingNotifier{}
+	n := Noop{}
+	manager.Add(&n)
+	manager.Add(&f)
+	err := manager.NotifyPodTermination(v1.Pod{}).(*multierror.Error)
+	suite.Require().Error(err)
+	suite.Require().Len(err.Errors, 1)
+}
+
+func TestNotifierSuite(t *testing.T) {
+	suite.Run(t, new(NotifierSuite))
+}