Skip to content

Commit

Permalink
Merge pull request #158 from GaruGaru/master
Browse files Browse the repository at this point in the history
Slack notification integration
  • Loading branch information
linki authored Dec 9, 2019
2 parents 8947011 + f6d6d5c commit 3ec7983
Show file tree
Hide file tree
Showing 9 changed files with 364 additions and 2 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
* [#158](https://github.com/linki/chaoskube/pull/158) Support for sending Slack notifications @GaruGaru

## v0.16.0 - 2019-11-08

Features:
Expand Down
11 changes: 9 additions & 2 deletions chaoskube/chaoskube.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"k8s.io/client-go/tools/reference"

"github.com/linki/chaoskube/metrics"
"github.com/linki/chaoskube/notifier"
"github.com/linki/chaoskube/terminator"
"github.com/linki/chaoskube/util"
)
Expand Down Expand Up @@ -67,6 +68,8 @@ type Chaoskube struct {
Now func() time.Time

MaxKill int
// chaos events notifier
Notifier notifier.Notifier
}

var (
Expand All @@ -90,7 +93,7 @@ var (
// * a logger implementing logrus.FieldLogger to send log output to
// * what specific terminator to use to imbue chaos on victim pods
// * whether to enable/disable dry-run mode
func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int) *Chaoskube {
func New(client kubernetes.Interface, labels, annotations, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier) *Chaoskube {
broadcaster := record.NewBroadcaster()
broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(v1.NamespaceAll)})
recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"})
Expand All @@ -114,6 +117,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces, namespace
EventRecorder: recorder,
Now: time.Now,
MaxKill: maxKill,
Notifier: notifier,
}
}

Expand Down Expand Up @@ -175,7 +179,6 @@ func (c *Chaoskube) TerminateVictims() error {
for _, victim := range victims {
err = c.DeletePod(victim)
result = multierror.Append(result, err)

}

return result.ErrorOrNil()
Expand Down Expand Up @@ -257,6 +260,10 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error {

c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.")

if err := c.Notifier.NotifyPodTermination(victim); err != nil {
c.Logger.WithField("err", err).Warn("failed to notify pod termination")
}

return nil
}

Expand Down
33 changes: 33 additions & 0 deletions chaoskube/chaoskube_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"k8s.io/client-go/kubernetes/fake"

"github.com/linki/chaoskube/internal/testutil"
"github.com/linki/chaoskube/notifier"
"github.com/linki/chaoskube/terminator"
"github.com/linki/chaoskube/util"

Expand All @@ -35,6 +36,7 @@ type podInfo struct {

var (
logger, logOutput = test.NewNullLogger()
testNotifier = &notifier.Noop{}
)

func (suite *Suite) SetupTest() {
Expand All @@ -59,6 +61,7 @@ func (suite *Suite) TestNew() {
dryRun = true
terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second)
maxKill = 1
notifier = testNotifier
)

chaoskube := New(
Expand All @@ -78,6 +81,7 @@ func (suite *Suite) TestNew() {
dryRun,
terminator,
maxKill,
notifier,
)
suite.Require().NotNil(chaoskube)

Expand Down Expand Up @@ -723,6 +727,10 @@ func (suite *Suite) assertVictim(chaoskube *Chaoskube, expected map[string]strin
suite.assertVictims(chaoskube, []map[string]string{expected})
}

func (suite *Suite) assertNotified(notifier *notifier.Noop) {
suite.Assert().Greater(notifier.Calls, 0)
}

func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration) *Chaoskube {
chaoskube := suite.setup(
labelSelector,
Expand Down Expand Up @@ -797,6 +805,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele
dryRun,
terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod),
maxKill,
testNotifier,
)
}

Expand Down Expand Up @@ -971,3 +980,27 @@ func (suite *Suite) TestFilterByOwnerReference() {
}
}
}

func (suite *Suite) TestNotifierCall() {
chaoskube := suite.setupWithPods(
labels.Everything(),
labels.Everything(),
labels.Everything(),
labels.Everything(),
&regexp.Regexp{},
&regexp.Regexp{},
[]time.Weekday{},
[]util.TimePeriod{},
[]time.Time{},
time.UTC,
time.Duration(0),
false,
10,
)

victim := util.NewPod("default", "foo", v1.PodRunning)
err := chaoskube.DeletePod(victim)

suite.Require().NoError(err)
suite.assertNotified(testNotifier)
}
16 changes: 16 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"k8s.io/klog"

"github.com/linki/chaoskube/chaoskube"
"github.com/linki/chaoskube/notifier"
"github.com/linki/chaoskube/terminator"
"github.com/linki/chaoskube/util"
)
Expand Down Expand Up @@ -56,6 +57,7 @@ var (
gracePeriod time.Duration
logFormat string
logCaller bool
slackWebhook string
)

func init() {
Expand Down Expand Up @@ -83,6 +85,7 @@ func init() {
kingpin.Flag("grace-period", "Grace period to terminate Pods. Negative values will use the Pod's grace period.").Default("-1s").DurationVar(&gracePeriod)
kingpin.Flag("log-format", "Specify the format of the log messages. Options are text and json. Defaults to text.").Default("text").EnumVar(&logFormat, "text", "json")
kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").BoolVar(&logCaller)
kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").StringVar(&slackWebhook)
}

func main() {
Expand Down Expand Up @@ -123,6 +126,7 @@ func main() {
"metricsAddress": metricsAddress,
"gracePeriod": gracePeriod,
"logFormat": logFormat,
"slackWebhook": slackWebhook,
}).Debug("reading config")

log.WithFields(log.Fields{
Expand Down Expand Up @@ -191,6 +195,8 @@ func main() {
"offset": offset / int(time.Hour/time.Second),
}).Info("setting timezone")

notifiers := createNotifier()

chaoskube := chaoskube.New(
client,
labelSelector,
Expand All @@ -208,6 +214,7 @@ func main() {
dryRun,
terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod),
maxKill,
notifiers,
)

if metricsAddress != "" {
Expand Down Expand Up @@ -277,6 +284,15 @@ func parseSelector(str string) labels.Selector {
return selector
}

func createNotifier() notifier.Notifier {
notifiers := notifier.New()
if slackWebhook != "" {
notifiers.Add(notifier.NewSlackNotifier(slackWebhook))
}

return notifiers
}

func serveMetrics() {
http.Handle("/metrics", promhttp.Handler())
http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
Expand Down
16 changes: 16 additions & 0 deletions notifier/noop.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package notifier

import (
v1 "k8s.io/api/core/v1"
)

const NotifierNoop = "noop"

type Noop struct {
Calls int
}

func (t *Noop) NotifyPodTermination(pod v1.Pod) error {
t.Calls++
return nil
}
32 changes: 32 additions & 0 deletions notifier/notifier.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package notifier

import (
multierror "github.com/hashicorp/go-multierror"
v1 "k8s.io/api/core/v1"
)

type Notifier interface {
NotifyPodTermination(pod v1.Pod) error
}

type Notifiers struct {
notifiers []Notifier
}

func New() *Notifiers {
return &Notifiers{notifiers: make([]Notifier, 0)}
}

func (m *Notifiers) NotifyPodTermination(pod v1.Pod) error {
var result error
for _, n := range m.notifiers {
if err := n.NotifyPodTermination(pod); err != nil {
result = multierror.Append(result, err)
}
}
return result
}

func (m *Notifiers) Add(notifier Notifier) {
m.notifiers = append(m.notifiers, notifier)
}
87 changes: 87 additions & 0 deletions notifier/notifier_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package notifier

import (
"fmt"
"github.com/hashicorp/go-multierror"
"testing"

v1 "k8s.io/api/core/v1"

"github.com/linki/chaoskube/internal/testutil"

"github.com/stretchr/testify/suite"
)

type NotifierSuite struct {
testutil.TestSuite
}

type FailingNotifier struct{}

func (f FailingNotifier) NotifyPodTermination(pod v1.Pod) error {
return fmt.Errorf("notify error")
}

func (suite *NotifierSuite) TestMultiNotifierWithoutNotifiers() {
manager := New()
err := manager.NotifyPodTermination(v1.Pod{})
suite.NoError(err)
}

func (suite *NotifierSuite) TestMultiNotifierWithNotifier() {
manager := New()
n := Noop{}
manager.Add(&n)
err := manager.NotifyPodTermination(v1.Pod{})
suite.Require().NoError(err)

suite.Equal(1, n.Calls)
}

func (suite *NotifierSuite) TestMultiNotifierWithMultipleNotifier() {
manager := New()
n1 := Noop{}
n2 := Noop{}
manager.Add(&n1)
manager.Add(&n2)

err := manager.NotifyPodTermination(v1.Pod{})
suite.Require().NoError(err)

suite.Equal(1, n1.Calls)
suite.Equal(1, n2.Calls)
}

func (suite *NotifierSuite) TestMultiNotifierWithNotifierError() {
manager := New()
f := FailingNotifier{}
manager.Add(&f)
err := manager.NotifyPodTermination(v1.Pod{})
suite.Require().Error(err)
}

func (suite *NotifierSuite) TestMultiNotifierWithNotifierMultipleError() {
manager := New()
f0 := FailingNotifier{}
f1 := FailingNotifier{}
manager.Add(&f0)
manager.Add(&f1)
err := manager.NotifyPodTermination(v1.Pod{}).(*multierror.Error)
suite.Require().Error(err)
suite.Require().Len(err.Errors, 2)
}

func (suite *NotifierSuite) TestMultiNotifierWithOneFailingNotifier() {
manager := New()
f := FailingNotifier{}
n := Noop{}
manager.Add(&n)
manager.Add(&f)
err := manager.NotifyPodTermination(v1.Pod{}).(*multierror.Error)
suite.Require().Error(err)
suite.Require().Len(err.Errors, 1)
}

func TestNotifierSuite(t *testing.T) {
suite.Run(t, new(NotifierSuite))
}
Loading

0 comments on commit 3ec7983

Please sign in to comment.