feature/grace seconds

alert.go

@@ -95,7 +95,7 @@ func (a *Alert) shouldAlert() bool {
 		return false
 	}
 	t := NewThrottler()
-	return !t.IsThrottled(a.Error)
+	return !t.IsThrottledOrGraced(a.Error)
 }
 
 func (a *Alert) isDoNotAlert() bool {

alert_test.go

@@ -159,6 +159,14 @@ func TestAlert_shouldAlert(t *testing.T) {
 			},
 			want: true,
 		},
+		{name: "shouldAlert_graced_false",
+			fields: fields{
+				Error: errors.New("alert this"),
+				DoNotAlertErrors: []error{
+					errors.New("do not alert"), errors.New("if this error then don't alert")},
+			},
+			want: false,
+		},
 		{name: "shouldAlert_true_disable_throttling",
 			fields: fields{
 				Error: errors.New("do not alert"),
@@ -173,14 +181,18 @@ func TestAlert_shouldAlert(t *testing.T) {
 			if tt.name == "shouldAlert_true_disable_throttling" {
 				os.Setenv("THROTTLE_ENABLED", "false")
 			}
+			if tt.name == "shouldAlert_graced_false" {
+				os.Setenv("THROTTLE_GRACE_SECONDS", "20")
+			}
 			a := &Alert{
 				Error:            tt.fields.Error,
 				DoNotAlertErrors: tt.fields.DoNotAlertErrors,
 			}
 			if err := a.RemoveCurrentThrotting(); err != nil {
 				t.Errorf("Alert.Notify() error = %+v", err)
 			}
-			if got := a.shouldAlert(); got != tt.want {
+			got := a.shouldAlert()
+			if got != tt.want {
 				t.Errorf("Alert.shouldAlert() = %v, want %v", got, tt.want)
 			}
 		})

throttler.go

@@ -13,6 +13,7 @@ import (
 type Throttler struct {
 	CacheOpt         string
 	ThrottleDuration int
+	GraceDuration    int
 }
 
 // ErrorOccurrence store error time and error
@@ -27,6 +28,7 @@ func NewThrottler() Throttler {
 	t := Throttler{
 		CacheOpt:         fmt.Sprintf("/tmp/cache/%v_throttler_disk_cache", os.Getenv("APP_NAME")),
 		ThrottleDuration: 5, // default 5mn
+		GraceDuration:    0, // default 0sc
 	}
 	if len(os.Getenv("THROTTLE_DURATION")) != 0 {
 		duration, err := strconv.Atoi(os.Getenv("THROTTLE_DURATION"))
@@ -35,6 +37,13 @@ func NewThrottler() Throttler {
 		}
 		t.ThrottleDuration = duration
 	}
+	if len(os.Getenv("THROTTLE_GRACE_SECONDS")) != 0 {
+		grace, err := strconv.Atoi(os.Getenv("THROTTLE_GRACE_SECONDS"))
+		if err != nil {
+			return t
+		}
+		t.GraceDuration = grace
+	}
 
 	if len(os.Getenv("THROTTLE_DISKCACHE_DIR")) != 0 {
 		t.CacheOpt = os.Getenv("THROTTLE_DISKCACHE_DIR")
@@ -44,17 +53,28 @@ func NewThrottler() Throttler {
 }
 
 // IsThrottled checks if the error has been throttled. If not, throttle it
-func (t *Throttler) IsThrottled(ocError error) bool {
+func (t *Throttler) IsThrottledOrGraced(ocError error) bool {
 	dc, err := t.getDiskCache()
 	if err != nil {
 		return false
 	}
-	cachedTime, throttled := dc.Get(ocError.Error())
+	cachedThrottleTime, throttled := dc.Get(ocError.Error())
+	cachedDetectionTime, graced := dc.Get(fmt.Sprintf("%v_detectionTime", ocError.Error()))
 
-	if throttled && !isOverThrottleDuration(string(cachedTime), t.ThrottleDuration) {
+	throttleIsOver := isOverThrottleDuration(string(cachedThrottleTime), t.ThrottleDuration)
+	if throttled && !throttleIsOver {
 		// already throttled and not over throttling duration, do nothing
 		return true
 	}
+
+	if !graced || isOverGracePlusThrottleDuration(string(cachedDetectionTime), t.GraceDuration, t.ThrottleDuration) {
+		cachedDetectionTime = t.InitGrace(ocError)
+	}
+	if cachedDetectionTime != nil && !isOverGraceDuration(string(cachedDetectionTime), t.GraceDuration) {
+		// grace duration is not over yet, do nothing
+		return true
+	}
+
 	// if it has not throttled yet or over throttle duration, throttle it and return false to send notification
 	// Rethrottler will also renew the timestamp in the throttler cache.
 	if err = t.ThrottleError(ocError); err != nil {
@@ -63,14 +83,35 @@ func (t *Throttler) IsThrottled(ocError error) bool {
 	return false
 }
 
+func isOverGracePlusThrottleDuration(cachedTime string, graceDurationInSec int, throttleDurationInMin int) bool {
+	detectionTime, err := time.Parse(time.RFC3339, string(cachedTime))
+	if err != nil {
+		return false
+	}
+	now := time.Now()
+	diff := int(now.Sub(detectionTime).Seconds())
+	overallDurationInSec := graceDurationInSec + throttleDurationInMin*60
+	return diff >= overallDurationInSec
+}
+
+func isOverGraceDuration(cachedTime string, graceDuration int) bool {
+	detectionTime, err := time.Parse(time.RFC3339, string(cachedTime))
+	if err != nil {
+		return false
+	}
+	now := time.Now()
+	diff := int(now.Sub(detectionTime).Seconds())
+	return diff >= graceDuration
+}
+
 func isOverThrottleDuration(cachedTime string, throttleDuration int) bool {
 	throttledTime, err := time.Parse(time.RFC3339, string(cachedTime))
 	if err != nil {
 		return false
 	}
 	now := time.Now()
 	diff := int(now.Sub(throttledTime).Minutes())
-	return diff > throttleDuration
+	return diff >= throttleDuration
 }
 
 // ThrottleError throttle the alert within the limited duration
@@ -79,12 +120,29 @@ func (t *Throttler) ThrottleError(errObj error) error {
 	if err != nil {
 		return err
 	}
+
 	now := time.Now().Format(time.RFC3339)
 	err = dc.Set(errObj.Error(), []byte(now))
 
 	return err
 }
 
+// ThrottleError throttle the alert within the limited duration
+func (t *Throttler) InitGrace(errObj error) []byte {
+	dc, err := t.getDiskCache()
+	if err != nil {
+		return nil
+	}
+	now := time.Now().Format(time.RFC3339)
+	cachedDetectionTime := []byte(now)
+	err = dc.Set(fmt.Sprintf("%v_detectionTime", errObj.Error()), cachedDetectionTime)
+	if err != nil {
+		return nil
+	}
+
+	return cachedDetectionTime
+}
+
 // CleanThrottlingCache clean all the diskcache in throttling cache directory
 func (t *Throttler) CleanThrottlingCache() (err error) {
 	dc, err := t.getDiskCache()

throttler_test.go

@@ -21,42 +21,51 @@ func TestNewThrottler(t *testing.T) {
 			want: Throttler{
 				CacheOpt:         fmt.Sprintf("/tmp/cache/%v_throttler_disk_cache", os.Getenv("APP_NAME")),
 				ThrottleDuration: 5,
+				GraceDuration:    0,
 			},
 		},
 		{
 			name: "change duration",
 			want: Throttler{
 				CacheOpt:         fmt.Sprintf("/tmp/cache/%v_throttler_disk_cache", os.Getenv("APP_NAME")),
 				ThrottleDuration: 7,
+				GraceDuration:    5,
 			},
 		},
 		{
 			name: "change both",
 			want: Throttler{
 				CacheOpt:         "new_cache_dir",
 				ThrottleDuration: 8,
+				GraceDuration:    0,
 			},
 		},
 	}
 	for _, tt := range tests {
 		if tt.name == "change duration" {
 			os.Setenv("THROTTLE_DURATION", "7")
+			os.Setenv("THROTTLE_GRACE_SECONDS", "5")
 		} else if tt.name == "change both" {
 			os.Setenv("THROTTLE_DURATION", "8")
+			os.Setenv("THROTTLE_GRACE_SECONDS", "0")
 			os.Setenv("THROTTLE_DISKCACHE_DIR", "new_cache_dir")
+		} else if tt.name == "default" {
+			os.Setenv("THROTTLE_GRACE_SECONDS", "")
 		}
 		t.Run(tt.name, func(t *testing.T) {
-			if got := NewThrottler(); !reflect.DeepEqual(got, tt.want) {
+			got := NewThrottler()
+			if !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("NewThrottler() = %v, want %v", got, tt.want)
 			}
 		})
 	}
 }
 
-func TestThrottler_IsThrottled(t *testing.T) {
+func TestThrottler_IsThrottledOrGraced(t *testing.T) {
 	type fields struct {
 		CacheOpt         string
 		ThrottleDuration int
+		GraceDuration    int
 	}
 	type args struct {
 		ocError error
@@ -72,6 +81,7 @@ func TestThrottler_IsThrottled(t *testing.T) {
 			fields: fields{
 				CacheOpt:         fmt.Sprintf("/tmp/cache/%v_throttler_disk_cache", os.Getenv("APP_NAME")),
 				ThrottleDuration: 5,
+				GraceDuration:    0,
 			},
 			args: args{
 				ocError: errors.New("test_throttling"),
@@ -83,6 +93,19 @@ func TestThrottler_IsThrottled(t *testing.T) {
 			fields: fields{
 				CacheOpt:         fmt.Sprintf("/tmp/cache/%v_throttler_disk_cache", os.Getenv("APP_NAME")),
 				ThrottleDuration: 5,
+				GraceDuration:    0,
+			},
+			args: args{
+				ocError: errors.New("test_throttling"),
+			},
+			want: true,
+		},
+		{
+			name: "graced_true",
+			fields: fields{
+				CacheOpt:         fmt.Sprintf("/tmp/cache/%v_throttler_disk_cache", os.Getenv("APP_NAME")),
+				ThrottleDuration: 5,
+				GraceDuration:    25,
 			},
 			args: args{
 				ocError: errors.New("test_throttling"),
@@ -95,14 +118,14 @@ func TestThrottler_IsThrottled(t *testing.T) {
 			th := &Throttler{
 				CacheOpt:         tt.fields.CacheOpt,
 				ThrottleDuration: tt.fields.ThrottleDuration,
+				GraceDuration:    tt.fields.GraceDuration,
 			}
 			if tt.name == "throttled_true" {
 				if err := th.ThrottleError(tt.args.ocError); err != nil {
 					t.Errorf("testing failed : %+v", err)
 				}
-
 			}
-			if got := th.IsThrottled(tt.args.ocError); got != tt.want {
+			if got := th.IsThrottledOrGraced(tt.args.ocError); got != tt.want {
 				t.Errorf("Throttler.IsThrottled() = %v, want %v", got, tt.want)
 			}
 			err := th.CleanThrottlingCache()
@@ -118,6 +141,7 @@ func TestThrottler_ThrottleError(t *testing.T) {
 	type fields struct {
 		CacheOpt         string
 		ThrottleDuration int
+		GraceDuration    int
 	}
 	type args struct {
 		errObj error
@@ -133,6 +157,7 @@ func TestThrottler_ThrottleError(t *testing.T) {
 			fields: fields{
 				CacheOpt:         fmt.Sprintf("/tmp/cache/%v_throttler_disk_cache", os.Getenv("APP_NAME")),
 				ThrottleDuration: 5,
+				GraceDuration:    0,
 			},
 			args: args{
 				errObj: errors.New("test_throttling"),
@@ -144,6 +169,7 @@ func TestThrottler_ThrottleError(t *testing.T) {
 			fields: fields{
 				CacheOpt:         "/no_permission_dir",
 				ThrottleDuration: 5,
+				GraceDuration:    0,
 			},
 			args: args{
 				errObj: errors.New("test_throttling"),
@@ -162,7 +188,7 @@ func TestThrottler_ThrottleError(t *testing.T) {
 			if err := th.ThrottleError(tt.args.errObj); (err != nil) != tt.wantErr {
 				t.Errorf("Throttler.ThrottleError() error = %v, wantErr %v", err, tt.wantErr)
 			}
-			if tt.name == "default" && !th.IsThrottled(tt.args.errObj) {
+			if tt.name == "default" && !th.IsThrottledOrGraced(tt.args.errObj) {
 				t.Errorf("Throttler.ThrottleError() error = %v, wantErr %v", errors.New("throttling failed"), tt.wantErr)
 			}
 			if !tt.wantErr {
@@ -229,6 +255,7 @@ func Test_isOverThrottleDuration(t *testing.T) {
 	type args struct {
 		cachedTime       string
 		throttleDuration int
+		graceDuration    int
 	}
 	tests := []struct {
 		name string
@@ -240,6 +267,7 @@ func Test_isOverThrottleDuration(t *testing.T) {
 			args: args{
 				cachedTime:       time.Now().Add(-3 * time.Minute).Format(time.RFC3339), // -3 minutes => pass 2 minutes durations
 				throttleDuration: 2,
+				graceDuration:    0,
 			},
 			want: true,
 		},
@@ -248,6 +276,7 @@ func Test_isOverThrottleDuration(t *testing.T) {
 			args: args{
 				cachedTime:       time.Now().Add(1 * time.Minute).Format(time.RFC3339), // 1 minute ahead of current < throtte duration 2
 				throttleDuration: 2,
+				graceDuration:    0,
 			},
 			want: false,
 		},
@@ -260,3 +289,43 @@ func Test_isOverThrottleDuration(t *testing.T) {
 		})
 	}
 }
+
+func Test_isOverGraceDuration(t *testing.T) {
+	type args struct {
+		cachedTime       string
+		throttleDuration int
+		graceDuration    int
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Test_isOverGraceDuration_true",
+			args: args{
+				cachedTime:       time.Now().Add(-5 * time.Second).Format(time.RFC3339), // 2 sec after grace duration is over
+				throttleDuration: 0,
+				graceDuration:    3,
+			},
+			want: true,
+		},
+		{
+			name: "Test_isOverGraceDuration_false",
+			args: args{
+				cachedTime:       time.Now().Add(2 * time.Second).Format(time.RFC3339), // still 8 sec left for grace duration
+				throttleDuration: 0,
+				graceDuration:    10,
+			},
+			want: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := isOverGraceDuration(tt.args.cachedTime, tt.args.graceDuration); got != tt.want {
+				t.Errorf("isOverGraceDuration() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+
+}