fix: canary scaling (#2247)

* fix: canary scaling

* fix: use pointers for replicas

* chore: go mod tidy

* chore: rename comment on replicas

* fix: don't add suspend annotations when replica is set to 0

api/v1/canary_types.go

@@ -41,7 +41,8 @@ const (
 type CanarySpec struct {
 	//+kubebuilder:default=1
 	//+optional
-	Replicas int `yaml:"replicas,omitempty" json:"replicas,omitempty"`
+	// Replicas pauses the canary if = 0.
+	Replicas *int `yaml:"replicas,omitempty" json:"replicas,omitempty"`
 
 	Env                map[string]VarSource      `yaml:"env,omitempty" json:"env,omitempty"`
 	HTTP               []HTTPCheck               `yaml:"http,omitempty" json:"http,omitempty"`
@@ -354,7 +355,7 @@ func (c Canary) GetKey(check external.Check) string {
 	if err != nil {
 		logger.Debugf("error marshalling the check")
 	}
-	var hash = md5.Sum(data)
+	hash := md5.Sum(data)
 	return fmt.Sprintf("%s/%s:%s/%s:%s", c.ID(), check.GetType(), check.GetName(), c.GetDescription(check), hex.EncodeToString(hash[:]))
 }
 

config/deploy/crd.yaml

@@ -11906,6 +11906,7 @@ spec:
                   type: array
                 replicas:
                   default: 1
+                  description: Replicas pauses the canary if = 0.
                   type: integer
                 restic:
                   items:

config/deploy/manifests.yaml

@@ -11905,6 +11905,7 @@ spec:
                   type: array
                 replicas:
                   default: 1
+                  description: Replicas pauses the canary if = 0.
                   type: integer
                 restic:
                   items:

go.mod

@@ -119,11 +119,9 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect
 	github.com/aws/smithy-go v1.20.4 // indirect
-	github.com/bahlo/generic-list-go v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
-	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
@@ -200,7 +198,6 @@ require (
 	github.com/hirochachacha/go-smb2 v1.1.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/invopop/jsonschema v0.12.0 // indirect
 	github.com/itchyny/gojq v0.12.16 // indirect
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 // indirect
@@ -275,14 +272,10 @@ require (
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
-	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
-	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
-	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect

go.sum

@@ -748,8 +748,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3Co
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
 github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
-github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
-github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -761,8 +759,6 @@ github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwN
 github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
 github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
-github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4=
 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
@@ -1176,8 +1172,6 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI=
-github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
 github.com/itchyny/gojq v0.12.13/go.mod h1:JzwzAqenfhrPUuwbmEz3nu3JQmFLlQTQMUcOdnu/Sf4=
 github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
 github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
@@ -1580,8 +1574,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
-github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
-github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -1590,12 +1582,6 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
 github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
-github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
-github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
-github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
-github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1v2SRTV4cUmp4=
 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=

pkg/controllers/canary_controller.go

@@ -139,22 +139,16 @@ func (r *CanaryReconciler) Reconcile(parentCtx gocontext.Context, req ctrl.Reque
 		}
 	}
 
-	if canaryForStatus.Status.Replicas != canary.Spec.Replicas {
-		if canary.Spec.Replicas == 0 {
+	if canary.Spec.Replicas != nil && canaryForStatus.Status.Replicas != *canary.Spec.Replicas {
+		if *canary.Spec.Replicas == 0 {
 			canaryJobs.Unschedule(canary.GetPersistedID())
-			if err := db.SuspendCanary(ctx, canary.GetPersistedID(), true); err != nil {
-				return ctrl.Result{Requeue: true, RequeueAfter: 2 * time.Minute}, err
-			}
 		} else {
-			if err := db.SuspendCanary(ctx, canary.GetPersistedID(), false); err != nil {
-				return ctrl.Result{Requeue: true, RequeueAfter: 2 * time.Minute}, err
-			}
 			if err := canaryJobs.SyncCanaryJob(ctx, *dbCanary); err != nil {
 				return ctrl.Result{Requeue: true, RequeueAfter: 2 * time.Minute}, err
 			}
 		}
 
-		canaryForStatus.Status.Replicas = canary.Spec.Replicas
+		canaryForStatus.Status.Replicas = *canary.Spec.Replicas
 	}
 
 	canaryForStatus.Status.Checks = dbCanary.Checks

pkg/db/canary.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-
 	"time"
 
 	apiContext "github.com/flanksource/canary-checker/api/context"
@@ -29,9 +28,7 @@ import (
 	"gorm.io/gorm/clause"
 )
 
-var (
-	PostgresDuplicateKeyError = &pgconn.PgError{Code: "23505"}
-)
+var PostgresDuplicateKeyError = &pgconn.PgError{Code: "23505"}
 
 func GetAllCanariesForSync(ctx context.Context, namespace string) ([]pkg.Canary, error) {
 	query := `
@@ -50,6 +47,7 @@ func GetAllCanariesForSync(ctx context.Context, namespace string) ([]pkg.Canary,
         WHERE
             deleted_at IS NULL AND
             agent_id = '00000000-0000-0000-0000-000000000000' AND
+            (spec->>'replicas' != '0' OR spec->'replicas' IS NULL) AND
 						(annotations->>'suspend' != 'true' OR annotations->>'suspend' IS NULL)
     `
 
@@ -430,7 +428,6 @@ func PersistCanaryModel(ctx context.Context, model pkg.Canary) (*pkg.Canary, boo
 		models.Canary{}.ConflictClause(),
 		clause.Returning{},
 	).Create(&model).Error
-
 	// Duplicate key happens when an already created canary is persisted
 	// We will ignore this error but act on other errors
 	// In this scenario PostgresDuplicateKeyError is checked primarily and
@@ -477,7 +474,7 @@ func PersistCanaryModel(ctx context.Context, model pkg.Canary) (*pkg.Canary, boo
 		return nil, false, err
 	}
 
-	var checks = make(map[string]string)
+	checks := make(map[string]string)
 	var newCheckIDs []string
 	for _, config := range spec.GetAllChecks() {
 		check := pkg.FromExternalCheck(model, config)
@@ -517,27 +514,3 @@ func PersistCanary(ctx context.Context, canary v1.Canary, source string) (*pkg.C
 
 	return PersistCanaryModel(ctx, model)
 }
-
-// SuspendCanary sets the suspend annotation on the canary table.
-func SuspendCanary(ctx context.Context, id string, suspend bool) error {
-	query := `
-	UPDATE canaries
-		SET annotations =
-			CASE
-				WHEN annotations IS NULL THEN '{"suspend": "true"}'::jsonb
-				ELSE jsonb_set(annotations, '{suspend}', '"true"')
-			END
-		WHERE id = ?;
-	`
-
-	if !suspend {
-		query = `
-		UPDATE canaries
-			SET annotations =
-				CASE WHEN annotations IS NOT NULL THEN annotations - 'suspend' END
-			WHERE id = ?;
-		`
-	}
-
-	return ctx.DB().Exec(query, id).Error
-}

pkg/jobs/canary/canary_jobs.go

@@ -65,7 +65,7 @@ func (j CanaryJob) GetNamespacedName() types.NamespacedName {
 }
 
 func (j CanaryJob) Run(ctx dutyjob.JobRuntime) error {
-	if runner.IsCanarySuspended(&j.Canary.ObjectMeta) || runner.IsCanaryIgnored(&j.Canary.ObjectMeta) {
+	if runner.IsCanarySuspended(j.Canary) || runner.IsCanaryIgnored(&j.Canary.ObjectMeta) {
 		return nil
 	}
 

pkg/jobs/canary/sync.go

@@ -94,7 +94,7 @@ func SyncCanaryJob(ctx context.Context, dbCanary pkg.Canary) error {
 		return nil
 	}
 
-	if runner.IsCanarySuspended(&canary.ObjectMeta) || runner.IsCanaryIgnored(&canary.ObjectMeta) {
+	if runner.IsCanarySuspended(*canary) || runner.IsCanaryIgnored(&canary.ObjectMeta) {
 		Unschedule(id)
 		return nil
 	}
@@ -208,7 +208,7 @@ func ScanCanaryConfigs(ctx context.Context) {
 		}
 
 		for _, canary := range configs {
-			if runner.IsCanarySuspended(&canary.ObjectMeta) || runner.IsCanaryIgnored(&canary.ObjectMeta) {
+			if runner.IsCanarySuspended(canary) || runner.IsCanaryIgnored(&canary.ObjectMeta) {
 				continue
 			}
 

pkg/runner/runner.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	v1 "github.com/flanksource/canary-checker/api/v1"
 	"github.com/flanksource/canary-checker/pkg/prometheus"
 	"github.com/flanksource/commons/collections"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -49,6 +50,7 @@ func IsCanaryIgnored(canary *metav1.ObjectMeta) bool {
 	return false
 }
 
-func IsCanarySuspended(canary *metav1.ObjectMeta) bool {
-	return canary.Annotations != nil && canary.Annotations["suspend"] == "true"
+func IsCanarySuspended(c v1.Canary) bool {
+	return (c.Spec.Replicas != nil && *c.Spec.Replicas == 0) ||
+		(c.ObjectMeta.Annotations != nil && c.ObjectMeta.Annotations["suspend"] == "true")
 }