diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2de363d5..faaef3cf 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,16 +1,19 @@ agents: queue: "public" +env: + GO_VERSION_FILE: "go1.22.3.linux-amd64.tar.gz" + # Mount the docker.sock as to the docker container, so that we are able to # run docker build command and kind is spawned as a sibling container. steps: - name: "Upgrade Test" command: - apk add g++ make bash gcompat curl mysql mysql-client libc6-compat - - wget https://golang.org/dl/go1.22.3.linux-amd64.tar.gz - - tar -C /usr/local -xzf go1.22.3.linux-amd64.tar.gz + - wget https://golang.org/dl/$GO_VERSION_FILE + - tar -C /usr/local -xzf $GO_VERSION_FILE - export PATH=$PATH:/usr/local/go/bin - - rm go1.22.3.linux-amd64.tar.gz + - rm $GO_VERSION_FILE - ln -s /lib/libc.so.6 /usr/lib/libresolv.so.2 - make upgrade-test concurrency: 1 @@ -26,10 +29,10 @@ steps: - name: "Backup Restore Test" command: - apk add g++ make bash gcompat curl mysql mysql-client libc6-compat - - wget https://golang.org/dl/go1.22.3.linux-amd64.tar.gz - - tar -C /usr/local -xzf go1.22.3.linux-amd64.tar.gz + - wget https://golang.org/dl/$GO_VERSION_FILE + - tar -C /usr/local -xzf $GO_VERSION_FILE - export PATH=$PATH:/usr/local/go/bin - - rm go1.22.3.linux-amd64.tar.gz + - rm $GO_VERSION_FILE - ln -s /lib/libc.so.6 /usr/lib/libresolv.so.2 - make backup-restore-test concurrency: 1 @@ -42,13 +45,32 @@ steps: volumes: - "/var/run/docker.sock:/var/run/docker.sock" + - name: "Backup Schedule Test" + command: + - apk add g++ make bash gcompat curl mysql mysql-client libc6-compat + - wget https://golang.org/dl/$GO_VERSION_FILE + - tar -C /usr/local -xzf $GO_VERSION_FILE + - export PATH=$PATH:/usr/local/go/bin + - rm $GO_VERSION_FILE + - ln -s /lib/libc.so.6 /usr/lib/libresolv.so.2 + - make backup-schedule-test + concurrency: 1 + concurrency_group: 'vtop/backup-schedule-test' + timeout_in_minutes: 60 + plugins: + - docker#v3.12.0: + image: "docker:latest" + propagate-environment: true + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + - name: "VTOrc and VTAdmin Test" command: - apk add g++ make bash gcompat curl mysql mysql-client libc6-compat chromium - - wget https://golang.org/dl/go1.22.3.linux-amd64.tar.gz - - tar -C /usr/local -xzf go1.22.3.linux-amd64.tar.gz + - wget https://golang.org/dl/$GO_VERSION_FILE + - tar -C /usr/local -xzf $GO_VERSION_FILE - export PATH=$PATH:/usr/local/go/bin - - rm go1.22.3.linux-amd64.tar.gz + - rm $GO_VERSION_FILE - ln -s /lib/libc.so.6 /usr/lib/libresolv.so.2 - make vtorc-vtadmin-test concurrency: 1 diff --git a/.gitignore b/.gitignore index 76cbd661..53742bf2 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,5 @@ tags ### Intellij IDEs ### .idea/* # End of https://www.gitignore.io/api/go,vim,emacs,visualstudiocode + +**/vtdataroot \ No newline at end of file diff --git a/Makefile b/Makefile index 29cc2164..16506fc5 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,10 @@ backup-restore-test: build e2e-test-setup echo "Running Backup-Restore test" test/endtoend/backup_restore_test.sh +backup-schedule-test: build e2e-test-setup + echo "Running Backup-Schedule test" + test/endtoend/backup_schedule_test.sh + vtorc-vtadmin-test: build e2e-test-setup echo "Running VTOrc and VtAdmin test" test/endtoend/vtorc_vtadmin_test.sh diff --git a/deploy/crds/planetscale.com_vitessbackupschedules.yaml b/deploy/crds/planetscale.com_vitessbackupschedules.yaml new file mode 100644 index 00000000..b0b5e6af --- /dev/null +++ b/deploy/crds/planetscale.com_vitessbackupschedules.yaml @@ -0,0 +1,171 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: vitessbackupschedules.planetscale.com +spec: + group: planetscale.com + names: + kind: VitessBackupSchedule + listKind: VitessBackupScheduleList + plural: vitessbackupschedules + singular: vitessbackupschedule + scope: Namespaced + versions: + - name: v2 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + affinity: + x-kubernetes-preserve-unknown-fields: true + allowedMissedRun: + minimum: 0 + type: integer + annotations: + additionalProperties: + type: string + type: object + cluster: + type: string + concurrencyPolicy: + enum: + - Allow + - Forbid + example: Forbid + type: string + failedJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + image: + type: string + imagePullPolicy: + type: string + jobTimeoutMinute: + default: 10 + format: int32 + minimum: 0 + type: integer + name: + example: every-day + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$ + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + schedule: + example: 0 0 * * * + minLength: 0 + type: string + startingDeadlineSeconds: + format: int64 + minimum: 0 + type: integer + strategies: + items: + properties: + extraFlags: + additionalProperties: + type: string + type: object + keyspace: + example: commerce + type: string + name: + enum: + - BackupShard + type: string + shard: + example: '-' + type: string + required: + - keyspace + - name + - shard + type: object + minItems: 1 + type: array + successfulJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + suspend: + type: boolean + required: + - cluster + - name + - resources + - schedule + - strategies + type: object + status: + properties: + active: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + lastScheduledTime: + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/crds/planetscale.com_vitessclusters.yaml b/deploy/crds/planetscale.com_vitessclusters.yaml index e54649dd..818304f1 100644 --- a/deploy/crds/planetscale.com_vitessclusters.yaml +++ b/deploy/crds/planetscale.com_vitessclusters.yaml @@ -152,6 +152,114 @@ spec: type: object minItems: 1 type: array + schedules: + items: + properties: + affinity: + x-kubernetes-preserve-unknown-fields: true + allowedMissedRun: + minimum: 0 + type: integer + annotations: + additionalProperties: + type: string + type: object + concurrencyPolicy: + enum: + - Allow + - Forbid + example: Forbid + type: string + failedJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + jobTimeoutMinute: + default: 10 + format: int32 + minimum: 0 + type: integer + name: + example: every-day + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$ + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + schedule: + example: 0 0 * * * + minLength: 0 + type: string + startingDeadlineSeconds: + format: int64 + minimum: 0 + type: integer + strategies: + items: + properties: + extraFlags: + additionalProperties: + type: string + type: object + keyspace: + example: commerce + type: string + name: + enum: + - BackupShard + type: string + shard: + example: '-' + type: string + required: + - keyspace + - name + - shard + type: object + minItems: 1 + type: array + successfulJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + suspend: + type: boolean + required: + - name + - resources + - schedule + - strategies + type: object + type: array subcontroller: properties: serviceAccountName: diff --git a/deploy/kustomization.yaml b/deploy/kustomization.yaml index 87aac4c5..e6a77663 100644 --- a/deploy/kustomization.yaml +++ b/deploy/kustomization.yaml @@ -11,3 +11,4 @@ resources: - crds/planetscale.com_vitessbackups.yaml - crds/planetscale.com_vitessbackupstorages.yaml - crds/planetscale.com_etcdlockservers.yaml +- crds/planetscale.com_vitessbackupschedules.yaml diff --git a/deploy/role.yaml b/deploy/role.yaml index ac7e8951..d482ebe6 100644 --- a/deploy/role.yaml +++ b/deploy/role.yaml @@ -68,5 +68,14 @@ rules: - vitessbackupstorages - vitessbackupstorages/status - vitessbackupstorages/finalizers + - vitessbackupschedules + - vitessbackupschedules/status + - vitessbackupschedules/finalizers + verbs: + - '*' +- apiGroups: + - batch + resources: + - jobs verbs: - '*' \ No newline at end of file diff --git a/docs/api/index.html b/docs/api/index.html index cbf2b048..41caa361 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -575,6 +575,16 @@

AzblobBackupLocation +

BackupStrategyName +(string alias)

+

+(Appears on: +VitessBackupScheduleStrategy) +

+

+

BackupStrategyName describes the vtctldclient command that will be used to take a backup. +When scheduling a backup, you must specify at least one strategy.

+

CephBackupLocation

@@ -679,8 +689,33 @@

ClusterBackupSpec

Subcontroller specifies any parameters needed for launching the VitessBackupStorage subcontroller pod.

+ + +schedules
+ + +[]VitessBackupScheduleTemplate + + + + +

Schedules defines how often we want to perform a backup and how to perform the backup. +This is a list of VitessBackupScheduleTemplate where the “name” field has to be unique +across all the items of the list.

+ + +

ConcurrencyPolicy +(string alias)

+

+(Appears on: +VitessBackupScheduleTemplate) +

+

+

ConcurrencyPolicy describes how the concurrency of new jobs created by VitessBackupSchedule +is handled, the default is set to AllowConcurrent.

+

EtcdLockserverSpec

@@ -2267,6 +2302,513 @@

VitessBackupLocation +

VitessBackupSchedule +

+

+

VitessBackupSchedule is the Schema for the VitessBackupSchedule API.

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+metadata
+ + +Kubernetes meta/v1.ObjectMeta + + +		 +Refer to the Kubernetes API documentation for the fields of the +metadata field. +
+spec
+ + +VitessBackupScheduleSpec + + +		 +
+
+ + + + + + + + + + + + + + + + + +
+VitessBackupScheduleTemplate
+ + +VitessBackupScheduleTemplate + + +		 +

+(Members of VitessBackupScheduleTemplate are embedded into this type.) +

+

VitessBackupScheduleTemplate contains the user-specific parts of VitessBackupScheduleSpec. +These are the parts that are configurable through the VitessCluster CRD.

+
+cluster
+ +string + +		 +

Cluster on which this schedule runs.

+
+image
+ +string + +		 +

Image should be any image that already contains vtctldclient installed. +The controller will re-use the vtctld image by default.

+
+imagePullPolicy
+ + +Kubernetes core/v1.PullPolicy + + +		 +

ImagePullPolicy defines the policy to pull the Docker image in the job’s pod. +The PullPolicy used will be the same as the one used to pull the vtctld image.

+
+
+status
+ + +VitessBackupScheduleStatus + + +		 +
+

VitessBackupScheduleSpec +

+

+(Appears on: +VitessBackupSchedule) +

+

+

VitessBackupScheduleSpec defines the desired state of VitessBackupSchedule.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+VitessBackupScheduleTemplate
+ + +VitessBackupScheduleTemplate + + +		 +

+(Members of VitessBackupScheduleTemplate are embedded into this type.) +

+

VitessBackupScheduleTemplate contains the user-specific parts of VitessBackupScheduleSpec. +These are the parts that are configurable through the VitessCluster CRD.

+
+cluster
+ +string + +		 +

Cluster on which this schedule runs.

+
+image
+ +string + +		 +

Image should be any image that already contains vtctldclient installed. +The controller will re-use the vtctld image by default.

+
+imagePullPolicy
+ + +Kubernetes core/v1.PullPolicy + + +		 +

ImagePullPolicy defines the policy to pull the Docker image in the job’s pod. +The PullPolicy used will be the same as the one used to pull the vtctld image.

+
+

VitessBackupScheduleStatus +

+

+(Appears on: +VitessBackupSchedule) +

+

+

VitessBackupScheduleStatus defines the observed state of VitessBackupSchedule

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+active
+ + +[]Kubernetes core/v1.ObjectReference + + +		 +(Optional) +

A list of pointers to currently running jobs.

+
+lastScheduledTime
+ + +Kubernetes meta/v1.Time + + +		 +(Optional) +

Information when was the last time the job was successfully scheduled.

+
+

VitessBackupScheduleStrategy +

+

+(Appears on: +VitessBackupScheduleTemplate) +

+

+

VitessBackupScheduleStrategy defines how we are going to take a backup. +The VitessBackupSchedule controller uses this data to build the vtctldclient +command line that will be executed in the Job’s pod.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+name
+ + +BackupStrategyName + + +		 +

Name of the backup strategy.

+
+keyspace
+ +string + +		 +

Keyspace defines the keyspace on which we want to take the backup.

+
+shard
+ +string + +		 +

Shard defines the shard on which we want to take a backup.

+
+extraFlags
+ +map[string]string + +		 +(Optional) +

ExtraFlags is a map of flags that will be sent down to vtctldclient when taking the backup.

+
+

VitessBackupScheduleTemplate +

+

+(Appears on: +ClusterBackupSpec, +VitessBackupScheduleSpec) +

+

+

VitessBackupScheduleTemplate contains all the user-specific fields that the user will be +able to define when writing their YAML file.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+name
+ +string + +		 +

Name is the schedule name, this name must be unique across all the different VitessBackupSchedule +objects in the cluster.

+
+schedule
+ +string + +		 +

The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron.

+
+strategies
+ + +[]VitessBackupScheduleStrategy + + +		 +

Strategy defines how we are going to take a backup. +If you want to take several backups within the same schedule you can add more items +to the Strategy list. Each VitessBackupScheduleStrategy will be executed by the same +kubernetes job. This is useful if for instance you have one schedule, and you want to +take a backup of all shards in a keyspace and don’t want to re-create a second schedule. +All the VitessBackupScheduleStrategy are concatenated into a single shell command that +is executed when the Job’s container starts.

+
+resources
+ + +Kubernetes core/v1.ResourceRequirements + + +		 +

Resources specify the compute resources to allocate for every Jobs’s pod.

+
+successfulJobsHistoryLimit
+ +int32 + +		 +(Optional) +

SuccessfulJobsHistoryLimit defines how many successful jobs will be kept around.

+
+failedJobsHistoryLimit
+ +int32 + +		 +(Optional) +

FailedJobsHistoryLimit defines how many failed jobs will be kept around.

+
+suspend
+ +bool + +		 +(Optional) +

Suspend pause the associated backup schedule. Pausing any further scheduled +runs until Suspend is set to false again. This is useful if you want to pause backup without +having to remove the entire VitessBackupSchedule object from the cluster.

+
+startingDeadlineSeconds
+ +int64 + +		 +(Optional) +

StartingDeadlineSeconds enables the VitessBackupSchedule to start a job even though it is late by +the given amount of seconds. Let’s say for some reason the controller process a schedule run on +second after its scheduled time, if StartingDeadlineSeconds is set to 0, the job will be skipped +as it’s too late, but on the other hand, if StartingDeadlineSeconds is greater than one second, +the job will be processed as usual.

+
+concurrencyPolicy
+ + +ConcurrencyPolicy + + +		 +(Optional) +

ConcurrencyPolicy specifies ho to treat concurrent executions of a Job. +Valid values are: +- “Allow” (default): allows CronJobs to run concurrently; +- “Forbid”: forbids concurrent runs, skipping next run if previous run hasn’t finished yet; +- “Replace”: cancels currently running job and replaces it with a new one.

+
+allowedMissedRun
+ +int + +		 +(Optional) +

AllowedMissedRuns defines how many missed run of the schedule will be allowed before giving up on finding the last job. +If the operator’s clock is skewed and we end-up missing a certain number of jobs, finding the last +job might be very time-consuming, depending on the frequency of the schedule and the duration during which +the operator’s clock was misbehaving. Also depending on how laggy the clock is, we can end-up with thousands +of missed runs. For this reason, AllowedMissedRun, which is set to 100 by default, will short circuit the search +and simply wait for the next job on the schedule. +Unless you are experiencing issue with missed runs due to a misconfiguration of the clock, we recommend leaving +this field to its default value.

+
+jobTimeoutMinute
+ +int32 + +		 +(Optional) +

JobTimeoutMinutes defines after how many minutes a job that has not yet finished should be stopped and removed. +Default value is 10 minutes.

+
+annotations
+ +map[string]string + +		 +(Optional) +

Annotations are the set of annotations that will be attached to the pods created by VitessBackupSchedule.

+
+affinity
+ + +Kubernetes core/v1.Affinity + + +		 +(Optional) +

Affinity allows you to set rules that constrain the scheduling of the pods that take backups. +WARNING: These affinity rules will override all default affinities that we set; in turn, we can’t +guarantee optimal scheduling of your pods if you choose to set this field.

+

VitessBackupSpec

diff --git a/docs/release-notes/2_13_0_summary.md b/docs/release-notes/2_13_0_summary.md new file mode 100644 index 00000000..22a5b168 --- /dev/null +++ b/docs/release-notes/2_13_0_summary.md @@ -0,0 +1,15 @@ +## Major Changes + +### Automated and Scheduled Backups + +As part of `v2.13.0` we are adding a new feature to the `vitess-operator`: automated and scheduled backups. The PR +implementing this change is available here: [#553](https://github.com/planetscale/vitess-operator/pull/553). + +This feature is for now experimental as we await feedback from the community on its usage. There are a few things to +take into account when using this feature: + +- If you are using the `xtrabackup` engine, your vttablet pods will need more memory, think about provisioning more memory for it. +- If you are using the `builtin` engine, you will lose a replica during the backup, think about adding a new tablet. + +If you are usually using specific flags when taking backups with `vtctldclient` you can set those flags on the `extraFlags` +field of the backup `strategy` (`VitessBackupScheduleStrategy`). \ No newline at end of file diff --git a/go.mod b/go.mod index 06aec4a8..44b536e0 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/google/uuid v1.6.0 github.com/planetscale/operator-sdk-libs v0.0.0-20220216002626-1af183733234 github.com/prometheus/client_golang v1.19.0 + github.com/robfig/cron v1.2.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 @@ -17,7 +18,7 @@ require ( k8s.io/klog v1.0.0 k8s.io/kubectl v0.21.9 k8s.io/kubernetes v1.28.5 - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 sigs.k8s.io/controller-runtime v0.16.3 sigs.k8s.io/controller-tools v0.11.3 sigs.k8s.io/kustomize v2.0.3+incompatible @@ -55,7 +56,7 @@ require ( github.com/docker/distribution v2.8.2+incompatible // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.7.1 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/color v1.16.0 // indirect @@ -65,10 +66,10 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.4 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/spec v0.19.5 // indirect - github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/gobuffalo/flect v0.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.1 // indirect @@ -186,10 +187,10 @@ require ( k8s.io/apiserver v0.28.5 // indirect k8s.io/component-base v0.28.5 // indirect k8s.io/gengo v0.0.0-20221011193443-fad74ee6edd9 // indirect - k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 46e3e813..7d8232f2 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,6 @@ github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qE github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/toxiproxy/v2 v2.9.0 h1:DIaDZG2/r/kv3Em6UxYBUVnnWl1mHlYTGFv+sTPV7VI= github.com/Shopify/toxiproxy/v2 v2.9.0/go.mod h1:2uPRyxR46fsx2yUr9i8zcejzdkWfK7p6G23jV/X6YNs= github.com/ahmetb/gen-crd-api-reference-docs v0.1.5-0.20190629210212-52e137b8d003 h1:mfPmvD5Nr9GTQnSgOg5OXusjS8uqDAZhJdYjmr1SWMc= @@ -98,7 +96,6 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -112,8 +109,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.7.1 h1:6/55d26lG3o9VCZX8lping+bZcmShseiqlh2bnUDiPA= github.com/ebitengine/purego v0.7.1/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= +github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -146,7 +143,6 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= @@ -155,24 +151,20 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw= -github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gobuffalo/flect v0.3.0 h1:erfPWM+K1rFNIQeRPdeEXxo8yFr/PO17lhRnS8FUrtk= github.com/gobuffalo/flect v0.3.0/go.mod h1:5pf3aGnsvqvCj50AVni7mJJF8ICxGZ8HomberC3pXLE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -226,8 +218,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= -github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/safehtml v0.1.0 h1:EwLKo8qawTKfsi0orxcQAZzu07cICaBeFMegAU9eaT8= @@ -330,8 +322,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -386,10 +376,10 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/ginkgo/v2 v2.17.2 h1:7eMhcy3GimbsA3hEnVKdw/PQM9XN9krpKVXsZdph0/g= +github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e h1:4cPxUYdgaGzZIT5/j0IfqOrrXmq6bG8AwvwisMXpdrg= @@ -450,6 +440,8 @@ github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052/go.mod h1 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -591,7 +583,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -773,16 +764,16 @@ k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= -k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f h1:0LQagt0gDpKqvIkAMPaRGcXawNMouPECM1+F9BVxEaM= +k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f/go.mod h1:S9tOR0FxgyusSNR+MboCuiDpVWkAifZvaYI1Q2ubgro= k8s.io/kubectl v0.28.5 h1:jq8xtiCCZPR8Cl/Qe1D7bLU0h8KtcunwfROqIekCUeU= k8s.io/kubectl v0.28.5/go.mod h1:9WiwzqeKs3vLiDtEQPbjhqqysX+BIVMLt7C7gN+T5w8= k8s.io/kubernetes v1.28.5 h1:sqgm0Tk6lqfsfcLUxZ0rfNmtugtABSLUhCqhXtfV1C0= k8s.io/kubernetes v1.28.5/go.mod h1:lRbvAZMgn+BrOtymmJAp85Jsa7GlL01av29axiPJ+E0= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/controller-tools v0.11.3 h1:T1xzLkog9saiyQSLz1XOImu4OcbdXWytc5cmYsBeBiE= @@ -791,8 +782,8 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMm sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/apis/planetscale/v2/labels.go b/pkg/apis/planetscale/v2/labels.go index 71c4421e..a88e6f62 100644 --- a/pkg/apis/planetscale/v2/labels.go +++ b/pkg/apis/planetscale/v2/labels.go @@ -40,6 +40,8 @@ const ( TabletPoolNameLabel = LabelPrefix + "/" + "pool-name" // TabletIndexLabel is the key for identifying the index of a Vitess tablet within its pool. TabletIndexLabel = LabelPrefix + "/" + "tablet-index" + // BackupScheduleLabel is the key for identifying to which VitessBackupSchedule a Job belongs to. + BackupScheduleLabel = LabelPrefix + "/" + "backup-schedule" // VtctldComponentName is the ComponentLabel value for vtctld. VtctldComponentName = "vtctld" diff --git a/pkg/apis/planetscale/v2/vitessbackupschedule_methods.go b/pkg/apis/planetscale/v2/vitessbackupschedule_methods.go new file mode 100644 index 00000000..86026b49 --- /dev/null +++ b/pkg/apis/planetscale/v2/vitessbackupschedule_methods.go @@ -0,0 +1,53 @@ +/* +Copyright 2024 PlanetScale Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2 + +// GetFailedJobsLimit returns the number of failed jobs to keep. +// Returns -1 if the value was not specified by the user. +func (vbsc *VitessBackupSchedule) GetFailedJobsLimit() int32 { + if vbsc.Spec.FailedJobsHistoryLimit == nil { + return -1 + } + return *vbsc.Spec.FailedJobsHistoryLimit +} + +// GetSuccessfulJobsLimit returns the number of failed jobs to keep. +// Returns -1 if the value was not specified by the user. +func (vbsc *VitessBackupSchedule) GetSuccessfulJobsLimit() int32 { + if vbsc.Spec.SuccessfulJobsHistoryLimit == nil { + return -1 + } + return *vbsc.Spec.SuccessfulJobsHistoryLimit +} + +// DefaultAllowedMissedRuns is the default that will be used in case of bug in the operator, +// which could be caused by the apiserver's clock for instance. In the event of such bug, +// the VitessBackupSchedule will try catching up the missed scheduled runs one by one +// this can be extremely lengthy in the even of a big clock skew, if the number of missed scheduled +// jobs reaches either DefaultAllowedMissedRuns or the value specified by the user, the controller +// will give up looking for the previously missed run and error out. +// Setting the default to 100 is fair, catching up a up to 100 missed scheduled runs is not lengthy. +const DefaultAllowedMissedRuns = 100 + +// GetMissedRunsLimit returns the maximum number of missed run we can allow. +// Returns DefaultAllowedMissedRuns if the value was not specified by the user. +func (vbsc *VitessBackupSchedule) GetMissedRunsLimit() int { + if vbsc.Spec.AllowedMissedRuns == nil { + return DefaultAllowedMissedRuns + } + return *vbsc.Spec.AllowedMissedRuns +} diff --git a/pkg/apis/planetscale/v2/vitessbackupschedule_types.go b/pkg/apis/planetscale/v2/vitessbackupschedule_types.go new file mode 100644 index 00000000..ff29654d --- /dev/null +++ b/pkg/apis/planetscale/v2/vitessbackupschedule_types.go @@ -0,0 +1,211 @@ +/* +Copyright 2024 PlanetScale Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ConcurrencyPolicy describes how the concurrency of new jobs created by VitessBackupSchedule +// is handled, the default is set to AllowConcurrent. +// +kubebuilder:validation:Enum=Allow;Forbid +type ConcurrencyPolicy string + +const ( + // AllowConcurrent allows CronJobs to run concurrently. + AllowConcurrent ConcurrencyPolicy = "Allow" + + // ForbidConcurrent forbids concurrent runs, skipping next run if previous hasn't finished yet. + ForbidConcurrent ConcurrencyPolicy = "Forbid" +) + +// BackupStrategyName describes the vtctldclient command that will be used to take a backup. +// When scheduling a backup, you must specify at least one strategy. +// +kubebuilder:validation:Enum=BackupShard +type BackupStrategyName string + +const ( + // BackupShard will use the "vtctldclient BackupShard" command to take a backup + BackupShard BackupStrategyName = "BackupShard" +) + +// VitessBackupSchedule is the Schema for the VitessBackupSchedule API. +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +type VitessBackupSchedule struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec VitessBackupScheduleSpec `json:"spec,omitempty"` + Status VitessBackupScheduleStatus `json:"status,omitempty"` +} + +// VitessBackupScheduleList contains a list of VitessBackupSchedule. +// +kubebuilder:object:root=true +type VitessBackupScheduleList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []VitessBackupSchedule `json:"items"` +} + +// VitessBackupScheduleSpec defines the desired state of VitessBackupSchedule. +type VitessBackupScheduleSpec struct { + // VitessBackupScheduleTemplate contains the user-specific parts of VitessBackupScheduleSpec. + // These are the parts that are configurable through the VitessCluster CRD. + VitessBackupScheduleTemplate `json:",inline"` + + // Cluster on which this schedule runs. + Cluster string `json:"cluster"` + + // Image should be any image that already contains vtctldclient installed. + // The controller will re-use the vtctld image by default. + Image string `json:"image,omitempty"` + + // ImagePullPolicy defines the policy to pull the Docker image in the job's pod. + // The PullPolicy used will be the same as the one used to pull the vtctld image. + ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` +} + +// VitessBackupScheduleTemplate contains all the user-specific fields that the user will be +// able to define when writing their YAML file. +type VitessBackupScheduleTemplate struct { + // Name is the schedule name, this name must be unique across all the different VitessBackupSchedule + // objects in the cluster. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=^[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$ + // +kubebuilder:example="every-day" + Name string `json:"name"` + + // The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron. + // +kubebuilder:validation:MinLength=0 + // +kubebuilder:example="0 0 * * *" + Schedule string `json:"schedule"` + + // Strategy defines how we are going to take a backup. + // If you want to take several backups within the same schedule you can add more items + // to the Strategy list. Each VitessBackupScheduleStrategy will be executed by the same + // kubernetes job. This is useful if for instance you have one schedule, and you want to + // take a backup of all shards in a keyspace and don't want to re-create a second schedule. + // All the VitessBackupScheduleStrategy are concatenated into a single shell command that + // is executed when the Job's container starts. + // +kubebuilder:validation:MinItems=1 + Strategy []VitessBackupScheduleStrategy `json:"strategies"` + + // Resources specify the compute resources to allocate for every Jobs's pod. + Resources corev1.ResourceRequirements `json:"resources"` + + // SuccessfulJobsHistoryLimit defines how many successful jobs will be kept around. + // +optional + // +kubebuilder:validation:Minimum=0 + SuccessfulJobsHistoryLimit *int32 `json:"successfulJobsHistoryLimit,omitempty"` + + // FailedJobsHistoryLimit defines how many failed jobs will be kept around. + // +optional + // +kubebuilder:validation:Minimum=0 + FailedJobsHistoryLimit *int32 `json:"failedJobsHistoryLimit,omitempty"` + + // Suspend pause the associated backup schedule. Pausing any further scheduled + // runs until Suspend is set to false again. This is useful if you want to pause backup without + // having to remove the entire VitessBackupSchedule object from the cluster. + // +optional + Suspend *bool `json:"suspend,omitempty"` + + // StartingDeadlineSeconds enables the VitessBackupSchedule to start a job even though it is late by + // the given amount of seconds. Let's say for some reason the controller process a schedule run on + // second after its scheduled time, if StartingDeadlineSeconds is set to 0, the job will be skipped + // as it's too late, but on the other hand, if StartingDeadlineSeconds is greater than one second, + // the job will be processed as usual. + // +optional + // +kubebuilder:validation:Minimum=0 + StartingDeadlineSeconds *int64 `json:"startingDeadlineSeconds,omitempty"` + + // ConcurrencyPolicy specifies ho to treat concurrent executions of a Job. + // Valid values are: + // - "Allow" (default): allows CronJobs to run concurrently; + // - "Forbid": forbids concurrent runs, skipping next run if previous run hasn't finished yet; + // - "Replace": cancels currently running job and replaces it with a new one. + // +optional + // +kubebuilder:example="Forbid" + ConcurrencyPolicy ConcurrencyPolicy `json:"concurrencyPolicy,omitempty"` + + // AllowedMissedRuns defines how many missed run of the schedule will be allowed before giving up on finding the last job. + // If the operator's clock is skewed and we end-up missing a certain number of jobs, finding the last + // job might be very time-consuming, depending on the frequency of the schedule and the duration during which + // the operator's clock was misbehaving. Also depending on how laggy the clock is, we can end-up with thousands + // of missed runs. For this reason, AllowedMissedRun, which is set to 100 by default, will short circuit the search + // and simply wait for the next job on the schedule. + // Unless you are experiencing issue with missed runs due to a misconfiguration of the clock, we recommend leaving + // this field to its default value. + // +optional + // +kubebuilder:validation:Minimum=0 + AllowedMissedRuns *int `json:"allowedMissedRun,omitempty"` + + // JobTimeoutMinutes defines after how many minutes a job that has not yet finished should be stopped and removed. + // Default value is 10 minutes. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=10 + JobTimeoutMinutes int32 `json:"jobTimeoutMinute,omitempty"` + + // Annotations are the set of annotations that will be attached to the pods created by VitessBackupSchedule. + // +optional + Annotations map[string]string `json:"annotations,omitempty"` + + // Affinity allows you to set rules that constrain the scheduling of the pods that take backups. + // WARNING: These affinity rules will override all default affinities that we set; in turn, we can't + // guarantee optimal scheduling of your pods if you choose to set this field. + // +optional + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + Affinity *corev1.Affinity `json:"affinity,omitempty"` +} + +// VitessBackupScheduleStrategy defines how we are going to take a backup. +// The VitessBackupSchedule controller uses this data to build the vtctldclient +// command line that will be executed in the Job's pod. +type VitessBackupScheduleStrategy struct { + // Name of the backup strategy. + Name BackupStrategyName `json:"name"` + + // Keyspace defines the keyspace on which we want to take the backup. + // +kubebuilder:example="commerce" + Keyspace string `json:"keyspace"` + + // Shard defines the shard on which we want to take a backup. + // +kubebuilder:example="-" + Shard string `json:"shard"` + + // ExtraFlags is a map of flags that will be sent down to vtctldclient when taking the backup. + // +optional + ExtraFlags map[string]string `json:"extraFlags,omitempty"` +} + +// VitessBackupScheduleStatus defines the observed state of VitessBackupSchedule +type VitessBackupScheduleStatus struct { + // A list of pointers to currently running jobs. + // +optional + Active []corev1.ObjectReference `json:"active,omitempty"` + + // Information when was the last time the job was successfully scheduled. + // +optional + LastScheduledTime *metav1.Time `json:"lastScheduledTime,omitempty"` +} + +func init() { + SchemeBuilder.Register(&VitessBackupSchedule{}, &VitessBackupScheduleList{}) +} diff --git a/pkg/apis/planetscale/v2/vitesscluster_types.go b/pkg/apis/planetscale/v2/vitesscluster_types.go index f796d11b..000088e9 100644 --- a/pkg/apis/planetscale/v2/vitesscluster_types.go +++ b/pkg/apis/planetscale/v2/vitesscluster_types.go @@ -320,6 +320,13 @@ type ClusterBackupSpec struct { Engine VitessBackupEngine `json:"engine,omitempty"` // Subcontroller specifies any parameters needed for launching the VitessBackupStorage subcontroller pod. Subcontroller *VitessBackupSubcontrollerSpec `json:"subcontroller,omitempty"` + + // Schedules defines how often we want to perform a backup and how to perform the backup. + // This is a list of VitessBackupScheduleTemplate where the "name" field has to be unique + // across all the items of the list. + // +patchMergeKey=name + // +patchStrategy=merge + Schedules []VitessBackupScheduleTemplate `json:"schedules,omitempty"` } // VitessBackupEngine is the backup implementation to use. diff --git a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go index 4243ff17..4db30177 100644 --- a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go +++ b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go @@ -57,6 +57,13 @@ func (in *ClusterBackupSpec) DeepCopyInto(out *ClusterBackupSpec) { *out = new(VitessBackupSubcontrollerSpec) **out = **in } + if in.Schedules != nil { + in, out := &in.Schedules, &out.Schedules + *out = make([]VitessBackupScheduleTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterBackupSpec. @@ -729,6 +736,187 @@ func (in *VitessBackupLocation) DeepCopy() *VitessBackupLocation { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VitessBackupSchedule) DeepCopyInto(out *VitessBackupSchedule) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VitessBackupSchedule. +func (in *VitessBackupSchedule) DeepCopy() *VitessBackupSchedule { + if in == nil { + return nil + } + out := new(VitessBackupSchedule) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VitessBackupSchedule) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VitessBackupScheduleList) DeepCopyInto(out *VitessBackupScheduleList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VitessBackupSchedule, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VitessBackupScheduleList. +func (in *VitessBackupScheduleList) DeepCopy() *VitessBackupScheduleList { + if in == nil { + return nil + } + out := new(VitessBackupScheduleList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VitessBackupScheduleList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VitessBackupScheduleSpec) DeepCopyInto(out *VitessBackupScheduleSpec) { + *out = *in + in.VitessBackupScheduleTemplate.DeepCopyInto(&out.VitessBackupScheduleTemplate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VitessBackupScheduleSpec. +func (in *VitessBackupScheduleSpec) DeepCopy() *VitessBackupScheduleSpec { + if in == nil { + return nil + } + out := new(VitessBackupScheduleSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VitessBackupScheduleStatus) DeepCopyInto(out *VitessBackupScheduleStatus) { + *out = *in + if in.Active != nil { + in, out := &in.Active, &out.Active + *out = make([]v1.ObjectReference, len(*in)) + copy(*out, *in) + } + if in.LastScheduledTime != nil { + in, out := &in.LastScheduledTime, &out.LastScheduledTime + *out = (*in).DeepCopy() + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VitessBackupScheduleStatus. +func (in *VitessBackupScheduleStatus) DeepCopy() *VitessBackupScheduleStatus { + if in == nil { + return nil + } + out := new(VitessBackupScheduleStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VitessBackupScheduleStrategy) DeepCopyInto(out *VitessBackupScheduleStrategy) { + *out = *in + if in.ExtraFlags != nil { + in, out := &in.ExtraFlags, &out.ExtraFlags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VitessBackupScheduleStrategy. +func (in *VitessBackupScheduleStrategy) DeepCopy() *VitessBackupScheduleStrategy { + if in == nil { + return nil + } + out := new(VitessBackupScheduleStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VitessBackupScheduleTemplate) DeepCopyInto(out *VitessBackupScheduleTemplate) { + *out = *in + if in.Strategy != nil { + in, out := &in.Strategy, &out.Strategy + *out = make([]VitessBackupScheduleStrategy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Resources.DeepCopyInto(&out.Resources) + if in.SuccessfulJobsHistoryLimit != nil { + in, out := &in.SuccessfulJobsHistoryLimit, &out.SuccessfulJobsHistoryLimit + *out = new(int32) + **out = **in + } + if in.FailedJobsHistoryLimit != nil { + in, out := &in.FailedJobsHistoryLimit, &out.FailedJobsHistoryLimit + *out = new(int32) + **out = **in + } + if in.Suspend != nil { + in, out := &in.Suspend, &out.Suspend + *out = new(bool) + **out = **in + } + if in.StartingDeadlineSeconds != nil { + in, out := &in.StartingDeadlineSeconds, &out.StartingDeadlineSeconds + *out = new(int64) + **out = **in + } + if in.AllowedMissedRuns != nil { + in, out := &in.AllowedMissedRuns, &out.AllowedMissedRuns + *out = new(int) + **out = **in + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VitessBackupScheduleTemplate. +func (in *VitessBackupScheduleTemplate) DeepCopy() *VitessBackupScheduleTemplate { + if in == nil { + return nil + } + out := new(VitessBackupScheduleTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VitessBackupSpec) DeepCopyInto(out *VitessBackupSpec) { *out = *in diff --git a/pkg/controller/add_vitessbackupschedule.go b/pkg/controller/add_vitessbackupschedule.go new file mode 100644 index 00000000..836fd975 --- /dev/null +++ b/pkg/controller/add_vitessbackupschedule.go @@ -0,0 +1,26 @@ +/* +Copyright 2024 PlanetScale Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "planetscale.dev/vitess-operator/pkg/controller/vitessbackupschedule" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, vitessbackupschedule.Add) +} diff --git a/pkg/controller/vitessbackupschedule/metrics.go b/pkg/controller/vitessbackupschedule/metrics.go new file mode 100644 index 00000000..03d3f343 --- /dev/null +++ b/pkg/controller/vitessbackupschedule/metrics.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 PlanetScale Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vitessbackupschedule + +import ( + "github.com/prometheus/client_golang/prometheus" + + "planetscale.dev/vitess-operator/pkg/operator/metrics" +) + +const ( + metricsSubsystemName = "backup_schedule" +) + +var ( + backupScheduleLabels = []string{ + metrics.BackupScheduleLabel, + metrics.ResultLabel, + } + + reconcileCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.Namespace, + Subsystem: metricsSubsystemName, + Name: "reconcile_count", + Help: "Reconciliation attempts for a VitessBackupSchedule", + }, backupScheduleLabels) + + timeoutJobsCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metrics.Namespace, + Subsystem: metricsSubsystemName, + Name: "timeout_jobs_removed_count", + Help: "Number of timed out jobs that were removed for a VitessBackupSchedule", + }, backupScheduleLabels) +) + +func init() { + metrics.Registry.MustRegister( + reconcileCount, + timeoutJobsCount, + ) +} diff --git a/pkg/controller/vitessbackupschedule/vitessbackupschedule_controller.go b/pkg/controller/vitessbackupschedule/vitessbackupschedule_controller.go new file mode 100644 index 00000000..4ce56b21 --- /dev/null +++ b/pkg/controller/vitessbackupschedule/vitessbackupschedule_controller.go @@ -0,0 +1,619 @@ +/* +Copyright 2024 PlanetScale Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vitessbackupschedule + +import ( + "context" + "flag" + "fmt" + "maps" + "math/rand/v2" + "strings" + + "sort" + "time" + + "github.com/robfig/cron" + "github.com/sirupsen/logrus" + kbatch "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apilabels "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/record" + "planetscale.dev/vitess-operator/pkg/operator/metrics" + "planetscale.dev/vitess-operator/pkg/operator/reconciler" + "planetscale.dev/vitess-operator/pkg/operator/results" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + ref "k8s.io/client-go/tools/reference" + planetscalev2 "planetscale.dev/vitess-operator/pkg/apis/planetscale/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + controllerName = "vitessbackupschedule-controller" + vtctldclientPath = "/vt/bin/vtctldclient" +) + +var ( + maxConcurrentReconciles = flag.Int("vitessbackupschedule_concurrent_reconciles", 10, "the maximum number of different vitessbackupschedule resources to reconcile concurrently") + + scheduledTimeAnnotation = "planetscale.com/backup-scheduled-at" + + log = logrus.WithField("controller", "VitessBackupSchedule") +) + +// watchResources should contain all the resource types that this controller creates. +var watchResources = []client.Object{ + &kbatch.Job{}, +} + +type ( + // ReconcileVitessBackupsSchedule reconciles a VitessBackupSchedule object + ReconcileVitessBackupsSchedule struct { + client client.Client + scheme *runtime.Scheme + recorder record.EventRecorder + reconciler *reconciler.Reconciler + } + + jobsList struct { + active []*kbatch.Job + successful []*kbatch.Job + failed []*kbatch.Job + } +) + +var _ reconcile.Reconciler = &ReconcileVitessBackupsSchedule{} + +// Add creates a new Controller and adds it to the Manager. +func Add(mgr manager.Manager) error { + r, err := newReconciler(mgr) + if err != nil { + return err + } + return add(mgr, r) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) (*ReconcileVitessBackupsSchedule, error) { + c := mgr.GetClient() + scheme := mgr.GetScheme() + recorder := mgr.GetEventRecorderFor(controllerName) + + return &ReconcileVitessBackupsSchedule{ + client: c, + scheme: scheme, + recorder: recorder, + reconciler: reconciler.New(c, scheme, recorder), + }, nil +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r *ReconcileVitessBackupsSchedule) error { + // Create a new controller + c, err := controller.New(controllerName, mgr, controller.Options{ + Reconciler: r, + MaxConcurrentReconciles: *maxConcurrentReconciles, + }) + if err != nil { + return err + } + + // Watch for changes to primary resource VitessBackupSchedule + if err := c.Watch(source.Kind(mgr.GetCache(), &planetscalev2.VitessBackupSchedule{}), &handler.EnqueueRequestForObject{}); err != nil { + return err + } + + // Watch for changes to kbatch.Job and requeue the owner VitessBackupSchedule. + for _, resource := range watchResources { + err := c.Watch(source.Kind(mgr.GetCache(), resource), handler.EnqueueRequestForOwner( + mgr.GetScheme(), + mgr.GetRESTMapper(), + &planetscalev2.VitessBackupStorage{}, + handler.OnlyControllerOwner(), + )) + if err != nil { + return err + } + } + + return nil +} + +// Reconcile implements the kubernetes Reconciler interface. +// The main goal of this function is to create new Job k8s object according to the VitessBackupSchedule schedule. +// It also takes care of removing old failed and successful jobs, given the settings of VitessBackupSchedule. +// The function is structured as follows: +// - Get the VitessBackupSchedule object +// - List all jobs and define the last scheduled Job +// - Clean up old Job objects +// - Create a new Job if needed +func (r *ReconcileVitessBackupsSchedule) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + resultBuilder := &results.Builder{} + + log = log.WithFields(logrus.Fields{ + "namespace": req.Namespace, + "VitessBackupSchedule": req.Name, + }) + log.Info("Reconciling VitessBackupSchedule") + + var err error + var vbsc planetscalev2.VitessBackupSchedule + if err = r.client.Get(ctx, req.NamespacedName, &vbsc); err != nil { + log.WithError(err).Error(" unable to fetch VitessBackupSchedule") + if apierrors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + return resultBuilder.Result() + } + // Error reading the object - requeue the request. + return resultBuilder.Error(err) + } + + // Register this reconciling attempt no matter if we fail or succeed. + defer func() { + reconcileCount.WithLabelValues(vbsc.Name, metrics.Result(err)).Inc() + }() + + jobs, mostRecentTime, err := r.getJobsList(ctx, req, vbsc.Name) + if err != nil { + // We had an error reading the jobs, we can requeue. + return resultBuilder.Error(err) + } + + err = r.updateVitessBackupScheduleStatus(ctx, mostRecentTime, vbsc, jobs.active) + if err != nil { + // We had an error updating the status, we can requeue. + return resultBuilder.Error(err) + } + + // We must clean up old jobs to not overcrowd the number of Pods and Jobs in the cluster. + // This will be done according to both failedJobsHistoryLimit and successfulJobsHistoryLimit fields. + r.cleanupJobsWithLimit(ctx, jobs.failed, vbsc.GetFailedJobsLimit()) + r.cleanupJobsWithLimit(ctx, jobs.successful, vbsc.GetSuccessfulJobsLimit()) + + err = r.removeTimeoutJobs(ctx, jobs.active, vbsc.Name, vbsc.Spec.JobTimeoutMinutes) + if err != nil { + // We had an error while removing timed out jobs, we can requeue + return resultBuilder.Error(err) + } + + // If the Suspend setting is set to true, we can skip adding any job, our work is done here. + if vbsc.Spec.Suspend != nil && *vbsc.Spec.Suspend { + log.Info("VitessBackupSchedule suspended, skipping") + return ctrl.Result{}, nil + } + + missedRun, nextRun, err := getNextSchedule(vbsc, time.Now()) + if err != nil { + log.Error(err, "unable to figure out VitessBackupSchedule schedule") + // Re-queuing here does not make sense as we have an error with the schedule and the user needs to fix it first. + return ctrl.Result{}, nil + } + + // Ask kubernetes to re-queue for the next scheduled job, and skip if we don't miss any run. + scheduledResult := ctrl.Result{RequeueAfter: nextRun.Sub(time.Now())} + if missedRun.IsZero() { + return scheduledResult, nil + } + + // Check whether we are too late to create this Job or not. The startingDeadlineSeconds field will help us + // schedule Jobs that are late. + tooLate := false + if vbsc.Spec.StartingDeadlineSeconds != nil { + tooLate = missedRun.Add(time.Duration(*vbsc.Spec.StartingDeadlineSeconds) * time.Second).Before(time.Now()) + } + if tooLate { + log.Infof("missed starting deadline for latest run; skipping; next run is scheduled for: %s", nextRun.Format(time.RFC3339)) + return scheduledResult, nil + } + + // Check concurrency policy and skip this job if we have ForbidConcurrent set plus an active job + if vbsc.Spec.ConcurrencyPolicy == planetscalev2.ForbidConcurrent && len(jobs.active) > 0 { + log.Infof("concurrency policy blocks concurrent runs: skipping, number of active jobs: %d", len(jobs.active)) + return scheduledResult, nil + } + + // Now that the different policies are checked, we can create and apply our new job. + job, err := r.createJob(ctx, &vbsc, missedRun) + if err != nil { + // Re-queuing here does not make sense as we have an error with the template and the user needs to fix it first. + log.WithError(err).Error("unable to construct job from template") + return ctrl.Result{}, err + } + if err = r.client.Create(ctx, job); err != nil { + // if the job already exists it means another reconciling loop created the job since we last fetched + // the list of jobs to create, we can safely return without failing. + if apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, nil + } + // Simply re-queue here + return resultBuilder.Error(err) + } + + log.Infof("created new job: %s, next job scheduled in %s", job.Name, scheduledResult.RequeueAfter.String()) + return scheduledResult, nil +} + +func getNextSchedule(vbsc planetscalev2.VitessBackupSchedule, now time.Time) (time.Time, time.Time, error) { + sched, err := cron.ParseStandard(vbsc.Spec.Schedule) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("unable to parse schedule %q: %v", vbsc.Spec.Schedule, err) + } + + // Set the last scheduled time by either looking at the VitessBackupSchedule's Status or + // by looking at its creation time. + var latestRun time.Time + if vbsc.Status.LastScheduledTime != nil { + latestRun = vbsc.Status.LastScheduledTime.Time + } else { + latestRun = vbsc.ObjectMeta.CreationTimestamp.Time + } + + if vbsc.Spec.StartingDeadlineSeconds != nil { + // controller is not going to schedule anything below this point + schedulingDeadline := now.Add(-time.Second * time.Duration(*vbsc.Spec.StartingDeadlineSeconds)) + + if schedulingDeadline.After(latestRun) { + latestRun = schedulingDeadline + } + } + + // Next schedule is later, simply return the next scheduled time. + if latestRun.After(now) { + return time.Time{}, sched.Next(now), nil + } + + var lastMissed time.Time + missedRuns := 0 + for t := sched.Next(latestRun); !t.After(now); t = sched.Next(t) { + lastMissed = t + missedRuns++ + + // If we have too many missed jobs, just bail out as the clock lag is too big + if missedRuns > vbsc.GetMissedRunsLimit() { + return time.Time{}, time.Time{}, fmt.Errorf("too many missed runs, check clock skew or increase .spec.allowedMissedRun") + } + } + + return lastMissed, sched.Next(now), nil +} + +func (r *ReconcileVitessBackupsSchedule) updateVitessBackupScheduleStatus(ctx context.Context, mostRecentTime *time.Time, vbsc planetscalev2.VitessBackupSchedule, activeJobs []*kbatch.Job) error { + if mostRecentTime != nil { + vbsc.Status.LastScheduledTime = &metav1.Time{Time: *mostRecentTime} + } else { + vbsc.Status.LastScheduledTime = nil + } + + vbsc.Status.Active = make([]corev1.ObjectReference, 0, len(activeJobs)) + for _, activeJob := range activeJobs { + jobRef, err := ref.GetReference(r.scheme, activeJob) + if err != nil { + log.WithError(err).Errorf("unable to make reference to active job: %s", jobRef.Name) + continue + } + vbsc.Status.Active = append(vbsc.Status.Active, *jobRef) + } + + if err := r.client.Status().Update(ctx, &vbsc); err != nil { + log.WithError(err).Error("unable to update VitessBackupSchedule status") + return err + } + return nil +} + +// getJobsList fetches all existing Jobs in the cluster and return them by categories: active, failed or successful. +// It also returns at what time was the last job created, which is needed to update VitessBackupSchedule's status, +// and plan future jobs. +func (r *ReconcileVitessBackupsSchedule) getJobsList(ctx context.Context, req ctrl.Request, vbscName string) (jobsList, *time.Time, error) { + var existingJobs kbatch.JobList + + err := r.client.List(ctx, &existingJobs, client.InNamespace(req.Namespace), client.MatchingLabels{planetscalev2.BackupScheduleLabel: vbscName}) + if err != nil && !apierrors.IsNotFound(err) { + log.WithError(err).Error("unable to list Jobs in cluster") + return jobsList{}, nil, err + } + + var jobs jobsList + + var mostRecentTime *time.Time + + for i, job := range existingJobs.Items { + _, jobType := isJobFinished(&job) + switch jobType { + case kbatch.JobFailed, kbatch.JobFailureTarget: + jobs.failed = append(jobs.failed, &existingJobs.Items[i]) + case kbatch.JobComplete: + jobs.successful = append(jobs.successful, &existingJobs.Items[i]) + case kbatch.JobSuspended, "": + jobs.active = append(jobs.active, &existingJobs.Items[i]) + default: + return jobsList{}, nil, fmt.Errorf("unknown job type: %s", jobType) + } + + scheduledTimeForJob, err := getScheduledTimeForJob(&job) + if err != nil { + log.WithError(err).Errorf("unable to parse schedule time for existing job, found: %s", job.Annotations[scheduledTimeAnnotation]) + continue + } + if scheduledTimeForJob != nil && (mostRecentTime == nil || mostRecentTime.Before(*scheduledTimeForJob)) { + mostRecentTime = scheduledTimeForJob + } + } + return jobs, mostRecentTime, nil +} + +// cleanupJobsWithLimit removes all Job objects from the cluster ordered by oldest to newest and +// respecting the given limit, keeping minimum "limit" jobs in the cluster. +func (r *ReconcileVitessBackupsSchedule) cleanupJobsWithLimit(ctx context.Context, jobs []*kbatch.Job, limit int32) { + if limit == -1 { + return + } + + sort.SliceStable(jobs, func(i, j int) bool { + if jobs[i].Status.StartTime == nil { + return jobs[j].Status.StartTime != nil + } + return jobs[i].Status.StartTime.Before(jobs[j].Status.StartTime) + }) + + for i, job := range jobs { + if int32(i) >= int32(len(jobs))-limit { + break + } + if err := r.client.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationBackground)); (err) != nil { + log.WithError(err).Errorf("unable to delete old job: %s", job.Name) + } else { + log.Infof("deleted old job: %s", job.Name) + } + } +} + +func (r *ReconcileVitessBackupsSchedule) removeTimeoutJobs(ctx context.Context, jobs []*kbatch.Job, vbscName string, timeout int32) error { + if timeout == -1 { + return nil + } + for _, job := range jobs { + jobStartTime, err := getScheduledTimeForJob(job) + if err != nil { + return err + } + if jobStartTime.Add(time.Minute * time.Duration(timeout)).Before(time.Now()) { + if err = r.client.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationBackground)); (err) != nil { + log.WithError(err).Errorf("unable to delete timed out job: %s", job.Name) + } else { + log.Infof("deleted timed out job: %s", job.Name) + } + timeoutJobsCount.WithLabelValues(vbscName, metrics.Result(err)).Inc() + } + } + return nil +} + +func isJobFinished(job *kbatch.Job) (bool, kbatch.JobConditionType) { + for _, c := range job.Status.Conditions { + if (c.Type == kbatch.JobComplete || c.Type == kbatch.JobFailed) && c.Status == corev1.ConditionTrue { + return true, c.Type + } + } + + return false, "" +} + +func getScheduledTimeForJob(job *kbatch.Job) (*time.Time, error) { + timeRaw := job.Annotations[scheduledTimeAnnotation] + if len(timeRaw) == 0 { + return nil, nil + } + + timeParsed, err := time.Parse(time.RFC3339, timeRaw) + if err != nil { + return nil, err + } + + return &timeParsed, nil +} + +func (r *ReconcileVitessBackupsSchedule) createJob(ctx context.Context, vbsc *planetscalev2.VitessBackupSchedule, scheduledTime time.Time) (*kbatch.Job, error) { + name := fmt.Sprintf("%s-%d", vbsc.Name, scheduledTime.Unix()) + + meta := metav1.ObjectMeta{ + Labels: map[string]string{ + planetscalev2.BackupScheduleLabel: vbsc.Name, + }, + Annotations: make(map[string]string), + Name: name, + Namespace: vbsc.Namespace, + } + maps.Copy(meta.Annotations, vbsc.Annotations) + maps.Copy(meta.Annotations, vbsc.Spec.Annotations) + + meta.Annotations[scheduledTimeAnnotation] = scheduledTime.Format(time.RFC3339) + + maps.Copy(meta.Labels, vbsc.Labels) + + pod, err := r.createJobPod(ctx, vbsc, name) + if err != nil { + return nil, err + } + job := &kbatch.Job{ + ObjectMeta: meta, + Spec: kbatch.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: meta, + Spec: pod, + }, + }, + } + + if err := ctrl.SetControllerReference(vbsc, job, r.scheme); err != nil { + return nil, err + } + + return job, nil +} + +func (r *ReconcileVitessBackupsSchedule) createJobPod(ctx context.Context, vbsc *planetscalev2.VitessBackupSchedule, name string) (pod corev1.PodSpec, err error) { + getVtctldServiceName := func(cluster string) (string, error) { + vtctldServiceName, vtctldServicePort, err := r.getVtctldServiceName(ctx, vbsc, cluster) + if err != nil { + return "", err + } + return fmt.Sprintf("--server=%s:%d", vtctldServiceName, vtctldServicePort), nil + } + + // It is fine to not have any default in the event there is no strategy as the CRD validation + // ensures that there will be at least one item in this list. The YAML cannot be applied with + // empty list of strategies. + var cmd strings.Builder + + addNewCmd := func(i int) { + if i > 0 { + cmd.WriteString(" && ") + } + } + + for i, strategy := range vbsc.Spec.Strategy { + vtctldclientServerArg, err := getVtctldServiceName(vbsc.Spec.Cluster) + if err != nil { + return corev1.PodSpec{}, err + } + + addNewCmd(i) + switch strategy.Name { + case planetscalev2.BackupShard: + createVtctldClientCommand(&cmd, vtctldclientServerArg, strategy.ExtraFlags, strategy.Keyspace, strategy.Shard) + } + + } + + pod = corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: name, + Image: vbsc.Spec.Image, + ImagePullPolicy: vbsc.Spec.ImagePullPolicy, + Resources: vbsc.Spec.Resources, + Args: []string{"/bin/sh", "-c", cmd.String()}, + }}, + RestartPolicy: corev1.RestartPolicyOnFailure, + Affinity: vbsc.Spec.Affinity, + } + return pod, nil +} + +func createVtctldClientCommand(cmd *strings.Builder, serverAddr string, extraFlags map[string]string, keyspace, shard string) { + cmd.WriteString(fmt.Sprintf("%s %s BackupShard", vtctldclientPath, serverAddr)) + + // Add any flags + for key, value := range extraFlags { + cmd.WriteString(fmt.Sprintf(" --%s=%s", key, value)) + } + + // Add keyspace/shard + cmd.WriteString(fmt.Sprintf(" %s/%s", keyspace, shard)) +} + +func (r *ReconcileVitessBackupsSchedule) getVtctldServiceName(ctx context.Context, vbsc *planetscalev2.VitessBackupSchedule, cluster string) (svcName string, svcPort int32, err error) { + svcList := &corev1.ServiceList{} + listOpts := &client.ListOptions{ + Namespace: vbsc.Namespace, + LabelSelector: apilabels.Set{ + planetscalev2.ClusterLabel: cluster, + planetscalev2.ComponentLabel: planetscalev2.VtctldComponentName, + }.AsSelector(), + } + if err = r.client.List(ctx, svcList, listOpts); err != nil { + return "", 0, fmt.Errorf("unable to list vtctld service in %q: %v", vbsc.Namespace, err) + } + + if len(svcList.Items) > 0 { + service := svcList.Items[rand.IntN(len(svcList.Items))] + svcName = service.Name + for _, port := range service.Spec.Ports { + if port.Name == planetscalev2.DefaultGrpcPortName { + svcPort = port.Port + break + } + } + } + + if svcName == "" || svcPort == 0 { + return "", 0, fmt.Errorf("no vtctld service found in %q namespace", vbsc.Namespace) + } + return svcName, svcPort, nil +} + +func (r *ReconcileVitessBackupsSchedule) getAllShardsInKeyspace(ctx context.Context, namespace, cluster, keyspace string) ([]string, error) { + shardsList := &planetscalev2.VitessShardList{} + listOpts := &client.ListOptions{ + Namespace: namespace, + LabelSelector: apilabels.Set{ + planetscalev2.ClusterLabel: cluster, + planetscalev2.KeyspaceLabel: keyspace, + }.AsSelector(), + } + if err := r.client.List(ctx, shardsList, listOpts); err != nil { + return nil, fmt.Errorf("unable to list shards of keyspace %s in %s: %v", keyspace, namespace, err) + } + var result []string + for _, item := range shardsList.Items { + result = append(result, item.Spec.Name) + } + return result, nil +} + +type keyspace struct { + name string + shards []string +} + +func (r *ReconcileVitessBackupsSchedule) getAllShardsInCluster(ctx context.Context, namespace, cluster string) ([]keyspace, error) { + ksList := &planetscalev2.VitessKeyspaceList{} + listOpts := &client.ListOptions{ + Namespace: namespace, + LabelSelector: apilabels.Set{ + planetscalev2.ClusterLabel: cluster, + }.AsSelector(), + } + if err := r.client.List(ctx, ksList, listOpts); err != nil { + return nil, fmt.Errorf("unable to list shards in namespace %s: %v", namespace, err) + } + result := make([]keyspace, 0, len(ksList.Items)) + for _, item := range ksList.Items { + ks := keyspace{ + name: item.Spec.Name, + } + for shardName := range item.Status.Shards { + ks.shards = append(ks.shards, shardName) + } + if len(ks.shards) > 0 { + result = append(result, ks) + } + } + return result, nil +} diff --git a/pkg/controller/vitesscluster/reconcile_backup_schedule.go b/pkg/controller/vitesscluster/reconcile_backup_schedule.go new file mode 100644 index 00000000..f8d777b3 --- /dev/null +++ b/pkg/controller/vitesscluster/reconcile_backup_schedule.go @@ -0,0 +1,76 @@ +/* +Copyright 2024 PlanetScale Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vitesscluster + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "planetscale.dev/vitess-operator/pkg/operator/vitessbackup" + "sigs.k8s.io/controller-runtime/pkg/client" + + planetscalev2 "planetscale.dev/vitess-operator/pkg/apis/planetscale/v2" + "planetscale.dev/vitess-operator/pkg/operator/reconciler" +) + +func (r *ReconcileVitessCluster) reconcileBackupSchedule(ctx context.Context, vt *planetscalev2.VitessCluster) error { + labels := map[string]string{ + planetscalev2.ClusterLabel: vt.Name, + } + + // Generate keys (object names) for all desired cells. + // Keep a map back from generated names to the specs. + var keys []client.ObjectKey + scheduleMap := make(map[client.ObjectKey]*planetscalev2.VitessBackupScheduleTemplate) + if vt.Spec.Backup != nil { + for i := range vt.Spec.Backup.Schedules { + schedule := &vt.Spec.Backup.Schedules[i] + key := client.ObjectKey{ + Namespace: vt.Namespace, + Name: vitessbackup.ScheduleName(vt.Name, schedule.Name), + } + keys = append(keys, key) + scheduleMap[key] = schedule + } + } + + return r.reconciler.ReconcileObjectSet(ctx, vt, keys, labels, reconciler.Strategy{ + Kind: &planetscalev2.VitessBackupSchedule{}, + + New: func(key client.ObjectKey) runtime.Object { + vbsc := vitessbackup.NewVitessBackupSchedule(key, vt, scheduleMap[key], labels) + if vbsc == nil { + return &planetscalev2.VitessBackupSchedule{} + } + return vbsc + }, + + UpdateInPlace: func(key client.ObjectKey, obj runtime.Object) { + newObj := obj.(*planetscalev2.VitessBackupSchedule) + newVbsc := vitessbackup.NewVitessBackupSchedule(key, vt, scheduleMap[key], labels) + if newVbsc == nil { + return + } + newObj.Spec = newVbsc.Spec + }, + + PrepareForTurndown: func(key client.ObjectKey, newObj runtime.Object) *planetscalev2.OrphanStatus { + // If we want to remove the schedule, delete it immediately. + return nil + }, + }) +} diff --git a/pkg/controller/vitesscluster/vitesscluster_controller.go b/pkg/controller/vitesscluster/vitesscluster_controller.go index 98933312..a7d2a7bb 100644 --- a/pkg/controller/vitesscluster/vitesscluster_controller.go +++ b/pkg/controller/vitesscluster/vitesscluster_controller.go @@ -186,6 +186,11 @@ func (r *ReconcileVitessCluster) Reconcile(cctx context.Context, request reconci resultBuilder.Error(err) } + // Create/update VitessBackupSchedule object. + if err := r.reconcileBackupSchedule(ctx, vt); err != nil { + resultBuilder.Error(err) + } + // Create/update desired VitessCells. if err := r.reconcileCells(ctx, vt); err != nil { resultBuilder.Error(err) diff --git a/pkg/operator/metrics/metrics.go b/pkg/operator/metrics/metrics.go index b53af6e3..198ac47a 100644 --- a/pkg/operator/metrics/metrics.go +++ b/pkg/operator/metrics/metrics.go @@ -34,6 +34,8 @@ const ( ShardLabel = "shard" // BackupStorageLabel is the label whose value gives the name of a VitessBackupStorage object. BackupStorageLabel = "backup_storage" + // BackupScheduleLabel is the label whose value gives the name of a VitessBackupSchedule object. + BackupScheduleLabel = "backup_schedule" // ResultLabel is a common metrics label for the success/failure of an operation. ResultLabel = "result" diff --git a/pkg/operator/vitessbackup/schedule.go b/pkg/operator/vitessbackup/schedule.go new file mode 100644 index 00000000..fc5ced6b --- /dev/null +++ b/pkg/operator/vitessbackup/schedule.go @@ -0,0 +1,53 @@ +/* +Copyright 2024 PlanetScale Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vitessbackup + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + planetscalev2 "planetscale.dev/vitess-operator/pkg/apis/planetscale/v2" + "planetscale.dev/vitess-operator/pkg/operator/names" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func ScheduleName(clusterName string, scheduleName string) string { + return names.JoinWithConstraints(names.DefaultConstraints, clusterName, "vbsc", scheduleName) +} + +func NewVitessBackupSchedule(key client.ObjectKey, vt *planetscalev2.VitessCluster, vbsc *planetscalev2.VitessBackupScheduleTemplate, labels map[string]string) *planetscalev2.VitessBackupSchedule { + if vt.Spec.Backup == nil || vbsc == nil || vbsc.Schedule == "" { + return nil + } + + return &planetscalev2.VitessBackupSchedule{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + Labels: labels, + }, + Spec: planetscalev2.VitessBackupScheduleSpec{ + // We simply re-apply the same template that was written by the user. + VitessBackupScheduleTemplate: *vbsc, + + Cluster: vt.Name, + + // To take backups we only care about having the vtctldclient installed in the container. + // For this reason, we re-use the vtctld Docker image and the same image pull policy. + Image: vt.Spec.Images.Vtctld, + ImagePullPolicy: vt.Spec.ImagePullPolicies.Vtctld, + }, + } +} diff --git a/test/endtoend/backup_restore_test.sh b/test/endtoend/backup_restore_test.sh index e79dde58..e0f3bcad 100755 --- a/test/endtoend/backup_restore_test.sh +++ b/test/endtoend/backup_restore_test.sh @@ -30,14 +30,14 @@ function resurrectShard() { sleep 5 echo "show databases;" | mysql | grep "commerce" > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo "Could not find commerce database" printMysqlErrorFiles exit 1 fi echo "show tables;" | mysql commerce | grep -E 'corder|customer|product' | wc -l | grep 3 > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo "Could not find commerce's tables" printMysqlErrorFiles exit 1 @@ -76,7 +76,7 @@ EOF } function setupKindConfig() { - if [ "$BUILDKITE_BUILD_ID" != "0" ]; then + if [[ "$BUILDKITE_BUILD_ID" != "0" ]]; then # The script is being run from buildkite, so we can't mount the current # working directory to kind. The current directory in the docker is workdir # So if we try and mount that, we get an error. Instead we need to mount the @@ -99,7 +99,7 @@ docker build -f build/Dockerfile.release -t vitess-operator-pr:latest . echo "Setting up the kind config" setupKindConfig echo "Creating Kind cluster" -kind create cluster --wait 30s --name kind-${BUILDKITE_BUILD_ID} --config ./vtdataroot/config.yaml --image kindest/node:v1.28.0 +kind create cluster --wait 30s --name kind-${BUILDKITE_BUILD_ID} --config ./vtdataroot/config.yaml --image ${KIND_VERSION} echo "Loading docker image into Kind cluster" kind load docker-image vitess-operator-pr:latest --name kind-${BUILDKITE_BUILD_ID} diff --git a/test/endtoend/backup_schedule_test.sh b/test/endtoend/backup_schedule_test.sh new file mode 100755 index 00000000..2a65dc7c --- /dev/null +++ b/test/endtoend/backup_schedule_test.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +source ./tools/test.env +source ./test/endtoend/utils.sh + +function takedownShard() { + echo "Apply 102_keyspace_teardown.yaml" + kubectl apply -f 102_keyspace_teardown.yaml + + # wait for all the vttablets to disappear + checkPodStatusWithTimeout "example-vttablet-zone1" 0 +} + +function checkVitessBackupScheduleStatusWithTimeout() { + regex=$1 + + for i in {1..1200} ; do + if [[ $(kubectl get VitessBackupSchedule | grep -E "${regex}" | wc -l) -eq 1 ]]; then + echo "$regex found" + return + fi + sleep 1 + done + echo -e "ERROR: checkPodStatusWithTimeout timeout to find pod matching:\ngot:\n$out\nfor regex: $regex" + exit 1 +} + +function verifyListBackupsOutputWithSchedule() { + echo -e "Check for VitessBackupSchedule status" + checkVitessBackupScheduleStatusWithTimeout "example-vbsc-every-minute(.*)" + checkVitessBackupScheduleStatusWithTimeout "example-vbsc-every-five-minute(.*)" + + echo -e "Check for number of backups in the cluster" + # Sleep for 6 minutes, during this time we should have at the very minimum 7 backups. + # At least: 6 backups from the every-minute schedule, and 1 backup from the every-five-minute schedule. + sleep 360 + + backupCount=$(kubectl get vtb --no-headers | wc -l) + if [[ "${backupCount}" -lt 7 ]]; then + echo "Did not find at least 7 backups" + return 0 + fi + + echo -e "Check for Jobs' pods" + checkPodStatusWithTimeout "example-vbsc-every-minute-(.*)0/1(.*)Completed(.*)" 3 + checkPodStatusWithTimeout "example-vbsc-every-five-minute-(.*)0/1(.*)Completed(.*)" 2 +} + +function setupKindConfig() { + if [[ "$BUILDKITE_BUILD_ID" != "0" ]]; then + # The script is being run from buildkite, so we can't mount the current + # working directory to kind. The current directory in the docker is workdir + # So if we try and mount that, we get an error. Instead we need to mount the + # path where the code was checked out be buildkite + dockerContainerName=$(docker container ls --filter "ancestor=docker" --format '{{.Names}}') + CHECKOUT_PATH=$(docker container inspect -f '{{range .Mounts}}{{ if eq .Destination "/workdir" }}{{println .Source }}{{ end }}{{end}}' "$dockerContainerName") + BACKUP_DIR="$CHECKOUT_PATH/vtdataroot/backup" + else + BACKUP_DIR="$PWD/vtdataroot/backup" + fi + cat ./test/endtoend/kindBackupConfig.yaml | sed "s,PATH,$BACKUP_DIR,1" > ./vtdataroot/config.yaml +} + +# Test setup +STARTING_DIR="$PWD" +echo "Make temporary directory for the test" +mkdir -p -m 777 ./vtdataroot/backup +echo "Building the docker image" +docker build -f build/Dockerfile.release -t vitess-operator-pr:latest . +echo "Setting up the kind config" +setupKindConfig +echo "Creating Kind cluster" +kind create cluster --wait 30s --name kind-${BUILDKITE_BUILD_ID} --config ./vtdataroot/config.yaml --image ${KIND_VERSION} +echo "Loading docker image into Kind cluster" +kind load docker-image vitess-operator-pr:latest --name kind-${BUILDKITE_BUILD_ID} + +cd "$PWD/test/endtoend/operator" +killall kubectl +setupKubectlAccessForCI + +get_started "operator-latest.yaml" "101_initial_cluster_backup_schedule.yaml" +verifyVtGateVersion "20.0.0" +checkSemiSyncSetup +verifyListBackupsOutputWithSchedule + +echo "Removing the temporary directory" +removeBackupFiles +rm -rf "$STARTING_DIR/vtdataroot" +echo "Deleting Kind cluster. This also deletes the volume associated with it" +kind delete cluster --name kind-${BUILDKITE_BUILD_ID} diff --git a/test/endtoend/operator/101_initial_cluster_backup_schedule.yaml b/test/endtoend/operator/101_initial_cluster_backup_schedule.yaml new file mode 100644 index 00000000..c1cf7737 --- /dev/null +++ b/test/endtoend/operator/101_initial_cluster_backup_schedule.yaml @@ -0,0 +1,248 @@ +# The following example is minimalist. The security policies +# and resource specifications are not meant to be used in production. +# Please refer to the operator documentation for recommendations on +# production settings. +apiVersion: planetscale.com/v2 +kind: VitessCluster +metadata: + name: example +spec: + backup: + engine: xtrabackup + locations: + - volume: + hostPath: + path: /backup + type: Directory + schedules: + - name: "every-minute" + schedule: "* * * * *" + resources: + requests: + cpu: 100m + memory: 1024Mi + limits: + memory: 1024Mi + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 3 + jobTimeoutMinute: 5 + strategies: + - name: BackupShard + keyspace: "commerce" + shard: "-" + - name: "every-five-minute" + schedule: "*/5 * * * *" + resources: + requests: + cpu: 100m + memory: 1024Mi + limits: + memory: 1024Mi + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 3 + jobTimeoutMinute: 5 + strategies: + - name: BackupShard + keyspace: "commerce" + shard: "-" + images: + vtctld: vitess/lite:latest + vtgate: vitess/lite:latest + vttablet: vitess/lite:latest + vtorc: vitess/lite:latest + vtbackup: vitess/lite:latest + mysqld: + mysql80Compatible: mysql:8.0.30 + mysqldExporter: prom/mysqld-exporter:v0.11.0 + cells: + - name: zone1 + gateway: + authentication: + static: + secret: + name: example-cluster-config + key: users.json + replicas: 1 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 256Mi + vitessDashboard: + cells: + - zone1 + extraFlags: + security_policy: read-only + replicas: 1 + resources: + limits: + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + + keyspaces: + - name: commerce + durabilityPolicy: semi_sync + turndownPolicy: Immediate + vitessOrchestrator: + resources: + limits: + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + extraFlags: + recovery-period-block-duration: 5s + partitionings: + - equal: + parts: 1 + shardTemplate: + databaseInitScriptSecret: + name: example-cluster-config + key: init_db.sql + tabletPools: + - cell: zone1 + type: replica + replicas: 3 + vttablet: + extraFlags: + db_charset: utf8mb4 + wait_for_backup_interval: "0" + resources: + limits: + memory: 1024Mi + requests: + cpu: 100m + memory: 1024Mi + mysqld: + resources: + limits: + memory: 1024Mi + requests: + cpu: 100m + memory: 512Mi + dataVolumeClaimTemplate: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi + updateStrategy: + type: Immediate +--- +apiVersion: v1 +kind: Secret +metadata: + name: example-cluster-config +type: Opaque +stringData: + users.json: | + { + "user": [{ + "UserData": "user", + "Password": "" + }] + } + init_db.sql: | + # This file is executed immediately after mysql_install_db, + # to initialize a fresh data directory. + + ############################################################################### + # Equivalent of mysql_secure_installation + ############################################################################### + + # We need to ensure that super_read_only is disabled so that we can execute + # these commands. Note that disabling it does NOT disable read_only. + # We save the current value so that we only re-enable it at the end if it was + # enabled before. + SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF'); + SET GLOBAL super_read_only='OFF'; + + # Changes during the init db should not make it to the binlog. + # They could potentially create errant transactions on replicas. + SET sql_log_bin = 0; + # Remove anonymous users. + DELETE FROM mysql.user WHERE User = ''; + + # Disable remote root access (only allow UNIX socket). + DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost'; + + # Remove test database. + DROP DATABASE IF EXISTS test; + + ############################################################################### + # Vitess defaults + ############################################################################### + + # Vitess-internal database. + CREATE DATABASE IF NOT EXISTS _vt; + # Note that definitions of local_metadata and shard_metadata should be the same + # as in production which is defined in go/vt/mysqlctl/metadata_tables.go. + CREATE TABLE IF NOT EXISTS _vt.local_metadata ( + name VARCHAR(255) NOT NULL, + value VARCHAR(255) NOT NULL, + db_name VARBINARY(255) NOT NULL, + PRIMARY KEY (db_name, name) + ) ENGINE=InnoDB; + CREATE TABLE IF NOT EXISTS _vt.shard_metadata ( + name VARCHAR(255) NOT NULL, + value MEDIUMBLOB NOT NULL, + db_name VARBINARY(255) NOT NULL, + PRIMARY KEY (db_name, name) + ) ENGINE=InnoDB; + + # Admin user with all privileges. + CREATE USER 'vt_dba'@'localhost'; + GRANT ALL ON *.* TO 'vt_dba'@'localhost'; + GRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost'; + + # User for app traffic, with global read-write access. + CREATE USER 'vt_app'@'localhost'; + GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, + REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, + LOCK TABLES, EXECUTE, REPLICATION CLIENT, CREATE VIEW, + SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER + ON *.* TO 'vt_app'@'localhost'; + + # User for app debug traffic, with global read access. + CREATE USER 'vt_appdebug'@'localhost'; + GRANT SELECT, SHOW DATABASES, PROCESS ON *.* TO 'vt_appdebug'@'localhost'; + + # User for administrative operations that need to be executed as non-SUPER. + # Same permissions as vt_app here. + CREATE USER 'vt_allprivs'@'localhost'; + GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, + REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, + LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW, + SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER + ON *.* TO 'vt_allprivs'@'localhost'; + + # User for slave replication connections. + # TODO: Should we set a password on this since it allows remote connections? + CREATE USER 'vt_repl'@'%'; + GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%'; + + # User for Vitess filtered replication (binlog player). + CREATE USER 'vt_filtered'@'localhost'; + GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, + REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES, + LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW, + SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER + ON *.* TO 'vt_filtered'@'localhost'; + + # User for Orchestrator (https://github.com/openark/orchestrator). + # TODO: Reenable when the password is randomly generated. + #CREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password'; + #GRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD + # ON *.* TO 'orc_client_user'@'%'; + #GRANT SELECT + # ON _vt.* TO 'orc_client_user'@'%'; + + FLUSH PRIVILEGES; + + RESET SLAVE ALL; + RESET MASTER; + + # We need to set super_read_only back to what it was before + SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'ON'); \ No newline at end of file diff --git a/test/endtoend/operator/operator-latest.yaml b/test/endtoend/operator/operator-latest.yaml index b687b7d7..b2e6122f 100644 --- a/test/endtoend/operator/operator-latest.yaml +++ b/test/endtoend/operator/operator-latest.yaml @@ -1,4 +1,3 @@ ---- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -382,6 +381,177 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: vitessbackupschedules.planetscale.com +spec: + group: planetscale.com + names: + kind: VitessBackupSchedule + listKind: VitessBackupScheduleList + plural: vitessbackupschedules + singular: vitessbackupschedule + scope: Namespaced + versions: + - name: v2 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + affinity: + x-kubernetes-preserve-unknown-fields: true + allowedMissedRun: + minimum: 0 + type: integer + annotations: + additionalProperties: + type: string + type: object + cluster: + type: string + concurrencyPolicy: + enum: + - Allow + - Forbid + example: Forbid + type: string + failedJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + image: + type: string + imagePullPolicy: + type: string + jobTimeoutMinute: + default: 10 + format: int32 + minimum: 0 + type: integer + name: + example: every-day + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$ + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + schedule: + example: 0 0 * * * + minLength: 0 + type: string + startingDeadlineSeconds: + format: int64 + minimum: 0 + type: integer + strategies: + items: + properties: + extraFlags: + additionalProperties: + type: string + type: object + keyspace: + example: commerce + type: string + name: + enum: + - BackupShard + type: string + shard: + example: '-' + type: string + required: + - keyspace + - name + - shard + type: object + minItems: 1 + type: array + successfulJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + suspend: + type: boolean + required: + - cluster + - name + - resources + - schedule + - strategies + type: object + status: + properties: + active: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + lastScheduledTime: + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.11.3 @@ -1470,6 +1640,114 @@ spec: type: object minItems: 1 type: array + schedules: + items: + properties: + affinity: + x-kubernetes-preserve-unknown-fields: true + allowedMissedRun: + minimum: 0 + type: integer + annotations: + additionalProperties: + type: string + type: object + concurrencyPolicy: + enum: + - Allow + - Forbid + example: Forbid + type: string + failedJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + jobTimeoutMinute: + default: 10 + format: int32 + minimum: 0 + type: integer + name: + example: every-day + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$ + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + schedule: + example: 0 0 * * * + minLength: 0 + type: string + startingDeadlineSeconds: + format: int64 + minimum: 0 + type: integer + strategies: + items: + properties: + extraFlags: + additionalProperties: + type: string + type: object + keyspace: + example: commerce + type: string + name: + enum: + - BackupShard + type: string + shard: + example: '-' + type: string + required: + - keyspace + - name + - shard + type: object + minItems: 1 + type: array + successfulJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + suspend: + type: boolean + required: + - name + - resources + - schedule + - strategies + type: object + type: array subcontroller: properties: serviceAccountName: @@ -2490,6 +2768,16 @@ spec: type: string durabilityPolicy: type: string + images: + properties: + mysqld: + properties: + mysql56Compatible: + type: string + mysql80Compatible: + type: string + type: object + type: object name: maxLength: 63 minLength: 1 @@ -6518,6 +6806,15 @@ rules: - vitessbackupstorages - vitessbackupstorages/status - vitessbackupstorages/finalizers + - vitessbackupschedules + - vitessbackupschedules/status + - vitessbackupschedules/finalizers + verbs: + - '*' + - apiGroups: + - batch + resources: + - jobs verbs: - '*' --- @@ -6533,6 +6830,22 @@ subjects: - kind: ServiceAccount name: vitess-operator --- +apiVersion: scheduling.k8s.io/v1 +description: Vitess components (vttablet, vtgate, vtctld, etcd) +globalDefault: false +kind: PriorityClass +metadata: + name: vitess +value: 1000 +--- +apiVersion: scheduling.k8s.io/v1 +description: The vitess-operator control plane. +globalDefault: false +kind: PriorityClass +metadata: + name: vitess-operator-control-plane +value: 5000 +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -6583,19 +6896,3 @@ spec: memory: 128Mi priorityClassName: vitess-operator-control-plane serviceAccountName: vitess-operator ---- -apiVersion: scheduling.k8s.io/v1 -description: The vitess-operator control plane. -globalDefault: false -kind: PriorityClass -metadata: - name: vitess-operator-control-plane -value: 5000 ---- -apiVersion: scheduling.k8s.io/v1 -description: Vitess components (vttablet, vtgate, vtctld, etcd) -globalDefault: false -kind: PriorityClass -metadata: - name: vitess -value: 1000 diff --git a/test/endtoend/operator/operator.yaml b/test/endtoend/operator/operator.yaml index ea437f21..755a1635 100644 --- a/test/endtoend/operator/operator.yaml +++ b/test/endtoend/operator/operator.yaml @@ -1,4 +1,3 @@ ---- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -382,6 +381,177 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: vitessbackupschedules.planetscale.com +spec: + group: planetscale.com + names: + kind: VitessBackupSchedule + listKind: VitessBackupScheduleList + plural: vitessbackupschedules + singular: vitessbackupschedule + scope: Namespaced + versions: + - name: v2 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + affinity: + x-kubernetes-preserve-unknown-fields: true + allowedMissedRun: + minimum: 0 + type: integer + annotations: + additionalProperties: + type: string + type: object + cluster: + type: string + concurrencyPolicy: + enum: + - Allow + - Forbid + example: Forbid + type: string + failedJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + image: + type: string + imagePullPolicy: + type: string + jobTimeoutMinute: + default: 10 + format: int32 + minimum: 0 + type: integer + name: + example: every-day + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$ + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + schedule: + example: 0 0 * * * + minLength: 0 + type: string + startingDeadlineSeconds: + format: int64 + minimum: 0 + type: integer + strategies: + items: + properties: + extraFlags: + additionalProperties: + type: string + type: object + keyspace: + example: commerce + type: string + name: + enum: + - BackupShard + type: string + shard: + example: '-' + type: string + required: + - keyspace + - name + - shard + type: object + minItems: 1 + type: array + successfulJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + suspend: + type: boolean + required: + - cluster + - name + - resources + - schedule + - strategies + type: object + status: + properties: + active: + items: + properties: + apiVersion: + type: string + fieldPath: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + resourceVersion: + type: string + uid: + type: string + type: object + x-kubernetes-map-type: atomic + type: array + lastScheduledTime: + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.11.3 @@ -1470,6 +1640,114 @@ spec: type: object minItems: 1 type: array + schedules: + items: + properties: + affinity: + x-kubernetes-preserve-unknown-fields: true + allowedMissedRun: + minimum: 0 + type: integer + annotations: + additionalProperties: + type: string + type: object + concurrencyPolicy: + enum: + - Allow + - Forbid + example: Forbid + type: string + failedJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + jobTimeoutMinute: + default: 10 + format: int32 + minimum: 0 + type: integer + name: + example: every-day + minLength: 1 + pattern: ^[A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z0-9])?$ + type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + schedule: + example: 0 0 * * * + minLength: 0 + type: string + startingDeadlineSeconds: + format: int64 + minimum: 0 + type: integer + strategies: + items: + properties: + extraFlags: + additionalProperties: + type: string + type: object + keyspace: + example: commerce + type: string + name: + enum: + - BackupShard + type: string + shard: + example: '-' + type: string + required: + - keyspace + - name + - shard + type: object + minItems: 1 + type: array + successfulJobsHistoryLimit: + format: int32 + minimum: 0 + type: integer + suspend: + type: boolean + required: + - name + - resources + - schedule + - strategies + type: object + type: array subcontroller: properties: serviceAccountName: @@ -2490,6 +2768,16 @@ spec: type: string durabilityPolicy: type: string + images: + properties: + mysqld: + properties: + mysql56Compatible: + type: string + mysql80Compatible: + type: string + type: object + type: object name: maxLength: 63 minLength: 1 @@ -6518,6 +6806,15 @@ rules: - vitessbackupstorages - vitessbackupstorages/status - vitessbackupstorages/finalizers + - vitessbackupschedules + - vitessbackupschedules/status + - vitessbackupschedules/finalizers + verbs: + - '*' + - apiGroups: + - batch + resources: + - jobs verbs: - '*' --- @@ -6533,6 +6830,22 @@ subjects: - kind: ServiceAccount name: vitess-operator --- +apiVersion: scheduling.k8s.io/v1 +description: Vitess components (vttablet, vtgate, vtctld, etcd) +globalDefault: false +kind: PriorityClass +metadata: + name: vitess +value: 1000 +--- +apiVersion: scheduling.k8s.io/v1 +description: The vitess-operator control plane. +globalDefault: false +kind: PriorityClass +metadata: + name: vitess-operator-control-plane +value: 5000 +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -6582,19 +6895,3 @@ spec: memory: 128Mi priorityClassName: vitess-operator-control-plane serviceAccountName: vitess-operator ---- -apiVersion: scheduling.k8s.io/v1 -description: The vitess-operator control plane. -globalDefault: false -kind: PriorityClass -metadata: - name: vitess-operator-control-plane -value: 5000 ---- -apiVersion: scheduling.k8s.io/v1 -description: Vitess components (vttablet, vtgate, vtctld, etcd) -globalDefault: false -kind: PriorityClass -metadata: - name: vitess -value: 1000 diff --git a/test/endtoend/upgrade_test.sh b/test/endtoend/upgrade_test.sh index 22d6e1cc..da32dab3 100755 --- a/test/endtoend/upgrade_test.sh +++ b/test/endtoend/upgrade_test.sh @@ -236,7 +236,7 @@ EOF echo "Building the docker image" docker build -f build/Dockerfile.release -t vitess-operator-pr:latest . echo "Creating Kind cluster" -kind create cluster --wait 30s --name kind-${BUILDKITE_BUILD_ID} --image kindest/node:v1.28.0 +kind create cluster --wait 30s --name kind-${BUILDKITE_BUILD_ID} --image ${KIND_VERSION} echo "Loading docker image into Kind cluster" kind load docker-image vitess-operator-pr:latest --name kind-${BUILDKITE_BUILD_ID} diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index d58a5eab..bf685730 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -20,7 +20,7 @@ function checkSemiSyncWithRetry() { vttablet=$1 for i in {1..600} ; do kubectl exec "$vttablet" -c mysqld -- mysql -S "/vt/socket/mysql.sock" -u root -e "show variables like 'rpl_semi_sync_%_enabled'" | grep "ON" - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then return fi sleep 1 @@ -44,7 +44,7 @@ function runSQLWithRetry() { query=$1 for i in {1..600} ; do mysql -e "$query" - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then return fi echo "failed to run query $query, retrying (attempt #$i) ..." @@ -94,7 +94,7 @@ function takeBackup() { for i in {1..600} ; do out=$(kubectl get vtb --no-headers | wc -l) echo "$out" | grep "$finalBackupCount" > /dev/null 2>&1 - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then echo "Backup created" return 0 fi @@ -109,7 +109,7 @@ function verifyListBackupsOutput() { for i in {1..600} ; do out=$(vtctldclient LegacyVtctlCommand -- ListBackups "$keyspaceShard" | wc -l) echo "$out" | grep "$backupCount" > /dev/null 2>&1 - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then echo "ListBackupsOutputCorrect" return 0 fi @@ -134,7 +134,7 @@ function checkPodStatusWithTimeout() { nb=$2 # Number of pods to match defaults to one - if [ -z "$nb" ]; then + if [[ -z "$nb" ]]; then nb=1 fi @@ -143,7 +143,7 @@ function checkPodStatusWithTimeout() { for i in {1..1200} ; do out=$(kubectl get pods) echo "$out" | grep -E "$regex" | wc -l | grep "$nb" > /dev/null 2>&1 - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then echo "$regex found" return fi @@ -151,7 +151,7 @@ function checkPodStatusWithTimeout() { done echo -e "ERROR: checkPodStatusWithTimeout timeout to find pod matching:\ngot:\n$out\nfor regex: $regex" echo "$regex" | grep "vttablet" > /dev/null 2>&1 - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then printMysqlErrorFiles fi exit 1 @@ -171,7 +171,7 @@ function ensurePodResourcesSet() { numContainers=$(echo "$out" | grep -E "$regex" | awk '{print $2}' | awk -F ',' '{print NF}') numContainersWithResources=$(echo "$out" | grep -E "$regex" | awk '{print $3}' | awk -F ',' '{print NF}') - if [ $numContainers != $numContainersWithResources ]; then + if [[ $numContainers != $numContainersWithResources ]]; then echo "one or more containers in pods with $regex do not have $resource set" exit 1 fi @@ -181,7 +181,7 @@ function ensurePodResourcesSet() { function insertWithRetry() { for i in {1..600} ; do mysql --table < ../common/delete_commerce_data.sql && mysql --table < ../common/insert_commerce_data.sql - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then return fi echo "failed to insert commerce data, retrying (attempt #$i) ..." @@ -194,7 +194,7 @@ function verifyVtGateVersion() { podName=$(kubectl get pods --no-headers -o custom-columns=":metadata.name" | grep "vtgate") data=$(kubectl logs "$podName" | head) echo "$data" | grep "$version" > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo -e "The vtgate version is incorrect, expected: $version, got:\n$data" exit 1 fi @@ -208,7 +208,7 @@ function verifyDurabilityPolicy() { durabilityPolicy=$2 data=$(vtctldclient LegacyVtctlCommand -- GetKeyspace "$keyspace") echo "$data" | grep "\"durability_policy\": \"$durabilityPolicy\"" > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo -e "The durability policy in $keyspace is incorrect, got:\n$data" exit 1 fi @@ -237,10 +237,10 @@ function applySchemaWithRetry() { drop_sql=$3 for i in {1..600} ; do vtctldclient ApplySchema --sql-file="$schema" $ks - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then return fi - if [ -n "$drop_sql" ]; then + if [[ -n "$drop_sql" ]]; then mysql --table < $drop_sql fi echo "failed to apply schema $schema, retrying (attempt #$i) ..." @@ -254,14 +254,14 @@ function assertSelect() { expected=$3 data=$(mysql --table < $sql) echo "$data" | grep "$expected" > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo -e "The data in $shard's tables is incorrect, got:\n$data" exit 1 fi } function setupKubectlAccessForCI() { - if [ "$BUILDKITE_BUILD_ID" != "0" ]; then + if [[ "$BUILDKITE_BUILD_ID" != "0" ]]; then # The script is being run from buildkite, so we need to do stuff # https://github.com/kubernetes-sigs/kind/issues/1846#issuecomment-691565834 # Since kind is running in a sibling container, communicating with it through kubectl is not trivial. @@ -301,7 +301,7 @@ function get_started() { applySchemaWithRetry create_commerce_schema.sql commerce drop_all_commerce_tables.sql vtctldclient ApplyVSchema --vschema-file="vschema_commerce_initial.json" commerce - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo "ApplySchema failed for initial commerce" printMysqlErrorFiles exit 1 @@ -309,14 +309,14 @@ function get_started() { sleep 5 echo "show databases;" | mysql | grep "commerce" > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo "Could not find commerce database" printMysqlErrorFiles exit 1 fi echo "show tables;" | mysql commerce | grep -E 'corder|customer|product' | wc -l | grep 3 > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo "Could not find commerce's tables" printMysqlErrorFiles exit 1 diff --git a/test/endtoend/vtorc_vtadmin_test.sh b/test/endtoend/vtorc_vtadmin_test.sh index de515671..91767fb5 100755 --- a/test/endtoend/vtorc_vtadmin_test.sh +++ b/test/endtoend/vtorc_vtadmin_test.sh @@ -31,7 +31,7 @@ function get_started_vtorc_vtadmin() { applySchemaWithRetry create_commerce_schema.sql commerce drop_all_commerce_tables.sql vtctldclient ApplyVSchema --vschema-file="vschema_commerce_initial.json" commerce - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo "ApplySchema failed for initial commerce" printMysqlErrorFiles exit 1 @@ -39,14 +39,14 @@ function get_started_vtorc_vtadmin() { sleep 5 echo "show databases;" | mysql | grep "commerce" > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo "Could not find commerce database" printMysqlErrorFiles exit 1 fi echo "show tables;" | mysql commerce | grep -E 'corder|customer|product' | wc -l | grep 3 > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo "Could not find commerce's tables" printMysqlErrorFiles exit 1 @@ -159,9 +159,9 @@ function chromiumHeadlessRequest() { for i in {1..600} ; do chromiumBinary=$(getChromiumBinaryName) res=$($chromiumBinary --headless --no-sandbox --disable-gpu --enable-logging --dump-dom --virtual-time-budget=900000000 "$url") - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then echo "$res" | grep "$dataToAssert" > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo -e "The data in $url is incorrect, got:\n$res, retrying" sleep 1 continue @@ -175,12 +175,12 @@ function chromiumHeadlessRequest() { function getChromiumBinaryName() { which chromium-browser > /dev/null - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then echo "chromium-browser" return fi which chromium > /dev/null - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then echo "chromium" return fi @@ -191,9 +191,9 @@ function curlGetRequestWithRetry() { dataToAssert=$2 for i in {1..600} ; do res=$(curl "$url") - if [ $? -eq 0 ]; then + if [[ $? -eq 0 ]]; then echo "$res" | grep "$dataToAssert" > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo -e "The data in $url is incorrect, got:\n$res" exit 1 fi @@ -208,12 +208,12 @@ function curlDeleteRequest() { url=$1 dataToAssert=$2 res=$(curl -X DELETE "$url") - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo -e "The DELETE request to $url failed\n" exit 1 fi echo "$res" | grep "$dataToAssert" > /dev/null 2>&1 - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo -e "The data in delete request to $url is incorrect, got:\n$res" exit 1 fi @@ -223,7 +223,7 @@ function curlPostRequest() { url=$1 data=$2 curl -X POST -d "$data" "$url" - if [ $? -ne 0 ]; then + if [[ $? -ne 0 ]]; then echo -e "The POST request to $url with data $data failed\n" exit 1 fi @@ -233,7 +233,7 @@ function curlPostRequest() { echo "Building the docker image" docker build -f build/Dockerfile.release -t vitess-operator-pr:latest . echo "Creating Kind cluster" -kind create cluster --wait 30s --name kind-${BUILDKITE_BUILD_ID} --image kindest/node:v1.28.0 +kind create cluster --wait 30s --name kind-${BUILDKITE_BUILD_ID} --image ${KIND_VERSION} echo "Loading docker image into Kind cluster" kind load docker-image vitess-operator-pr:latest --name kind-${BUILDKITE_BUILD_ID} diff --git a/tools/test.env b/tools/test.env index c427317c..6532efd6 100755 --- a/tools/test.env +++ b/tools/test.env @@ -3,3 +3,5 @@ # We add the tools/_bin directory to PATH variable # since this is where we install the binaries that are needed export PATH="$PATH:$PWD/tools/_bin" + +export KIND_VERSION=kindest/node:v1.28.0 \ No newline at end of file