diff --git a/CHANGELOG.md b/CHANGELOG.md index 8de192768..4a1f56433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Added +- Add tls settings for BackupPolicy [#580](https://github.com/cybozu-go/moco/pull/580) + ## [0.17.0] - 2023-09-11 ### Breaking Changes diff --git a/api/v1beta1/job_types.go b/api/v1beta1/job_types.go index 7bd4c73b2..847f13b6a 100644 --- a/api/v1beta1/job_types.go +++ b/api/v1beta1/job_types.go @@ -218,4 +218,8 @@ type BucketConfig struct { // +kubebuilder:default=s3 // +optional BackendType string `json:"backendType,omitempty"` + + // Path to SSL CA certificate file used in addition to system default. + // +optional + CaCert string `json:"caCert,omitempty"` } diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go index 09ba74f0c..1849f4dac 100644 --- a/api/v1beta1/zz_generated.conversion.go +++ b/api/v1beta1/zz_generated.conversion.go @@ -433,6 +433,7 @@ func autoConvert__BucketConfig_To_v1beta2_BucketConfig(in *BucketConfig, out *v1 out.EndpointURL = in.EndpointURL out.UsePathStyle = in.UsePathStyle out.BackendType = in.BackendType + out.CaCert = in.CaCert return nil } @@ -447,6 +448,7 @@ func autoConvert_v1beta2_BucketConfig_To__BucketConfig(in *v1beta2.BucketConfig, out.EndpointURL = in.EndpointURL out.UsePathStyle = in.UsePathStyle out.BackendType = in.BackendType + out.CaCert = in.CaCert return nil } diff --git a/api/v1beta2/job_types.go b/api/v1beta2/job_types.go index 60f13ed93..8e59547c6 100644 --- a/api/v1beta2/job_types.go +++ b/api/v1beta2/job_types.go @@ -201,6 +201,10 @@ type BucketConfig struct { // +kubebuilder:default=s3 // +optional BackendType string `json:"backendType,omitempty"` + + // Path to SSL CA certificate file used in addition to system default. + // +optional + CaCert string `json:"caCert,omitempty"` } // AffinityApplyConfiguration is the type defined to implement the DeepCopy method. diff --git a/charts/moco/templates/generated/crds/moco_crds.yaml b/charts/moco/templates/generated/crds/moco_crds.yaml index 2c74256c9..c1c31c1dd 100644 --- a/charts/moco/templates/generated/crds/moco_crds.yaml +++ b/charts/moco/templates/generated/crds/moco_crds.yaml @@ -422,6 +422,9 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -2452,6 +2455,9 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -7722,6 +7728,9 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -13606,6 +13615,9 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* diff --git a/cmd/moco-backup/cmd/root.go b/cmd/moco-backup/cmd/root.go index a71419d1b..a42e287a2 100644 --- a/cmd/moco-backup/cmd/root.go +++ b/cmd/moco-backup/cmd/root.go @@ -2,8 +2,11 @@ package cmd import ( "context" + "crypto/tls" + "crypto/x509" "errors" "fmt" + "net/http" "net/url" "os" @@ -15,12 +18,13 @@ import ( ) var commonArgs struct { - workDir string - threads int - region string - endpointURL string - usePathStyle bool - backendType string + workDir string + threads int + region string + endpointURL string + usePathStyle bool + backendType string + caCertFilePath string } func makeBucket(bucketName string) (bucket.Bucket, error) { @@ -45,6 +49,27 @@ func makeS3Bucket(bucketName string) (bucket.Bucket, error) { if commonArgs.usePathStyle { opts = append(opts, bucket.WithPathStyle()) } + if len(commonArgs.caCertFilePath) > 0 { + caCertFile, err := os.ReadFile(commonArgs.caCertFilePath) + if err != nil { + return nil, err + } + caCertPool, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + if ok := caCertPool.AppendCertsFromPEM(caCertFile); !ok { + return nil, fmt.Errorf("failed to add ca cert") + } + transport := http.DefaultTransport.(*http.Transport).Clone() + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.RootCAs = caCertPool + opts = append(opts, bucket.WithHTTPClient(&http.Client{ + Transport: transport, + })) + } return bucket.NewS3Bucket(bucketName, opts...) } @@ -95,4 +120,5 @@ func init() { pf.StringVar(&commonArgs.endpointURL, "endpoint", "", "Object storage API endpoint URL") pf.BoolVar(&commonArgs.usePathStyle, "use-path-style", false, "Use path-style S3 API") pf.StringVar(&commonArgs.backendType, "backend-type", "s3", "The identifier for the object storage to be used.") + pf.StringVar(&commonArgs.caCertFilePath, "ca-cert", "", "Path to SSL CA certificate file used in addition to system default") } diff --git a/config/crd/bases/moco.cybozu.com_backuppolicies.yaml b/config/crd/bases/moco.cybozu.com_backuppolicies.yaml index e46c5380f..fda95959c 100644 --- a/config/crd/bases/moco.cybozu.com_backuppolicies.yaml +++ b/config/crd/bases/moco.cybozu.com_backuppolicies.yaml @@ -462,6 +462,10 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition + t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -2649,6 +2653,10 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition + t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* diff --git a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml index b034e42a1..21659d923 100644 --- a/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml +++ b/config/crd/bases/moco.cybozu.com_mysqlclusters.yaml @@ -3999,6 +3999,10 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition + t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -10430,6 +10434,10 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition + t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* diff --git a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml index da23fbdc5..e905e470f 100644 --- a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml +++ b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_backuppolicies.moco.cybozu.com.yaml @@ -461,6 +461,10 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition + t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -2648,6 +2652,10 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition + t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* diff --git a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml index c93e56071..7d698b41c 100644 --- a/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml +++ b/config/crd/tests/apiextensions.k8s.io_v1_customresourcedefinition_mysqlclusters.moco.cybozu.com.yaml @@ -4009,6 +4009,10 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition + t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* @@ -10440,6 +10444,10 @@ spec: description: The name of the bucket minLength: 1 type: string + caCert: + description: Path to SSL CA certificate file used in addition + t + type: string endpointURL: description: The API endpoint URL. pattern: ^https?://.* diff --git a/controllers/mysqlcluster_controller.go b/controllers/mysqlcluster_controller.go index 60b9a7cc3..ce9d015b5 100644 --- a/controllers/mysqlcluster_controller.go +++ b/controllers/mysqlcluster_controller.go @@ -997,6 +997,9 @@ func bucketArgs(bc mocov1beta2.BucketConfig) []string { if bc.BackendType != "" { args = append(args, "--backend-type="+bc.BackendType) } + if bc.CaCert != "" { + args = append(args, "--ca-cert="+bc.CaCert) + } return append(args, bc.BucketName) } diff --git a/docs/crd_backuppolicy_v1beta1.md b/docs/crd_backuppolicy_v1beta1.md index 4f1aa3fb6..84f519a26 100644 --- a/docs/crd_backuppolicy_v1beta1.md +++ b/docs/crd_backuppolicy_v1beta1.md @@ -60,6 +60,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | +| caCert | Path to SSL CA certificate file used in addition to system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_backuppolicy_v1beta2.md b/docs/crd_backuppolicy_v1beta2.md index 4f1aa3fb6..84f519a26 100644 --- a/docs/crd_backuppolicy_v1beta2.md +++ b/docs/crd_backuppolicy_v1beta2.md @@ -60,6 +60,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | +| caCert | Path to SSL CA certificate file used in addition to system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_mysqlcluster_v1beta1.md b/docs/crd_mysqlcluster_v1beta1.md index f82d11b5f..ccd95fb4d 100644 --- a/docs/crd_mysqlcluster_v1beta1.md +++ b/docs/crd_mysqlcluster_v1beta1.md @@ -181,6 +181,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | +| caCert | Path to SSL CA certificate file used in addition to system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/crd_mysqlcluster_v1beta2.md b/docs/crd_mysqlcluster_v1beta2.md index 1f85b7c3d..5b88b1a28 100644 --- a/docs/crd_mysqlcluster_v1beta2.md +++ b/docs/crd_mysqlcluster_v1beta2.md @@ -195,6 +195,7 @@ BucketConfig is a set of parameter to access an object storage bucket. | endpointURL | The API endpoint URL. Set this for non-S3 object storages. | string | false | | usePathStyle | Allows you to enable the client to use path-style addressing, i.e., https?://ENDPOINT/BUCKET/KEY. By default, a virtual-host addressing is used (https?://BUCKET.ENDPOINT/KEY). | bool | false | | backendType | BackendType is an identifier for the object storage to be used. | string | false | +| caCert | Path to SSL CA certificate file used in addition to system default. | string | false | [Back to Custom Resources](#custom-resources) diff --git a/docs/moco-backup.md b/docs/moco-backup.md index 6e5315110..b56dafdfb 100644 --- a/docs/moco-backup.md +++ b/docs/moco-backup.md @@ -19,6 +19,7 @@ Global Flags: --threads int The number of threads to be used (default 4) --use-path-style Use path-style S3 API --work-dir string The writable working directory (default "/work") + --ca-cert string Path to SSL CA certificate file used in addition to system default ``` ## Subcommands diff --git a/e2e/Makefile b/e2e/Makefile index 963e926ed..d36089083 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -48,6 +48,7 @@ endif $(KUSTOMIZE) build . | $(KUBECTL) apply -f - $(KUBECTL) -n moco-system wait --for=condition=available --timeout=180s --all deployments $(KUBECTL) apply -f minio.yaml + $(KUBECTL) apply -f minio-tls.yaml $(KUBECTL) apply -f fake-gcs-server.yaml $(KUBECTL) wait --timeout=60s --for=condition=Ready --all pods diff --git a/e2e/backup_test.go b/e2e/backup_test.go index d9e6cdffd..aa0772e50 100644 --- a/e2e/backup_test.go +++ b/e2e/backup_test.go @@ -4,8 +4,6 @@ import ( "bytes" _ "embed" "encoding/json" - "errors" - "fmt" "strconv" "strings" "text/template" @@ -19,6 +17,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +//go:embed testdata/makebucket.yaml +var makeBucketYAML string + //go:embed testdata/backup.yaml var backupYAML string @@ -33,28 +34,27 @@ var _ = Context("backup", func() { var restorePoint time.Time It("should create a bucket", func() { - kubectlSafe(nil, "run", "--command", "make-bucket", "--image=moco-backup:dev", "--", - "s3cmd", "--host=minio.default.svc:9000", "--host-bucket=minio.default.svc:9000", "--no-ssl", - "--access_key=minioadmin", "--secret_key=minioadmin", "mb", "s3://moco") + kubectlSafe([]byte(makeBucketYAML), "apply", "-f", "-") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "get", "jobs", "make-bucket", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "make-bucket has not been finished") + }).Should(Succeed()) }) It("should construct a source cluster", func() { kubectlSafe(fillTemplate(backupYAML), "apply", "-f", "-") - Eventually(func() error { + Eventually(func(g Gomega) { cluster, err := getCluster("backup", "source") - if err != nil { - return err - } - for _, cond := range cluster.Status.Conditions { - if cond.Type != mocov1beta2.ConditionHealthy { - continue - } - if cond.Status == metav1.ConditionTrue { - return nil - } - return fmt.Errorf("cluster is not healthy: %s", cond.Status) - } - return errors.New("no health condition") + g.Expect(err).NotTo(HaveOccurred()) + condHealthy, err := getClusterCondition(cluster, mocov1beta2.ConditionHealthy) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condHealthy.Status).To(Equal(metav1.ConditionTrue)) }).Should(Succeed()) kubectlSafe(nil, "moco", "-n", "backup", "mysql", "-u", "moco-writable", "source", "--", @@ -67,24 +67,15 @@ var _ = Context("backup", func() { It("should take a full dump", func() { kubectlSafe(nil, "-n", "backup", "create", "job", "--from=cronjob/moco-backup-source", "backup-1") - Eventually(func() error { + Eventually(func(g Gomega) { out, err := kubectl(nil, "-n", "backup", "get", "jobs", "backup-1", "-o", "json") - if err != nil { - return err - } + g.Expect(err).NotTo(HaveOccurred()) job := &batchv1.Job{} - if err := json.Unmarshal(out, job); err != nil { - return err - } - for _, cond := range job.Status.Conditions { - if cond.Type != batchv1.JobComplete { - continue - } - if cond.Status == corev1.ConditionTrue { - return nil - } - } - return errors.New("backup-1 has not been finished") + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "backup-1 has not been finished") }).Should(Succeed()) }) @@ -101,24 +92,15 @@ var _ = Context("backup", func() { time.Sleep(100 * time.Millisecond) kubectlSafe(nil, "-n", "backup", "create", "job", "--from=cronjob/moco-backup-source", "backup-2") - Eventually(func() error { + Eventually(func(g Gomega) { out, err := kubectl(nil, "-n", "backup", "get", "jobs", "backup-2", "-o", "json") - if err != nil { - return err - } + g.Expect(err).NotTo(HaveOccurred()) job := &batchv1.Job{} - if err := json.Unmarshal(out, job); err != nil { - return err - } - for _, cond := range job.Status.Conditions { - if cond.Type != batchv1.JobComplete { - continue - } - if cond.Status == corev1.ConditionTrue { - return nil - } - } - return errors.New("backup-2 has not been finished") + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "backup-2 has not been finished") }).Should(Succeed()) cluster, err := getCluster("backup", "source") @@ -142,20 +124,12 @@ var _ = Context("backup", func() { Expect(err).NotTo(HaveOccurred()) kubectlSafe(buf.Bytes(), "apply", "-f", "-") - Eventually(func() error { + Eventually(func(g Gomega) { cluster, err := getCluster("backup", "target") - if err != nil { - return err - } - for _, cond := range cluster.Status.Conditions { - if cond.Type != mocov1beta2.ConditionHealthy { - continue - } - if cond.Status == metav1.ConditionTrue { - return nil - } - } - return errors.New("target is not healthy") + g.Expect(err).NotTo(HaveOccurred()) + condHealthy, err := getClusterCondition(cluster, mocov1beta2.ConditionHealthy) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condHealthy.Status).To(Equal(metav1.ConditionTrue), "target is not healthy") }).Should(Succeed()) out := kubectlSafe(nil, "moco", "-n", "backup", "mysql", "target", "--", @@ -168,19 +142,13 @@ var _ = Context("backup", func() { It("should delete clusters", func() { kubectlSafe(nil, "delete", "-n", "backup", "mysqlclusters", "--all") - Eventually(func() error { + Eventually(func(g Gomega) { out, err := kubectl(nil, "get", "-n", "backup", "pod", "-o", "json") - if err != nil { - return err - } + g.Expect(err).NotTo(HaveOccurred()) pods := &corev1.PodList{} - if err := json.Unmarshal(out, pods); err != nil { - return err - } - if len(pods.Items) > 0 { - return errors.New("wait until all Pods are deleted") - } - return nil + err = json.Unmarshal(out, pods) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(pods.Items)).To(BeNumerically(">", 0), "wait until all Pods are deleted") }).Should(Succeed()) }) }) diff --git a/e2e/backup_tls_test.go b/e2e/backup_tls_test.go new file mode 100644 index 000000000..15e23f71c --- /dev/null +++ b/e2e/backup_tls_test.go @@ -0,0 +1,161 @@ +package e2e + +import ( + "bytes" + _ "embed" + "encoding/json" + "strconv" + "strings" + "text/template" + "time" + + mocov1beta2 "github.com/cybozu-go/moco/api/v1beta2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +//go:embed testdata/makebucket_tls.yaml +var makeBucketTLSYAML string + +//go:embed testdata/backup_tls.yaml +var backupTLSYAML string + +//go:embed testdata/restore_tls.yaml +var restoreTLSYAML string + +var _ = Context("backup-tls", func() { + if doUpgrade { + return + } + + var restorePoint time.Time + + It("should create a bucket", func() { + kubectlSafe([]byte(makeBucketTLSYAML), "apply", "-f", "-") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "get", "jobs", "make-bucket-tls", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "make-bucket-tls has not been finished") + }).Should(Succeed()) + }) + + It("should construct a source cluster", func() { + kubectlSafe(fillTemplate(backupTLSYAML), "apply", "-f", "-") + secjson := kubectlSafe(nil, "get", "secret", "-o", "json", "minio-cert") + sec := &corev1.Secret{} + json.Unmarshal(secjson, sec) + sec.SetNamespace("backup-tls") + secjson, err := json.Marshal(sec) + Expect(err).NotTo(HaveOccurred()) + kubectlSafe(secjson, "apply", "-f", "-") + Eventually(func(g Gomega) { + cluster, err := getCluster("backup-tls", "source") + g.Expect(err).NotTo(HaveOccurred()) + condHealthy, err := getClusterCondition(cluster, mocov1beta2.ConditionHealthy) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condHealthy.Status).To(Equal(metav1.ConditionTrue)) + }).Should(Succeed()) + + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-e", "CREATE DATABASE test") + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "-e", "CREATE TABLE t (id INT NOT NULL AUTO_INCREMENT, data VARCHAR(32) NOT NULL, PRIMARY KEY (id), KEY key1 (data), KEY key2 (data, id)) ENGINE=InnoDB") + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('aaa')") + }) + + It("should take a full dump", func() { + kubectlSafe(nil, "-n", "backup-tls", "create", "job", "--from=cronjob/moco-backup-source", "backup-tls-1") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "-n", "backup-tls", "get", "jobs", "backup-tls-1", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "backup-tls-1 has not been finished") + }).Should(Succeed()) + }) + + It("should take an incremental backup", func() { + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('bbb')") + time.Sleep(1100 * time.Millisecond) + restorePoint = time.Now().UTC() + time.Sleep(1100 * time.Millisecond) + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-admin", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "FLUSH LOCAL BINARY LOGS") + kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "-u", "moco-writable", "source", "--", + "-D", "test", "--init_command=SET autocommit=1", "-e", "INSERT INTO t (data) VALUES ('ccc')") + time.Sleep(100 * time.Millisecond) + + kubectlSafe(nil, "-n", "backup-tls", "create", "job", "--from=cronjob/moco-backup-source", "backup-tls-2") + Eventually(func(g Gomega) { + out, err := kubectl(nil, "-n", "backup-tls", "get", "jobs", "backup-tls-2", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = json.Unmarshal(out, job) + g.Expect(err).NotTo(HaveOccurred()) + condComplete, err := getJobCondition(job, batchv1.JobComplete) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condComplete.Status).To(Equal(corev1.ConditionTrue), "backup-tls-2 has not been finished") + }).Should(Succeed()) + + cluster, err := getCluster("backup-tls", "source") + Expect(err).NotTo(HaveOccurred()) + Expect(cluster.Status.Backup.BinlogSize).NotTo(Equal(int64(0))) + }) + + It("should destroy the source then restore the backup data", func() { + kubectlSafe(nil, "-n", "backup-tls", "delete", "mysqlclusters", "source") + + tmpl, err := template.New("").Parse(restoreTLSYAML) + Expect(err).NotTo(HaveOccurred()) + buf := new(bytes.Buffer) + err = tmpl.Execute(buf, struct { + MySQLVersion string + RestorePoint string + }{ + mysqlVersion, + restorePoint.Format(time.RFC3339), + }) + Expect(err).NotTo(HaveOccurred()) + + kubectlSafe(buf.Bytes(), "apply", "-f", "-") + Eventually(func(g Gomega) { + cluster, err := getCluster("backup-tls", "target") + g.Expect(err).NotTo(HaveOccurred()) + condHealthy, err := getClusterCondition(cluster, mocov1beta2.ConditionHealthy) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(condHealthy.Status).To(Equal(metav1.ConditionTrue), "target is not healthy") + }).Should(Succeed()) + + out := kubectlSafe(nil, "moco", "-n", "backup-tls", "mysql", "target", "--", + "-N", "-D", "test", "-e", "SELECT COUNT(*) FROM t") + count, err := strconv.Atoi(strings.TrimSpace(string(out))) + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(2)) + }) + + It("should delete clusters", func() { + kubectlSafe(nil, "delete", "-n", "backup-tls", "mysqlclusters", "--all") + + Eventually(func(g Gomega) { + out, err := kubectl(nil, "get", "-n", "backup-tls", "pod", "-o", "json") + g.Expect(err).NotTo(HaveOccurred()) + pods := &corev1.PodList{} + err = json.Unmarshal(out, pods) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(pods.Items)).To(BeNumerically(">", 0), "wait until all Pods are deleted") + }).Should(Succeed()) + }) +}) diff --git a/e2e/minio-tls.yaml b/e2e/minio-tls.yaml new file mode 100644 index 000000000..770cf9bae --- /dev/null +++ b/e2e/minio-tls.yaml @@ -0,0 +1,70 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + namespace: default + name: default-selfsigned-issuer +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + namespace: default + name: minio-cert +spec: + commonName: minio cert + issuerRef: + kind: Issuer + name: default-selfsigned-issuer + secretName: minio-cert + dnsNames: + - minio-tls.default.svc +--- +apiVersion: v1 +kind: Service +metadata: + namespace: default + name: minio-tls +spec: + ports: + - name: minio + port: 9000 + targetPort: minio + protocol: TCP + selector: + name: minio-tls +--- +apiVersion: v1 +kind: Pod +metadata: + namespace: default + name: minio-tls + labels: + name: minio-tls +spec: + containers: + - name: minio + image: minio/minio + args: + - server + - /data + ports: + - name: minio + containerPort: 9000 + protocol: TCP + volumeMounts: + - name: data + mountPath: /data + - name: secret-volume + mountPath: /root/.minio/certs + volumes: + - name: data + emptyDir: {} + - name: secret-volume + secret: + secretName: minio-cert + items: + - key: tls.crt + path: public.crt + - key: tls.key + path: private.key diff --git a/e2e/minio.yaml b/e2e/minio.yaml index 64bc6ee70..3c7bbb712 100644 --- a/e2e/minio.yaml +++ b/e2e/minio.yaml @@ -5,10 +5,10 @@ metadata: name: minio spec: ports: - - name: minio - port: 9000 - targetPort: minio - protocol: TCP + - name: minio + port: 9000 + targetPort: minio + protocol: TCP selector: name: minio --- @@ -21,18 +21,18 @@ metadata: name: minio spec: containers: - - name: minio - image: minio/minio - args: - - server - - /data - ports: - name: minio - containerPort: 9000 - protocol: TCP - volumeMounts: - - name: data - mountPath: /data + image: minio/minio + args: + - server + - /data + ports: + - name: minio + containerPort: 9000 + protocol: TCP + volumeMounts: + - name: data + mountPath: /data volumes: - - name: data - emptyDir: {} + - name: data + emptyDir: {} diff --git a/e2e/run_test.go b/e2e/run_test.go index 0d64ed202..b7a625ce5 100644 --- a/e2e/run_test.go +++ b/e2e/run_test.go @@ -13,8 +13,10 @@ import ( . "github.com/onsi/gomega" dto "github.com/prometheus/client_model/go" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func kubectl(input []byte, args ...string) ([]byte, error) { @@ -57,6 +59,26 @@ func getCluster(ns, name string) (*mocov1beta2.MySQLCluster, error) { return cluster, nil } +func getClusterCondition(cluster *mocov1beta2.MySQLCluster, condType string) (metav1.Condition, error) { + for _, cond := range cluster.Status.Conditions { + if cond.Type != condType { + continue + } + return cond, nil + } + return metav1.Condition{}, fmt.Errorf("no %s condition", condType) +} + +func getJobCondition(job *batchv1.Job, condType batchv1.JobConditionType) (batchv1.JobCondition, error) { + for _, cond := range job.Status.Conditions { + if cond.Type != condType { + continue + } + return cond, nil + } + return batchv1.JobCondition{}, fmt.Errorf("no %s condition", condType) +} + func fillTemplate(tmpl string) []byte { return fillTemplateWithVersion(tmpl, mysqlVersion) } diff --git a/e2e/testdata/backup.yaml b/e2e/testdata/backup.yaml index dfebab319..0ddde2c30 100644 --- a/e2e/testdata/backup.yaml +++ b/e2e/testdata/backup.yaml @@ -27,10 +27,10 @@ spec: jobConfig: serviceAccountName: backup-owner env: - - name: AWS_ACCESS_KEY_ID - value: minioadmin - - name: AWS_SECRET_ACCESS_KEY - value: minioadmin + - name: AWS_ACCESS_KEY_ID + value: minioadmin + - name: AWS_SECRET_ACCESS_KEY + value: minioadmin bucketConfig: bucketName: moco endpointURL: http://minio.default.svc:9000 @@ -50,13 +50,13 @@ spec: podTemplate: spec: containers: - - name: mysqld - image: ghcr.io/cybozu-go/moco/mysql:{{ . }} + - name: mysqld + image: ghcr.io/cybozu-go/moco/mysql:{{ . }} volumeClaimTemplates: - - metadata: - name: mysql-data - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/e2e/testdata/backup_tls.yaml b/e2e/testdata/backup_tls.yaml new file mode 100644 index 000000000..525b12ffc --- /dev/null +++ b/e2e/testdata/backup_tls.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: backup-tls +--- +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: backup-tls + name: mycnf +data: + innodb_log_file_size: "10M" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: backup-tls + name: backup-owner +--- +apiVersion: moco.cybozu.com/v1beta1 +kind: BackupPolicy +metadata: + namespace: backup-tls + name: daily +spec: + schedule: "@daily" + jobConfig: + serviceAccountName: backup-owner + env: + - name: AWS_ACCESS_KEY_ID + value: minioadmin + - name: AWS_SECRET_ACCESS_KEY + value: minioadmin + volumeMounts: + - mountPath: /minio-cert + name: minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert + bucketConfig: + bucketName: moco + endpointURL: https://minio-tls.default.svc:9000 + usePathStyle: true + caCert: /minio-cert/ca.crt + workVolume: + emptyDir: {} +--- +apiVersion: moco.cybozu.com/v1beta2 +kind: MySQLCluster +metadata: + namespace: backup-tls + name: source +spec: + mysqlConfigMapName: mycnf + replicas: 3 + backupPolicyName: daily + podTemplate: + spec: + containers: + - name: mysqld + image: ghcr.io/cybozu-go/moco/mysql:{{ . }} + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/e2e/testdata/makebucket.yaml b/e2e/testdata/makebucket.yaml new file mode 100644 index 000000000..18175e425 --- /dev/null +++ b/e2e/testdata/makebucket.yaml @@ -0,0 +1,22 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: make-bucket + namespace: default +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - command: + - s3cmd + - --host=minio.default.svc:9000 + - --host-bucket=minio.default.svc:9000 + - --access_key=minioadmin + - --secret_key=minioadmin + - --no-ssl + - mb + - s3://moco + image: moco-backup:dev + imagePullPolicy: IfNotPresent + name: make-bucket diff --git a/e2e/testdata/makebucket_tls.yaml b/e2e/testdata/makebucket_tls.yaml new file mode 100644 index 000000000..1ff37d6ea --- /dev/null +++ b/e2e/testdata/makebucket_tls.yaml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: make-bucket-tls + namespace: default +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - command: + - s3cmd + - --host=minio-tls.default.svc:9000 + - --host-bucket=minio-tls.default.svc:9000 + - --access_key=minioadmin + - --secret_key=minioadmin + - --ssl + - --ca-certs=/minio-cert/ca.crt + - mb + - s3://moco + image: moco-backup:dev + imagePullPolicy: IfNotPresent + name: make-bucket-tls + volumeMounts: + - name: minio-cert + mountPath: /minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert + items: + - key: ca.crt + path: ca.crt diff --git a/e2e/testdata/restore.yaml b/e2e/testdata/restore.yaml index 019f5c05c..7042a851a 100644 --- a/e2e/testdata/restore.yaml +++ b/e2e/testdata/restore.yaml @@ -13,10 +13,10 @@ spec: jobConfig: serviceAccountName: backup-owner env: - - name: AWS_ACCESS_KEY_ID - value: minioadmin - - name: AWS_SECRET_ACCESS_KEY - value: minioadmin + - name: AWS_ACCESS_KEY_ID + value: minioadmin + - name: AWS_SECRET_ACCESS_KEY + value: minioadmin bucketConfig: bucketName: moco endpointURL: http://minio.default.svc:9000 @@ -26,13 +26,13 @@ spec: podTemplate: spec: containers: - - name: mysqld - image: ghcr.io/cybozu-go/moco/mysql:{{ .MySQLVersion }} + - name: mysqld + image: ghcr.io/cybozu-go/moco/mysql:{{ .MySQLVersion }} volumeClaimTemplates: - - metadata: - name: mysql-data - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/e2e/testdata/restore_tls.yaml b/e2e/testdata/restore_tls.yaml new file mode 100644 index 000000000..f15fb21d1 --- /dev/null +++ b/e2e/testdata/restore_tls.yaml @@ -0,0 +1,46 @@ +apiVersion: moco.cybozu.com/v1beta2 +kind: MySQLCluster +metadata: + namespace: backup-tls + name: target +spec: + mysqlConfigMapName: mycnf + replicas: 1 + restore: + sourceName: source + sourceNamespace: backup-tls + restorePoint: "{{ .RestorePoint }}" + jobConfig: + serviceAccountName: backup-owner + env: + - name: AWS_ACCESS_KEY_ID + value: minioadmin + - name: AWS_SECRET_ACCESS_KEY + value: minioadmin + volumeMounts: + - mountPath: /minio-cert + name: minio-cert + volumes: + - name: minio-cert + secret: + secretName: minio-cert + bucketConfig: + bucketName: moco + endpointURL: https://minio-tls.default.svc:9000 + usePathStyle: true + caCert: /minio-cert/ca.crt + workVolume: + emptyDir: {} + podTemplate: + spec: + containers: + - name: mysqld + image: ghcr.io/cybozu-go/moco/mysql:{{ .MySQLVersion }} + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi