diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a192e54eb..9f72b24c6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,6 +19,7 @@ jobs: - run: make test - run: make check-generate - run: make envtest + - run: make release-build test-dbop: name: Test pkg/dbop strategy: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dcfea8b85..0c6be3ecc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - run: docker build . -t moco:dev + - run: docker build -t moco:dev . - name: Login to ghcr.io run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin - run: docker tag moco:dev ghcr.io/cybozu-go/moco:${GITHUB_REF#refs/tags/v} @@ -25,44 +25,13 @@ jobs: - uses: actions/setup-go@v2 with: go-version: ${{ env.go-version }} - - run: | - make release-build + - run: make release-build - name: Create Release - id: create_release - uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - body: | - See [CHANGELOG.md](./CHANGELOG.md) for details. - draft: false - prerelease: ${{ contains(github.ref, '-') }} - - name: Upload kubectl-moco for linux-amd64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./build/kubectl-moco-linux-amd64 - asset_name: kubectl-moco-linux-amd64 - asset_content_type: application/octet-stream - - name: Upload kubectl-moco for windows-amd64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./build/kubectl-moco-windows-amd64.exe - asset_name: kubectl-moco-windows-amd64.exe - asset_content_type: application/octet-stream - - name: Upload kubectl-moco for darwin-amd64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./build/kubectl-moco-darwin-amd64 - asset_name: kubectl-moco-darwin-amd64 - asset_content_type: application/octet-stream + run: | + tagname="${GITHUB_REF#refs/tags/}" + if echo ${{ github.ref }} | grep -q -e '-'; then prerelease=-p; fi + gh release create -t "Release $tagname" $prerelease \ + -n "See [CHANGELOG.md](./CHANGELOG.md) for details." \ + "$tagname" build/* diff --git a/CHANGELOG.md b/CHANGELOG.md index f6dae7ffb..dd791dd19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +## [0.8.0] - 2021-04-27 + +### Changed +- Everything. There is no backward compatibility. (#228) +- The older release must be uninstalled before installing this version. + ## [0.7.0] - 2021-02-22 Since v0.7.0, MOCO will no longer use CronJob for log rotation. @@ -131,7 +137,8 @@ The `MySQLCluster` created by MOCO `< v0.5.0` has no compatibility with `>= v0.5 - Bootstrap a vanilla MySQL cluster with no replicas (#2). -[Unreleased]: https://github.com/cybozu-go/moco/compare/v0.7.0...HEAD +[Unreleased]: https://github.com/cybozu-go/moco/compare/v0.8.0...HEAD +[0.8.0]: https://github.com/cybozu-go/moco/compare/v0.8.0...v0.8.0 [0.7.0]: https://github.com/cybozu-go/moco/compare/v0.6.0...v0.7.0 [0.6.0]: https://github.com/cybozu-go/moco/compare/v0.5.1...v0.6.0 [0.5.1]: https://github.com/cybozu-go/moco/compare/v0.5.0...v0.5.1 diff --git a/DEVELOP.md b/DEVELOP.md new file mode 100644 index 000000000..a898325d3 --- /dev/null +++ b/DEVELOP.md @@ -0,0 +1,45 @@ +# How to develop MOCO + +## Running tests + +MOCO has the following 4 kinds of tests: + +1. Tests that do not depend on MySQL or Kubernetes +2. `pkg/dbop` tests that depend on MySQL version +3. Tests that depend on Kubernetes and therefore run by controller-runtime's envtest +4. End-to-end tests + +To run these tests, use the following make targets respectively: + +1. `make test` +2. `make test-dbop` +3. `make envtest` +4. Read [`e2e/README.md`](e2e/README.md) + +## Generated files + +Some files in the repository are auto-generated. + +- [`docs/crd_mysqlcluster.md`](docs/crd_mysqlcluster.md) is generated by `make apidoc`. +- Some files under `config` are generated by `make manifests`. +- `api/**/*.deepcopy.go` are generated by `make generate`. + +CI checks and fails if they need to be rebuilt. + +## Testing with unreleased moco-agent + +MOCO depends on [moco-agent][] that is released from a different repository. +The dependency is therefore managed in `go.mod` file. + +To run e2e tests with an unreleased moco-agent, follow the instructions in +[`e2e/README.md`](e2e/README.md). + +## Updating moco-agent + +Run `go get github.com/cybozu-go/moco-agent@latest`. + +## Updating fluent-bit + +Edit `FluentBitImage` in [`version.go`](versoin.go). + +[moco-agent]: https://github.com/cybozu-go/moco-agent diff --git a/Makefile b/Makefile index 5c5ffbd32..6091c5b6c 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,11 @@ SHELL = /bin/bash # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS = "crd:crdVersions=v1" +# for Go +GOOS = $(shell go env GOOS) +GOARCH = $(shell go env GOARCH) +SUFFIX = + .PHONY: all all: build @@ -49,7 +54,7 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and .PHONY: apidoc apidoc: crd-to-markdown $(wildcard api/*/*_types.go) echo $(wildcard api/*/*_types.go) - $(CRD_TO_MARKDOWN) -f api/v1beta1/mysqlcluster_types.go -n MySQLCluster > docs/crd_mysqlcluster.md + $(CRD_TO_MARKDOWN) --links docs/links.csv -f api/v1beta1/mysqlcluster_types.go -n MySQLCluster > docs/crd_mysqlcluster.md .PHONY: check-generate check-generate: @@ -97,6 +102,22 @@ build: mkdir -p bin GOBIN=$(shell pwd)/bin go install ./cmd/... +.PHONY: release-build +release-build: kustomize + mkdir -p build + $(MAKE) kubectl-moco GOOS=windows GOARCH=amd64 SUFFIX=.exe + $(MAKE) kubectl-moco GOOS=darwin GOARCH=amd64 + $(MAKE) kubectl-moco GOOS=darwin GOARCH=arm64 + $(MAKE) kubectl-moco GOOS=linux GOARCH=amd64 + $(MAKE) kubectl-moco GOOS=linux GOARCH=arm64 + $(KUSTOMIZE) build . > build/moco.yaml + +.PHONY: kubectl-moco +kubectl-moco: build/kubectl-moco-$(GOOS)-$(GOARCH)$(SUFFIX) + +build/kubectl-moco-$(GOOS)-$(GOARCH)$(SUFFIX): + CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ ./cmd/kubectl-moco + ##@ Tools CONTROLLER_GEN := $(shell pwd)/bin/controller-gen diff --git a/README.md b/README.md index fbf817bff..d1e30e99a 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,117 @@ [![GitHub release](https://img.shields.io/github/release/cybozu-go/moco.svg?maxAge=60)][releases] -[![CircleCI](https://circleci.com/gh/cybozu-go/moco.svg?style=svg)](https://circleci.com/gh/cybozu-go/moco) +[![CI](https://github.com/cybozu-go/moco/actions/workflows/ci.yaml/badge.svg)](https://github.com/cybozu-go/moco/actions/workflows/ci.yaml) [![PkgGoDev](https://pkg.go.dev/badge/github.com/cybozu-go/moco)](https://pkg.go.dev/github.com/cybozu-go/moco) [![Go Report Card](https://goreportcard.com/badge/github.com/cybozu-go/moco)](https://goreportcard.com/report/github.com/cybozu-go/moco) # MOCO -MOCO is a Kubernetes operator for MySQL. -Its primary function is to manage a cluster of MySQL using binlog-based, semi-synchronous replication. +MOCO is a Kubernetes operator for [MySQL][]. +Its primary function is to manage MySQL clusters using [GTID-based](https://dev.mysql.com/doc/refman/8.0/en/replication-gtids.html) [semi-synchronous](https://dev.mysql.com/doc/refman/8.0/en/replication-semisync.html) replication. It does _not_ manage [group replication](https://dev.mysql.com/doc/refman/8.0/en/group-replication.html) clusters. -MOCO is designed for the following properties: +MOCO is designed to have the following properties. -- Durability - - Do not lose any data under a given degree of faults. +- Compatibility with the standard MySQL + - This is the reason that MOCO does not adopt group replication that has [a number of limitations](https://dev.mysql.com/doc/refman/8.0/en/group-replication-limitations.html). +- Safety + - MOCO only allows writes to a single instance called the primary at a time. + - MOCO configures loss-less semi-synchronous replication with sufficient replicas. + - MOCO detects and excludes instances having [errant transactions](https://www.percona.com/blog/2014/05/19/errant-transactions-major-hurdle-for-gtid-based-failover-in-mysql-5-6/). - Availability - - Keep the MySQL cluster available under a given degree of faults. -- Business Continuity - - Perform a quick recovery if some failure is occurred. + - MOCO can quickly switch the primary in case of the primary failure or restart. + - MOCO allows up to 5 instances in a cluster. -## Features - -TBD +## Supported software -## Supported MySQL versions +- MySQL: 8.0.18, 8.0.20, and 8.0.24 +- Kubernetes: 1.19 and 1.20 -8.0.18 and 8.0.20 +Other versions may work, though not tested. -## Supported Kubernetes version +## Features -1.20 and 1.19 +- Cluster of 1, 3, or 5 MySQL instances +- Replication from an external MySQL instance +- Manual and automatic switchover of the primary instance +- Automatic failover of the primary instance +- Errant transaction detection +- Different MySQL versions for each cluster +- Upgrading MySQL version of a cluster +- Monitor for replication delays +- Service for the primary and replicas, respectively +- Custom `my.cnf` configurations +- Custom Pod, Service, and PersistentVolumeClaim templates +- Redirect slow query logs to a sidecar container +- (planning) Backup and [Point-in-Time Recovery](https://dev.mysql.com/doc/refman/8.0/en/point-in-time-recovery-positions.html) + +## Quick start + +You can quickly run MOCO using [kind](https://kind.sigs.k8s.io/). + +1. Prepare a Linux machine and install Docker. +2. Checkout MOCO and go to `e2e` directory. +3. Run `make start` + +You can then create a three-instance MySQL cluster as follows: + +```console +$ cat > mycluster.yaml <<'EOF' +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + replicas: 3 + podTemplate: + spec: + containers: + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi +EOF + +$ export KUBECONFIG=$(pwd)/.kubeconfig +$ ../bin/kubectl apply -f mycluster.yaml +``` + +Check the status of MySQLCluster until it becomes healthy as follows: + +```console +$ ../bin/kubectl get mysqlcluster test +NAME AVAILABLE HEALTHY PRIMARY SYNCED REPLICAS ERRANT REPLICAS +test True True 0 3 +``` + +Once it becomes healthy, you can use `kubectl-moco` to play with `mysql` client. + +```console +$ ../bin/kubectl moco mysql -it test +``` + +To destroy the Kubernetes cluster, run: + +```console +$ make stop +``` ## Documentation -[docs](docs/) directory contains documents about designs and specifications. +- [`docs/setup.md`](docs/setup.md) for installing MOCO. +- [`docs/usage.md`](docs/usage.md) is the user manual of MOCO. +- [`examples`](examples/) directory contains example MySQLCluster manifests. -## Docker images - -Docker images are available on [Quay.io](https://quay.io/repository/cybozu/moco) +[`docs`](docs/) directory also contains other design or specification documents. -## License +## Docker images -MOCO is licensed under MIT license. +Docker images are available on [ghcr.io/cybozu-go/moco](https://github.com/orgs/cybozu-go/packages/container/package/moco). [releases]: https://github.com/cybozu-go/moco/releases -[godoc]: https://godoc.org/github.com/cybozu-go/moco +[MySQL]: https://www.mysql.com/ diff --git a/clustering/mock_test.go b/clustering/mock_test.go index c8c352476..2a65ad6b7 100644 --- a/clustering/mock_test.go +++ b/clustering/mock_test.go @@ -93,22 +93,25 @@ func (a *mockAgentConn) Clone(ctx context.Context, in *agent.CloneRequest, opts } func setPodReadiness(ctx context.Context, name string, ready bool) { - pod := &corev1.Pod{} - err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: name}, pod) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - if ready { - pod.Status.Conditions = []corev1.PodCondition{ - { - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }, + EventuallyWithOffset(1, func() error { + pod := &corev1.Pod{} + err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: name}, pod) + if err != nil { + return err } - } else { - pod.Status.Conditions = nil - } - err = k8sClient.Status().Update(ctx, pod) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + if ready { + pod.Status.Conditions = []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + } + } else { + pod.Status.Conditions = nil + } + return k8sClient.Status().Update(ctx, pod) + }).Should(Succeed()) } type mockAgentFactory struct { diff --git a/clustering/suite_test.go b/clustering/suite_test.go index cea5c1b1c..02b8d3083 100644 --- a/clustering/suite_test.go +++ b/clustering/suite_test.go @@ -35,7 +35,7 @@ var mysqlPassword *password.MySQLPassword func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) - SetDefaultEventuallyTimeout(10 * time.Second) + SetDefaultEventuallyTimeout(30 * time.Second) SetDefaultEventuallyPollingInterval(100 * time.Millisecond) SetDefaultConsistentlyDuration(3 * time.Second) SetDefaultConsistentlyPollingInterval(100 * time.Millisecond) diff --git a/controllers/mysql_container.go b/controllers/mysql_container.go index 243397afc..c071a932b 100644 --- a/controllers/mysql_container.go +++ b/controllers/mysql_container.go @@ -204,7 +204,7 @@ func (r *MySQLClusterReconciler) makeV1InitContainer(cluster *mocov1beta1.MySQLC Image: image, Command: []string{ constants.InitCommand, - "--base-dir=" + constants.MySQLDataPath, + "--data-dir=" + constants.MySQLDataPath, "--conf-dir=" + constants.MySQLInitConfPath, fmt.Sprintf("%d", cluster.Spec.ServerIDBase), }, diff --git a/docs/design.md b/docs/design.md index eb107045d..c8de54c53 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1,36 +1,31 @@ Design notes ============ -Motivation ----------- +## Motivation -This Kubernetes operator automates operations for the binlog-based replication on MySQL. +We are creating our own Kubernetes operator for clustering MySQL instances for the following reasons: -InnoDB cluster can be used for the replication purpose, but we choose not to use InnoDB cluster because it does not allow large (>2GB) transactions. +Firstly, our application requires strict-compatibility to the traditional MySQL. Although recent MySQL provides an advanced clustering solution called [group replication](https://dev.mysql.com/doc/refman/8.0/en/group-replication.html) that is based on [Paxos](https://en.wikipedia.org/wiki/Paxos_(computer_science)), we cannot use it because of [various limitations from group replication](https://dev.mysql.com/doc/refman/8.0/en/group-replication-limitations.html). -There are some existing operators which deploy a group of MySQL servers without InnoDB cluster but they does not support the Point-in-Time-Recovery(PiTR) feature. +Secondly, we want to have a Kubernetes native and the simplest operator. For example, we can use Kubernetes Service to load-balance read queries to multiple replicas. Also, we do not want to support non-GTID based replications. -- [oracle/mysql-operator](https://github.com/oracle/mysql-operator) takes backups only with `mysqldump` -- [presslabs/mysql-operator](https://github.com/presslabs/mysql-operator) does not restore clusters to the state at the desired Point-in-Time +Lastly, none of the existing operators could satisfy our requirements. -This operator deploys a group of MySQL servers which replicates data semi-synchronously to the replicas and takes backups with both `mysqlpump` and `mysqlbinlog`. +## Goals -In this context, we call the group of MySQL servers as MySQL cluster. - -Goals ------ - -- Have the primary replicate data semi-synchronously to the multiple replicas +- Manage primary-replica cluster of MySQL instances. + - The primary instance is the only instance that allows writes. + - Replica instances replicate data from the primary. +- Support replication from an external MySQL instance. - Support all the four transaction isolation levels. -- Avoid split-brain. +- No split-brain is allowed. - Accept large transactions. -- Upgrade this operator without restarting MySQL `Pod`s. +- Upgrade this operator without restarting MySQL Pods. - Support multiple MySQL versions and automatic upgrading. - Support automatic primary selection and switchover. - Support automatic failover. -- Support backups at least once in a day. -- Support a quick recovery by combining full backup and binary logs. -- Support asynchronous replication between remote data centers. +- Backup and restore features. +- Support point-in-time-recovery (PiTR). - Tenant users can specify the following parameters: - The version of MySQL instances. - The number of processor cores for each MySQL instance. @@ -41,32 +36,27 @@ Goals - Allow `CREATE / DROP TEMPORARY TABLE` during a transaction. - Use Custom Resource Definition(CRD) to automate construction of MySQL database using replication on Kubernetes. -Non-goals ---------- +## Non-goals -- Support for InnoDB cluster. -- Zero downtime upgrade. - Node fencing. - - Fencing should be done externally. Once Pod and PVC/PV are removed as a consequence of node fencing, the operator will restore the cluster appropriately. -Components ----------- + Fencing should be done externally. Once Pod and PVC/PV are removed as a consequence of node fencing, the operator will restore the cluster appropriately. + +## Components ### Workloads - Operator: Custom controller which automates MySQL cluster management with the following namespaced custom resources: - - [MySQLCluster](crd_mysql_cluster.md) represents a MySQL cluster. - - [ObjectStorage](crd_object_storage.md) represents a connection setting to an object storage which has Amazon S3 compatible API (e.g. Ceph RGW). + - [MySQLCluster](crd_mysql_cluster.md) represents a MySQL cluster. + - [ObjectStorage](crd_object_storage.md) represents a connection setting to an object storage which has Amazon S3 compatible API (e.g. Ceph RGW). - Admission Webhook: Webhook for validating custom resources (e.g. validate the object storage for backup exists). - [cert-manager](https://cert-manager.io/): Provide client certifications and primary-replica certifications automatically. ### Tools - `kubectl-moco`: CLI to manipulate MySQL cluster. It provides functionalities such as: - - Change primary manually. - - Port-forward to MySQL servers. - - Execute SQL like `mysql -u -p` without a credential file on a local environment. - - Fetch a credential file to a local environment. + - Execute `mysql` client for a MySQL instance running on Kubernetes. + - Fetch a credential file to a local environment. ### Diagram @@ -85,199 +75,5 @@ In this section, the name of `MySQLCluster` is assumed to be `mysql`. - `Service` for accessing replicas, both for MySQL protocol and X protocol. - `Secrets` to store credentials. - `ConfigMap` to store cluster configuration. - - `ServiceAccount`, `Role` and `RoleBinding` to allow Pods to access resources. Read [reconcile.md](reconcile.md) on how MOCO reconciles the StatefulSet. - -#### How to implement the initialization of MySQL pods with avoiding unnecessary restart at the operator update - -When initializing MySQL pods, the following procedures should be executed in their init containers. - -- Initialize data-dir. -- Create mysql users. -- Create `my.cnf`. -- etc. - -When upgrading the operator, we want to avoid unnecessary restarts of MySQL pods. -Creating mysql users and initializing data-dir are done in an init container of which image is the same with -the `mysqld` container, so the MySQL pods are not restarted at the upgrade. - -`my.cnf` is created by merging the following three configurations. - -- Default: This is created by the operator and contains default value of `my.cnf`. -- User: This is created by users and contains user-defined values of `my.cnf`. This overwrites Default values. -- Constant: This is created by the operator. This overwrites User values. - -The operator contains the Default and Constant configurations for all MySQL clusters, -and the User configuration for a certain MySQL cluster specified in the cluster's CR. -The operator merges these three files and creates a ConfigMap that will be mounted on MySQL Pods. -For instance-specific parameters such as `server-id`, an init container creates a supplementary config file. -The main `my.cnf` contains a stanza to include this supplementary config file. - -If users want to change their MySQL cluster configurations without restarting Pods, they should use `SET GLOBAL ...`. - -So, in short we prepare an init container that does the following two things. - -1. Initialize the MySQL cluster, for example, creating necessary users. -2. Create a supplementary config file for `my.cnf`. - -For this purpose, the `moco-agent` binary is provided by [cybozu-go/moco-agent](https://github.com/cybozu-go/moco-agent). - -Behaviors ---------- - -### How to watch the status of instances - -Fetch the following information with mysql-client from each instance: - -- SHOW MASTER STATUS - - Executed_Gtid_Set -- SHOW SLAVE STATUS - - Master_Host - - Executed_Gtid_Set - - Retrieved_Gtid_Set - - Slave_IO_Running - - Slave_SQL_Running - - Last_IO_Errno - - Last_IO_Error - - Last_SQL_Errno - - Last_SQL_Error -- select @@global.read_only, @@global.super_read_only; - - read_only - - super_read_only -- performance_schema.clone_status table - - STATE - -### How to bootstrap MySQL Cluster - -When all instances are the initial state, bootstrap the cluster as follows: - -1. Select the first instance as the primary. -2. Set the replication source using `CHANGE MASTER TO` for a replica instances. -3. Start replication using `START SLAVE` on each replicas. -4. Create Service resources to access to primary and replicas. -5. Update `MySQLCluster.status.currentPrimaryIndex` with the primary name. -6. Turn off read-only mode on the primary. - -### How to execute failover when the primary fails - -When the primary fails, the cluster is recovered in the following process: - -1. Stop the `IO_THREAD` of all replicas. -2. Select and configure new primary. -3. Update `MySQLCluster.status.currentPrimaryIndex` with the new primary name. -4. Turn off read-only mode on the new primary - -In the process, the operator configures the old primary as replica if the server is ready. - -### How to execute failover when a replica fails - -When a replica fails once and it restarts afterwards, the operator basically configures it to follow the primary. - -#### How to handle the case a replica continues to fail and restart - -If one of the replicas fails again after restarting, the `StatefulSet` controller restarts the replica again. -This means that the replica may continue to fail and restart again and again. -This loop can occur, for example, in the case that the data in the replica is corrupted. -Users must handle this failure manually by deleting the `Pod` or/and `PersistentVolumeClaim`. -Then, the replica `Pod` is scheduled again onto a different node and the operator configures it to follow the primary automatically. - -Users can keep the data with a [`VolumeSnapshot`](https://kubernetes.io/docs/concepts/storage/volume-snapshots/) and -create `PersistentVolumeClaim` from the snapshot even after this failure happens. -This feature is available only if the underlying `StorageClass` supports it. - -### How to execute switchover - -Users can execute primary switchover by applying `SwitchoverJob` CR which contains the primary index to be switched to. - -Note that while any `SwitchoverJob` is running, another `SwitchoverJob` can be created but the operator waits for the completion of running jobs. - -### How to make a backup - -When you create `MySQLBackupSchedule` CR, it creates `CronJob` which stores dump and binlog to an object storage: - -If we want to make backups only once, set `MySQLBackupSchedule.spec.schedules` to run once. - -### How to perform PiTR - -When we create a `MySQLCluster` with `.spec.restore` specified, the operator performs PiTR with the following procedure. - -`.spec.restore` is unable to be updated, so PiTR can be executed only when the cluster is being creating. - -1. The operator sets the source cluster's `.status.ready` as `False` and make the MySQL cluster block incoming transactions. -2. The operator makes the MySQL cluster flush binlogs from the source `MySQLCluster`. This binlog is used for recovery if the PiTR fails. -3. The operator lists `MySQLBackup` candidates based on `MySQLCluster.spec.restore.sourceClusterName`. -4. The operator selects the corresponding `MySQLBackup` CRs according to `MySQLCluster.spec.restore.pointInTime`. -5. The operator downloads the dump file and the binlogs for `MySQLCluster.spec.restore.pointInTime` from the object storage. -6. The operator restores the MySQL servers to the state at `MySQLCluster.spec.restore.pointInTime`. -7. If the recovery succeeds, the operator sets the source cluster's `.status.ready` as `True`. - -### How to upgrade MySQL version of primary and replicas - -MySQL software upgrade is triggered by changing container image specified in `MySQLCluster.spec.podTemplate`. -In this section, the name of `StatefulSet` is assumed to be `mysql`. - -1. Switch primary to the pod `mysql-0` if the current primary is not `mysql-0`. -2. Update and apply the `StatefulSet` manifest for `mysql`, to trigger upgrading the replicas as follows: - - Set `.spec.updateStrategy.rollingUpdate.partition` as `1`. - - Set new image version in `.spec.template.spec.containers`. -3. Wait for all the replicas to be upgraded. -4. Switch primary to `mysql-1`. -5. Update and apply the `StatefulSet` manifest to trigger upgrading `mysql-0` as follows: - - Remove `.spec.updateStrategy.rollingUpdate.partition`. -6. Wait for `mysql-0` to be upgraded. - -### How to manage recovery from blackouts - -In the scenario of the recovery from data center blackouts, all members of the MySQL cluster perform cold boots. - -The operator waits for all members to boot up again. -It automatically recovers the cluster only after all members come back, not just after the quorum come back. -This is to prevent the data loss even in corner cases. - -If a part of the cluster never finishes boot-up, users must intervene the recovery process. -The process is as follows. -1. Users delete the problematic Pods and/or PVCs. (or they might have been deleted already) - - Users need to delete PVCs before Pods if they want to delete both due to the [Storage Object in Use Protection](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#storage-object-in-use-protection). - - [The StatefulSet controller may recreate Pods improperly](https://github.com/kubernetes/kubernetes/issues/74374). - Users may need to delete the Nodes in this case. -2. The StatefulSet controller creates new blank Pods and/or PVCs. -3. The operator tries to select the primary of the cluster only after all members are up and running, and the quorum of the cluster has data. - - If the quorum of the cluster does not have data, users need to recover the cluster from a backup. -4. The operator initializes the new blank Pods as new replicas. - -### How to collect metrics for Prometheus - -To avoid unnecessary restart when the operator is upgraded, we prepare external Pods which collect mysqld metrics over the network, -and export them as Prometheus metrics. - -The detail is TBD. - -### How to manage log files - -The operator configures MySQL to output error logs and slow query logs into files. - -To avoid exhaustion of storage resources, the operator appends a sidecar container to the MySQL Pod. -The sidecar container rotates and deletes the log files based on the cron spec in `MySQLCluster.spec.logRotationSchedule`. - -The operator does not care about gathering the contents of the log files. -Tenant users can extract the contents by defining sidecar containers for `slow.log` and `error.log`. - -You can see an example with Fluent Bit in [Example Document](example_mysql_cluster.md). - -### How to delete resources (garbage collection) - -The operator sets an owner reference of MySQLCluster to the child resources. -By the owner reference, when the parent MySQLCluster is deleted, child resources are automatically deleted. - -The data volumes (PVCs) are also deleted automatically. -If you want to prevent the PVCs deletion, please edit the PVCs manifest and remove the owner reference manually. - -### TBD - -- Write merge strategy of `my.cnf`. -- How to clean up zombie backup files on a object storage which does not exist in Kubernetes. - -### Candidates of additional features - -- Backup files verification. diff --git a/docs/images/overview.png b/docs/images/overview.png deleted file mode 100644 index 799b1ff06..000000000 Binary files a/docs/images/overview.png and /dev/null differ diff --git a/links.csv b/docs/links.csv similarity index 100% rename from links.csv rename to docs/links.csv diff --git a/docs/metrics.md b/docs/metrics.md index 53ba93aea..982d48df4 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -24,6 +24,12 @@ All these metrics are prefixed with `moco_cluster_` and have `name` and `namespa TBD +## MySQL instance + +For each `mysqld` instance, [moco-agent][] exposes a set of metrics. +Read [github.com/cybozu-go/moco-agent/blob/main/docs/metrics.md](https://github.com/cybozu-go/moco-agent/blob/main/docs/metrics.md) for details. + [standard]: https://povilasv.me/prometheus-go-metrics/ [controller-runtime]: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics [errant]: https://www.percona.com/blog/2014/05/19/errant-transactions-major-hurdle-for-gtid-based-failover-in-mysql-5-6/ +[moco-agent]: https://github.com/cybozu-go/moco-agent/ diff --git a/docs/setup.md b/docs/setup.md index 1bc8443ff..9141d1325 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -1,43 +1,28 @@ Setup ===== -## Prerequisites +## Quick setup -* Installation components for this operator is in [config](../config) directory. +1. Download and install the latest [cert-manager](https://cert-manager.io/). -* The operator manifests are managed with [kustomize](https://kustomize.io/), so it is necessary to install it first. + ```console + $ curl -fsLO https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml + $ kubectl apply -f cert-manager.yaml + ``` -## Setup procesure +2. Download and install the latest MOCO. -### Quick setup + ```console + $ curl -fsLO https://github.com/cybozu-go/moco/releases/latest/download/moco.yaml + $ kubectl apply -f moco.yaml + ``` -You can point at [config](../config) to build kustomized components, like so +That's all! -```shell -# Assumption: you're in the project root -$ cd config -$ kubectl apply -k . -or -$ kustomize build | kubectl apply -f - -``` +## Customize manifests -### Setup components in order +If you want to edit the manifest, [`config/`](../config/) directory contains the source YAML for [kustomize](https://kustomize.io/). -If you want to see what to be installed, you can also install components in order. +## Next step -```shell -# Assumption: you're in the project root -$ cd config -# Create namespace -$ kubectl apply -f namespace.yaml -# Create CRD -$ kubectl apply -k crd -# Create RBAC -$ kubectl apply -k rbac -# Create manager deployment -$ kubectl apply -k manager -``` - -### Next step - -If you want to create a MySQL Cluster, read [Example of MySQLCluster Custom Resource](./example_mysql_cluster.md) and [MySQLCluster](./crd_mysql_cluster.md). +Read [`usage.md`](usage.md) and create your first MySQL cluster! diff --git a/docs/usage.md b/docs/usage.md index ed887b0ca..fefed707d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -9,11 +9,12 @@ After [setting up MOCO](setup.md), you can create MySQL clusters with a custom r - [Creating clusters](#creating-clusters) - [Creating an empty cluster](#creating-an-empty-cluster) - [Creating a cluster that replicates data from an external mysqld](#creating-a-cluster-that-replicates-data-from-an-external-mysqld) + - [Bring your own image](#bring-your-own-image) - [Configurations](#configurations) - [Using the cluster](#using-the-cluster) - [`kubectl moco`](#kubectl-moco) - - [Connecting to the primary instance](#connecting-to-the-primary-instance) - - [Connecting to read-only replicas](#connecting-to-read-only-replicas) + - [MySQL users](#mysql-users) + - [Connecting to `mysqld` over network](#connecting-to-mysqld-over-network) - [Deleting the cluster](#deleting-the-cluster) - [Status, metrics, and logs](#status-metrics-and-logs) - [Cluster status](#cluster-status) @@ -40,15 +41,74 @@ In a cluster, there is only one instance called _primary_. The primary instance ### Errant replicas +An inherent limitation of GTID-based semi-synchronous replication is that a failed instance would have [errant transactions](https://www.percona.com/blog/2014/05/19/errant-transactions-major-hurdle-for-gtid-based-failover-in-mysql-5-6/). If this happens, the instance needs to be re-created by removing all data. + +MOCO does not re-create such an instance. It only detects instances having errant transactions and excludes them from the cluster. Users need to monitor them and re-create the instances. + ### Read-only primary -MOCO may set the primary mysqld instance read-only for a switchover or other reasons. +MOCO from time to time sets the primary mysqld instance read-only for a switchover or other reasons. Applications that use MOCO MySQL need to be aware of this. ## Creating clusters ### Creating an empty cluster +An empty cluster always has a writable instance called _the primary_. All other instances are called _replicas_. Replicas are read-only and replicate data from the primary. + +The following YAML is to create a three-instance cluster. It has an anti-affinity for Pods so that all instances will be scheduled to different Nodes. It also has the same values for memory and CPU requests and limits making the Pod to have [Guaranteed](https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/) QoS. + +```yaml +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + # replicas is the number of mysqld Pods. The default is 1. + replicas: 3 + podTemplate: + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - moco + - key: app.kubernetes.io/instance + operator: In + values: + - test + topologyKey: "kubernetes.io/hostname" + containers: + # At least a container named "mysqld" must be defined. + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + resources: + requests: + cpu: "10" + memory: "10Gi" + limits: + cpu: "10" + memory: "10Gi" + volumeClaimTemplates: + # At least a PVC named "mysql-data" must be defined. + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi +``` + +There are other example manifests in [`examples`](../examples/) directory. + +The complete reference of MySQLCluster is [`crd_mysqlcluster.md`](crd_mysqlcluster.md). + ### Creating a cluster that replicates data from an external mysqld Let's call the source mysqld instance _donor_. @@ -66,7 +126,7 @@ mysql> CREATE USER 'clone-init'@'localhost' IDENTIFIED BY 'yyyyyyyyyyy'; mysql> GRANT ALL ON *.* TO 'clone-init'@'localhost' WITH GRANT OPTION; ``` -You may change the user names and should change the passwords. +You may change the user names and should change their passwords. Then create a Secret in the same namespace as MySQLCluster: @@ -98,9 +158,22 @@ spec: containers: - name: mysqld image: quay.io/cybozu/moco-mysql:8.0.24 # must be the same version as the donor + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi ``` -You can stop the replication from the donor by setting `spec.replicationSourceSecretName` to `null` afterwards. +To stop the replication from the donor, update MySQLCluster with `spec.replicationSourceSecretName: null`. + +### Bring your own image + +We provide a pre-built MySQL container image at [quay.io/cybozu/moco-mysql](http://quay.io/cybozu/moco-mysql). +If you want to build and use your own image, read [`custom-mysqld.md`](custom-mysqld.md). ### Configurations @@ -115,8 +188,8 @@ metadata: namespace: foo name: mycnf data: - long_query_time: "10" - innodb_buffer_pool_size: 10G + long_query_time: "5" + innodb_buffer_pool_size: "10G" ``` and set the name of the ConfigMap in MySQLCluster as follows: @@ -138,9 +211,103 @@ If `innodb_buffer_pool_size` is not given, MOCO sets it automatically to 70% of ### `kubectl moco` -### Connecting to the primary instance +From outside of your Kubernetes cluster, you can access MOCO MySQL instances using `kubectl-moco`. +`kubectl-moco` is [a plugin for `kubectl`](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/). +Pre-built binaries are available on [GitHub releases](https://github.com/cybozu-go/moco/releases/latest). + +The following is an example to run `mysql` command interactively to access the primary instance of `test` MySQLCluster in `foo` namespace. -### Connecting to read-only replicas +```console +$ kubectl moco -n foo mysql -it test +``` + +Read [the reference manual of `kubectl-moco`](kubectl-moco.md) for further details and examples. + +### MySQL users + +MOCO prepares a set of users. + +- `moco-readonly` can read all tables of all databases. +- `moco-writable` can create users, databases, or tables. +- `moco-admin` is the super user. + +The exact privileges that `moco-readonly` has are: + +- PROCESS +- REPLICATION CLIENT +- REPLICATION SLAVE +- SELECT +- SHOW DATABASES +- SHOW VIEW + +The exact privileges that `moco-writable` has are: + +- ALTER +- ALTER ROUTINE +- CREATE +- CREATE ROLE +- CREATE ROUTINE +- CREATE TEMPORARY TABLES +- CREATE USER +- CREATE VIEW +- DELETE +- DROP +- DROP ROLE +- EVENT +- EXECUTE +- INDEX +- INSERT +- LOCK TABLES +- PROCESS +- REFERENCES +- REPLICATION CLIENT +- REPLICATION SLAVE +- SELECT +- SHOW DATABASES +- SHOW VIEW +- TRIGGER +- UPDATE + +`moco-writable` cannot edit tables in `mysql` database, though. + +You can create other users and grant them certain privileges as either `moco-writable` or `moco-admin`. + +```console +$ kubectl moco mysql -u moco-writable test -- -e "CREATE USER 'foo'@'%' IDENTIFIED BY 'bar'" +$ kubectl moco mysql -u moco-writable test -- -e "CREATE DATABASE db1" +$ kubectl moco mysql -u moco-writable test -- -e "GRANT ALL ON db1.* TO 'foo'@'%'" +``` + +### Connecting to `mysqld` over network + +MOCO prepares two Services for each MySQLCluster. +For example, a MySQLCluster named `test` in `foo` Namespace has the following Services. + +| Service Name | DNS Name | Description | +| ------------------- | --------------------------- | -------------------------------- | +| `moco-test-primary` | `moco-test-primary.foo.svc` | Connect to the primary instance. | +| `moco-test-replica` | `moco-test-replica.foo.svc` | Connect to replica instances. | + +`moco-test-replica` can be used only for read access. + +The type of these Services is usually ClusterIP. +The following is an example to change Service type to LoadBalancer and add an annotation for [MetalLB][]. + +```yaml +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: foo + name: test +spec: + serviceTemplate: + metadata: + annotations: + metallb.universe.tf/address-pool: production-public-ips + spec: + type: LoadBalancer +... +``` ## Deleting the cluster @@ -152,20 +319,80 @@ If you want to keep the PersistentVolumeClaims, remove `metadata.ownerReferences ### Cluster status +You can see the health and availability status of MySQLCluster as follows: + +```console +$ kubectl get mysqlcluster +NAME AVAILABLE HEALTHY PRIMARY SYNCED REPLICAS ERRANT REPLICAS +test True True 0 3 +``` + +- The cluster is available when the primary Pod is running and ready. +- The cluster is healthy when there is no problems. +- `PRIMARY` is the index of the current primary instance Pod. +- `SYNCED REPLICAS` is the number of ready Pods. +- `ERRANT REPLICAS` is the number of instances having errant transactions. + +You can also use `kubectl describe mysqlcluster` to see the recent events on the cluster. + ### Pod status MOCO adds mysqld containers a liveness probe and a readiness probe to check the replication status in addition to the process status. A replica Pod is _ready_ only when it is replicating data from the primary without a significant delay. +The default threshold of the delay is 60 seconds. The threshold can be configured as follows. + +```yaml +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: foo + name: test +spec: + maxDelaySeconds: 180 + ... +``` + +Unready replica Pods are automatically excluded from the load-balancing targets so that users will not see too old data. ### Metrics +See [`metrics.md`](metrics.md) for available metrics in Prometheus format. + ### Logs +Error logs from `mysqld` can be viewed as follows: + +```console +$ kubectl logs moco-test-0 mysqld +``` + +Slow logs from `mysqld` can be viewed as follows: + +```console +$ kubectl logs moco-test-0 slow-log +``` + ## Maintenance ### Increasing the number of instances in the cluster +Edit `spec.replicas` field of MySQLCluster: + +```yaml +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: foo + name: test +spec: + replicas: 5 + ... +``` + +You can only increase the number of instances in a MySQLCluster from 1 to 3 or 5, or from 3 to 5. +Decreasing the number of instances is not allowed. + ### Switchover Switchver is an operation to change the live primary to one of the replicas. @@ -175,15 +402,36 @@ MOCO automatically switch the primary when the Pod of the primary instance is to Users can manually trigger a switchover by annotating the Pod of the primary instance with `moco.cybozu.com/demote: true`. You can use `kubectl` to do this: ```console -$ kubectl annotate mysqlclusters moco.cybozu.com/demote=true +$ kubectl annotate mysqlcluster moco.cybozu.com/demote=true ``` ### Failover +Failover is an operation to replace the dead primary with the most advanced replica. +MOCO automatically does this as soon as it detects that the primary is down. + +The most advanced replica is a replica who has retrieved the most up-to-date transaction from the dead primary. +Since MOCO configures loss-less semi-synchronous replication, the failover is guaranteed not to lose any user data. + +After a failover, the old primary may become an errant replica [as described](#errant-replicas). + ### Upgrading mysql version +TBD + ### Re-initializing an errant replica +Delete the PVC and Pod of the errant replica, like this: + +```console +$ kubectl delete --wait=false pvc mysql-data-moco-test-0 +$ kubectl delete --grace-period=1 pods moco-test-0 +``` + +Depending on your Kubernetes version, StatefulSet controller may create a pending Pod before PVC gets deleted. +Delete such pending Pods until PVC is actually removed. + [semisync]: https://dev.mysql.com/doc/refman/8.0/en/replication-semisync.html [GTID]: https://dev.mysql.com/doc/refman/8.0/en/replication-gtids.html [CLONE]: https://dev.mysql.com/doc/refman/8.0/en/clone-plugin.html +[MetalLB]: https://metallb.universe.tf/ diff --git a/e2e/lifecycle_test.go b/e2e/lifecycle_test.go index b1f8baa67..f54c0f092 100644 --- a/e2e/lifecycle_test.go +++ b/e2e/lifecycle_test.go @@ -57,12 +57,16 @@ var _ = Context("lifecycle", func() { }) It("should update the configmap and restart mysqld", func() { - cluster, err := getCluster("foo", "single") - Expect(err).NotTo(HaveOccurred()) - - cluster.Spec.MySQLConfigMapName = nil - data, _ := json.Marshal(cluster) - kubectlSafe(data, "apply", "-f", "-") + Eventually(func() error { + cluster, err := getCluster("foo", "single") + if err != nil { + return err + } + cluster.Spec.MySQLConfigMapName = nil + data, _ := json.Marshal(cluster) + _, err = kubectl(data, "apply", "-f", "-") + return err + }).Should(Succeed()) Eventually(func() float64 { out, err := kubectl(nil, "moco", "-n", "foo", "mysql", "single", "--", "-N", "-e", "SELECT @@long_query_time") @@ -154,10 +158,15 @@ var _ = Context("lifecycle", func() { if err != nil { return err } - if len(cms.Items) == 1 { - return nil + + for _, cm := range cms.Items { + switch cm.Name { + case "kube-root-ca.crt", "mycnf": + default: + return fmt.Errorf("pending config map %+v", cm) + } } - return fmt.Errorf("pending config maps: %+v", cms.Items) + return nil }).Should(Succeed()) Eventually(func() error { secrets := &corev1.SecretList{} diff --git a/e2e/replication_test.go b/e2e/replication_test.go index 2d516b790..020b9aaae 100644 --- a/e2e/replication_test.go +++ b/e2e/replication_test.go @@ -138,14 +138,16 @@ var _ = Context("replication", func() { }) It("should be able to scale out the cluster", func() { - out := kubectlSafe(nil, "-n", "repl", "get", "mysqlcluster", "test", "-o", "json") - cluster := &mocov1beta1.MySQLCluster{} - err := json.Unmarshal(out, cluster) - Expect(err).NotTo(HaveOccurred()) - - cluster.Spec.Replicas = 5 - data, _ := json.Marshal(cluster) - kubectlSafe(data, "-n", "repl", "apply", "-f", "-") + Eventually(func() error { + cluster, err := getCluster("repl", "test") + if err != nil { + return err + } + cluster.Spec.Replicas = 5 + data, _ := json.Marshal(cluster) + _, err = kubectl(data, "apply", "-f", "-") + return err + }).Should(Succeed()) Eventually(func() error { cluster, err := getCluster("repl", "test") @@ -169,10 +171,16 @@ var _ = Context("replication", func() { }) It("should detect errant transactions", func() { - kubectlSafe(nil, "moco", "-n", "repl", "mysql", "-u", "moco-admin", "--index", "0", "test", "--", - "-e", "SET GLOBAL read_only=0") - kubectlSafe(nil, "moco", "-n", "repl", "mysql", "-u", "moco-admin", "--index", "0", "test", "--", - "-e", "CREATE DATABASE errant") + Eventually(func() error { + _, err := kubectl(nil, "moco", "-n", "repl", "mysql", "-u", "moco-admin", "--index", "0", "test", "--", + "-e", "SET GLOBAL read_only=0") + if err != nil { + return err + } + _, err = kubectl(nil, "moco", "-n", "repl", "mysql", "-u", "moco-admin", "--index", "0", "test", "--", + "-e", "CREATE DATABASE errant") + return err + }).Should(Succeed()) Eventually(func() int { cluster, err := getCluster("repl", "test") @@ -296,12 +304,16 @@ var _ = Context("replication", func() { }) It("should be able to stop replication from the donor", func() { - cluster, err := getCluster("repl", "test") - Expect(err).NotTo(HaveOccurred()) - - cluster.Spec.ReplicationSourceSecretName = nil - data, _ := json.Marshal(cluster) - kubectlSafe(data, "apply", "-f", "-") + Eventually(func() error { + cluster, err := getCluster("repl", "test") + if err != nil { + return err + } + cluster.Spec.ReplicationSourceSecretName = nil + data, _ := json.Marshal(cluster) + _, err = kubectl(data, "apply", "-f", "-") + return err + }).Should(Succeed()) Eventually(func() error { _, err := kubectl(nil, "moco", "-n", "repl", "mysql", "-u", "moco-writable", "test", "--", diff --git a/examples/anti-affinity.yaml b/examples/anti-affinity.yaml new file mode 100644 index 000000000..31b93b1a2 --- /dev/null +++ b/examples/anti-affinity.yaml @@ -0,0 +1,44 @@ +# This example shows how to schedule Pods on different Nodes. +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + replicas: 3 + podTemplate: + spec: + affinity: + # The anti-affinity for Pods + # cf. https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - moco + - key: app.kubernetes.io/instance + operator: In + values: + - test + topologyKey: "kubernetes.io/hostname" + containers: + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + resources: + requests: + cpu: "10" + memory: "10Gi" + limits: + cpu: "10" + memory: "10Gi" + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi diff --git a/examples/custom-mycnf.yaml b/examples/custom-mycnf.yaml new file mode 100644 index 000000000..bd6f311a7 --- /dev/null +++ b/examples/custom-mycnf.yaml @@ -0,0 +1,35 @@ +# This example shows how to set MySQL server system variables. +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: default + name: mycnf +data: + # key-value in data field will become server system variable names and values. + # https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html + # https://dev.mysql.com/doc/refman/8.0/en/innodb-parameters.html + long_query_time: "5" + innodb_buffer_pool_size: "70G" +--- +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + replicas: 3 + # ConfigMap name in the same namespace. + mysqlConfigMapName: mycnf + podTemplate: + spec: + containers: + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi diff --git a/examples/guaranteed.yaml b/examples/guaranteed.yaml new file mode 100644 index 000000000..7d2f8568e --- /dev/null +++ b/examples/guaranteed.yaml @@ -0,0 +1,30 @@ +# This example shows how to assign Guaranteed QoS class to Pods. +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + replicas: 3 + podTemplate: + spec: + containers: + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + # By allocating the same CPU and memory for both requests and limits, + # the Pod gets assigned Guaranteed QoS class. + resources: + requests: + cpu: "10" + memory: "10Gi" + limits: + cpu: "10" + memory: "10Gi" + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi diff --git a/examples/loadbalancer.yaml b/examples/loadbalancer.yaml new file mode 100644 index 000000000..7a1299160 --- /dev/null +++ b/examples/loadbalancer.yaml @@ -0,0 +1,29 @@ +# This example shows how to change Service type to LoadBalancer +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + replicas: 3 + # serviceTemplate allows you to specify annotations, labels, and spec + # of Services to be generated for this MySQLCluster. + serviceTemplate: + metadata: + annotations: + metallb.universe.tf/address-pool: production-public-ips + spec: + type: LoadBalancer + podTemplate: + spec: + containers: + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi diff --git a/go.mod b/go.mod index 42719945b..39af7b512 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cybozu-go/moco go 1.16 require ( - github.com/cybozu-go/moco-agent v0.6.0 + github.com/cybozu-go/moco-agent v0.6.1 github.com/go-logr/logr v0.4.0 github.com/go-logr/stdr v0.4.0 github.com/go-sql-driver/mysql v1.6.0 diff --git a/go.sum b/go.sum index 052967e02..0f2dbee41 100644 --- a/go.sum +++ b/go.sum @@ -106,8 +106,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cybozu-go/log v1.5.0/go.mod h1:zpfovuCgUx+a/ErvQrThoT+/z1RVQoLDOf95wkBeRiw= github.com/cybozu-go/log v1.6.0/go.mod h1:2iAEvn2cL5dy/1uP5Jfb0Ao9+DUnDr//V0Bk3WDJX1U= -github.com/cybozu-go/moco-agent v0.6.0 h1:3anuGMTMUh5BbBp7yRvGpWhLi+kKmL7O2zke0lfNnWk= -github.com/cybozu-go/moco-agent v0.6.0/go.mod h1:emhcINL6P81LZ4o355DETRXs8S63HUAXJN7SK01e7MY= +github.com/cybozu-go/moco-agent v0.6.1 h1:Cn9//cBSsZoXNzTfeoaNyyWVKmp+m1LJdV2hNvKdGrA= +github.com/cybozu-go/moco-agent v0.6.1/go.mod h1:emhcINL6P81LZ4o355DETRXs8S63HUAXJN7SK01e7MY= github.com/cybozu-go/netutil v1.2.0/go.mod h1:Wx92iF1dPrtuSzLUMEidtrKTFiDWpLcsYvbQ1lHSmxY= github.com/cybozu-go/well v1.10.0/go.mod h1:OQdjEXQpbG+kSgEF3t3IYUx5y1R4qeBGvzL4gmi61qE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/kustomization.yaml b/kustomization.yaml index 2ee3bdb33..90c9acc2d 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -3,4 +3,4 @@ resources: images: - name: ghcr.io/cybozu-go/moco - newTag: 0.7.0 + newTag: 0.8.0 diff --git a/pkg/mycnf/generator.go b/pkg/mycnf/generator.go index e28d90ecc..4c22400de 100644 --- a/pkg/mycnf/generator.go +++ b/pkg/mycnf/generator.go @@ -88,6 +88,7 @@ var DefaultMycnf = map[string]string{ "innodb_flush_neighbors": "0", "innodb_random_read_ahead": "false", "innodb_read_ahead_threshold": "0", + "innodb_log_write_ahead_size": "512", } // ConstMycnf is the mysqld configurations that MOCO applies forcibly. diff --git a/pkg/mycnf/testdata/bufsize.cnf b/pkg/mycnf/testdata/bufsize.cnf index b6c361fc5..84caa3992 100644 --- a/pkg/mycnf/testdata/bufsize.cnf +++ b/pkg/mycnf/testdata/bufsize.cnf @@ -31,6 +31,7 @@ innodb_flush_neighbors = 0 innodb_lock_wait_timeout = 60 innodb_log_file_size = 800M innodb_log_files_in_group = 2 +innodb_log_write_ahead_size = 512 innodb_online_alter_log_max_size = 1073741824 innodb_print_all_deadlocks = 1 innodb_random_read_ahead = false diff --git a/pkg/mycnf/testdata/loose.cnf b/pkg/mycnf/testdata/loose.cnf index 7b03cd428..4b95f0552 100644 --- a/pkg/mycnf/testdata/loose.cnf +++ b/pkg/mycnf/testdata/loose.cnf @@ -31,6 +31,7 @@ innodb_flush_neighbors = 0 innodb_lock_wait_timeout = 60 innodb_log_file_size = 800M innodb_log_files_in_group = 2 +innodb_log_write_ahead_size = 512 innodb_numa_interleave = OFF innodb_online_alter_log_max_size = 1073741824 innodb_print_all_deadlocks = 1 diff --git a/pkg/mycnf/testdata/nil.cnf b/pkg/mycnf/testdata/nil.cnf index 93917b5bd..6b582a81e 100644 --- a/pkg/mycnf/testdata/nil.cnf +++ b/pkg/mycnf/testdata/nil.cnf @@ -31,6 +31,7 @@ innodb_flush_neighbors = 0 innodb_lock_wait_timeout = 60 innodb_log_file_size = 800M innodb_log_files_in_group = 2 +innodb_log_write_ahead_size = 512 innodb_online_alter_log_max_size = 1073741824 innodb_print_all_deadlocks = 1 innodb_random_read_ahead = false diff --git a/pkg/mycnf/testdata/normalize.cnf b/pkg/mycnf/testdata/normalize.cnf index 5c7027687..2fdc4662d 100644 --- a/pkg/mycnf/testdata/normalize.cnf +++ b/pkg/mycnf/testdata/normalize.cnf @@ -32,6 +32,7 @@ innodb_flush_neighbors = 0 innodb_lock_wait_timeout = 60 innodb_log_file_size = 800M innodb_log_files_in_group = 2 +innodb_log_write_ahead_size = 512 innodb_online_alter_log_max_size = 1073741824 innodb_print_all_deadlocks = 1 innodb_random_read_ahead = false diff --git a/version.go b/version.go index 274ba95ab..7057d7d82 100644 --- a/version.go +++ b/version.go @@ -5,5 +5,5 @@ const ( Version = "0.8.0" // FluentBitImage is the image for slow-log sidecar container. - FluentBitImage = "quay.io/cybozu/fluent-bit:1.7.2.2" + FluentBitImage = "quay.io/cybozu/fluent-bit:1.7.4.1" )