Implement features for upgrading MySQL version

Read docs/upgrading.md for details.

Added features are:

- Choose the lowest ordinal instance for switchover
- `spec.startupDelaySeconds` field to MySQLCluster

.github/workflows/ci.yaml

@@ -61,3 +61,28 @@ jobs:
  with:
  name: logs.tar.gz
  path: e2e/logs.tar.gz
+ upgrade:
+ name: Upgrade Test
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-go@v2
+ with:
+ go-version: ${{ env.go-version }}
+ - run: |
+ swapon > swapon.txt
+ sudo swapoff -a
+ cat swapon.txt | tail -n+2 | awk '$2=="file" {print $1}' | sudo xargs --no-run-if-empty rm
+ - run: sudo mkdir /mnt/local-path-provisioner0 /mnt/local-path-provisioner1 /mnt/local-path-provisioner2
+ - run: make start KIND_CONFIG=kind-config_actions.yaml
+ working-directory: e2e
+ - run: make test-upgrade
+ working-directory: e2e
+ - run: make logs
+ working-directory: e2e
+ if: always()
+ - uses: actions/upload-artifact@v2
+ if: always()
+ with:
+ name: logs.tar.gz
+ path: e2e/logs.tar.gz

README.md

@@ -37,7 +37,7 @@ Other versions may work, though not tested.
 - Different MySQL versions for each cluster
 - Upgrading MySQL version of a cluster
 - Monitor for replication delays
-- Service for the primary and replicas, respectively
+- Services for the primary and replicas, respectively
 - Custom `my.cnf` configurations
 - Custom Pod, Service, and PersistentVolumeClaim templates
 - Redirect slow query logs to a sidecar container

api/v1beta1/mysqlcluster_types.go

@@ -60,6 +60,13 @@ type MySQLClusterSpec struct {
  // +optional
  MaxDelaySeconds int `json:"maxDelaySeconds,omitempty"`
 
+ // StartupWaitSeconds is the maximum duration to wait for `mysqld` container to start working.
+ // The default is 3600 seconds.
+ // +kubebuilder:validation:Minimum=0
+ // +kubebuilder:default=3600
+ // +optional
+ StartupWaitSeconds int32 `json:"startupDelaySeconds,omitempty"`
+
  // LogRotationSchedule specifies the schedule to rotate MySQL logs.
  // If not set, the default is to rotate logs every 5 minutes.
  // See https://pkg.go.dev/github.com/robfig/cron?#hdr-CRON_Expression_Format for the field format.

clustering/operations.go

@@ -310,7 +310,7 @@ func (p *managerProcess) configure(ctx context.Context, ss *StatusSet) (bool, er
  if ss.Cluster.Spec.ReplicationSourceSecretName == nil {
  pst := ss.MySQLStatus[ss.Primary]
  op := ss.DBOps[ss.Primary]
- if pst.GlobalVariables.ReadOnly || pst.ReplicaStatus != nil {
+ if pst.GlobalVariables.ReadOnly {
  redo = true
  p.log.Info("set read_only=0", "instance", ss.Primary)
  if err := op.SetReadOnly(ctx, false); err != nil {

clustering/status.go

@@ -4,6 +4,7 @@ import (
  "context"
  "fmt"
  "os"
+ "sort"
  "strconv"
  "strings"
  "sync"
@@ -75,16 +76,17 @@ func (s ClusterState) String() string {
 // and later operations.
 type StatusSet struct {
  Primary int
- Candidate int
  Cluster *mocov1beta1.MySQLCluster
  Password *password.MySQLPassword
  Pods []*corev1.Pod
  DBOps []dbop.Operator
  MySQLStatus []*dbop.MySQLInstanceStatus
  ExecutedGTID string
  Errants []int
+ Candidates []int
 
  NeedSwitch bool
+ Candidate int
  State ClusterState
 }
 
@@ -100,7 +102,6 @@ func (ss *StatusSet) Close() {
 // DecideState decides the ClusterState and set it to `ss.State`.
 // It may also set `ss.NeedSwitch` and `ss.Candidate` for switchover.
 func (ss *StatusSet) DecideState() {
- ss.NeedSwitch = needSwitch(ss.Pods[ss.Primary])
  switch {
  case isHealthy(ss):
  ss.State = StateHealthy
@@ -115,6 +116,12 @@ func (ss *StatusSet) DecideState() {
  default:
  ss.State = StateIncomplete
  }
+ if len(ss.Candidates) > 0 {
+ ss.NeedSwitch = needSwitch(ss.Pods[ss.Primary])
+ // Choose the lowest ordinal for a switchover target.
+ sort.Ints(ss.Candidates)
+ ss.Candidate = ss.Candidates[0]
+ }
 }
 
 // GatherStatus collects information and Kubernetes resources and construct
@@ -309,7 +316,7 @@ func isHealthy(ss *StatusSet) bool {
  if ist.ReplicaStatus.MasterHost != primaryHostname {
  return false
  }
- ss.Candidate = i
+ ss.Candidates = append(ss.Candidates, i)
  }
 
  pst := ss.MySQLStatus[ss.Primary]
@@ -403,7 +410,7 @@ func isDegraded(ss *StatusSet) bool {
  continue
  }
  okReplicas++
- ss.Candidate = i
+ ss.Candidates = append(ss.Candidates, i)
  }
 
  return okReplicas >= (int(ss.Cluster.Spec.Replicas)/2) && okReplicas != int(ss.Cluster.Spec.Replicas-1)

config/crd/bases/moco.cybozu.com_mysqlclusters.yaml

@@ -3974,6 +3974,12 @@ spec:
  type: string
  type: object
  type: object
+ startupDelaySeconds:
+ default: 3600
+ description: StartupWaitSeconds is the maximum duration to wait for `mysqld` container to start working. The default is 3600 seconds.
+ format: int32
+ minimum: 0
+ type: integer
  volumeClaimTemplates:
  description: VolumeClaimTemplates is a list of `PersistentVolumeClaim` templates for MySQL server container. A claim named "mysql-data" must be included in the list.
  items:

controllers/mysql_container.go

@@ -11,7 +11,7 @@ import (
  "k8s.io/apimachinery/pkg/util/intstr"
 )
 
-func (r *MySQLClusterReconciler) makeV1MySQLDContainer(desired, current []corev1.Container) (corev1.Container, error) {
+func (r *MySQLClusterReconciler) makeV1MySQLDContainer(cluster *mocov1beta1.MySQLCluster, desired, current []corev1.Container) (corev1.Container, error) {
  var source *corev1.Container
  for i, c := range desired {
  if c.Name == constants.MysqldContainerName {
@@ -36,6 +36,10 @@ func (r *MySQLClusterReconciler) makeV1MySQLDContainer(desired, current []corev1
  corev1.ContainerPort{ContainerPort: constants.MySQLAdminPort, Name: constants.MySQLAdminPortName, Protocol: corev1.ProtocolTCP},
  corev1.ContainerPort{ContainerPort: constants.MySQLHealthPort, Name: constants.MySQLHealthPortName, Protocol: corev1.ProtocolTCP},
  )
+ failureThreshold := cluster.Spec.StartupWaitSeconds / 10
+ if failureThreshold < 1 {
+ failureThreshold = 1
+ }
  c.StartupProbe = &corev1.Probe{
  Handler: corev1.Handler{
  HTTPGet: &corev1.HTTPGetAction{
@@ -45,7 +49,7 @@ func (r *MySQLClusterReconciler) makeV1MySQLDContainer(desired, current []corev1
  },
  },
  PeriodSeconds: 10,
- FailureThreshold: 360, // tolerate up to 1 hour of startup time
+ FailureThreshold: failureThreshold,
  }
  c.LivenessProbe = &corev1.Probe{
  Handler: corev1.Handler{

controllers/mysqlcluster_controller.go

@@ -558,6 +558,9 @@ func (r *MySQLClusterReconciler) reconcileV1StatefulSet(ctx context.Context, req
  MatchLabels: labelSet(cluster, false),
  }
  sts.Spec.PodManagementPolicy = appsv1.ParallelPodManagement
+ sts.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{
+ Type: appsv1.RollingUpdateStatefulSetStrategyType,
+ }
  sts.Spec.ServiceName = cluster.HeadlessServiceName()
 
  sts.Spec.VolumeClaimTemplates = make([]corev1.PersistentVolumeClaim, len(cluster.Spec.VolumeClaimTemplates))
@@ -648,7 +651,7 @@ func (r *MySQLClusterReconciler) reconcileV1StatefulSet(ctx context.Context, req
  }
 
  containers := make([]corev1.Container, 0, 4)
- mysqldContainer, err := r.makeV1MySQLDContainer(podSpec.Containers, sts.Spec.Template.Spec.Containers)
+ mysqldContainer, err := r.makeV1MySQLDContainer(cluster, podSpec.Containers, sts.Spec.Template.Spec.Containers)
  if err != nil {
  return err
  }

controllers/mysqlcluster_controller_test.go

@@ -455,17 +455,20 @@ var _ = Describe("MySQLCluster reconciler", func() {
 
  Expect(headless.Spec.PublishNotReadyAddresses).To(BeTrue())
 
- cluster = &mocov1beta1.MySQLCluster{}
- err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster)
- Expect(err).NotTo(HaveOccurred())
- cluster.Spec.ServiceTemplate = &mocov1beta1.ServiceTemplate{
- ObjectMeta: mocov1beta1.ObjectMeta{
- Annotations: map[string]string{"foo": "bar"},
- Labels: map[string]string{"foo": "baz"},
- },
- }
- err = k8sClient.Update(ctx, cluster)
- Expect(err).NotTo(HaveOccurred())
+ Eventually(func() error {
+ cluster = &mocov1beta1.MySQLCluster{}
+ err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster)
+ if err != nil {
+ return err
+ }
+ cluster.Spec.ServiceTemplate = &mocov1beta1.ServiceTemplate{
+ ObjectMeta: mocov1beta1.ObjectMeta{
+ Annotations: map[string]string{"foo": "bar"},
+ Labels: map[string]string{"foo": "baz"},
+ },
+ }
+ return k8sClient.Update(ctx, cluster)
+ }).Should(Succeed())
 
  Eventually(func() error {
  headless = &corev1.Service{}
@@ -486,16 +489,19 @@ var _ = Describe("MySQLCluster reconciler", func() {
  Expect(err).NotTo(HaveOccurred())
  Expect(newPrimary.Spec.ClusterIP).To(Equal(primary.Spec.ClusterIP))
 
- cluster = &mocov1beta1.MySQLCluster{}
- err = k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster)
- Expect(err).NotTo(HaveOccurred())
- cluster.Spec.ServiceTemplate = &mocov1beta1.ServiceTemplate{
- Spec: &corev1.ServiceSpec{
- Type: corev1.ServiceTypeLoadBalancer,
- },
- }
- err = k8sClient.Update(ctx, cluster)
- Expect(err).NotTo(HaveOccurred())
+ Eventually(func() error {
+ cluster = &mocov1beta1.MySQLCluster{}
+ err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: "test"}, cluster)
+ if err != nil {
+ return err
+ }
+ cluster.Spec.ServiceTemplate = &mocov1beta1.ServiceTemplate{
+ Spec: &corev1.ServiceSpec{
+ Type: corev1.ServiceTypeLoadBalancer,
+ },
+ }
+ return k8sClient.Update(ctx, cluster)
+ }).Should(Succeed())
 
  Eventually(func() error {
  primary = &corev1.Service{}
@@ -541,6 +547,8 @@ var _ = Describe("MySQLCluster reconciler", func() {
  case constants.MysqldContainerName:
  foundMysqld = true
  Expect(c.Image).To(Equal("moco-mysql:latest"))
+ Expect(c.StartupProbe).NotTo(BeNil())
+ Expect(c.StartupProbe.FailureThreshold).To(Equal(int32(360)))
  case constants.AgentContainerName:
  foundAgent = true
  Expect(c.Image).To(Equal(testAgentImage))
@@ -620,6 +628,7 @@ var _ = Describe("MySQLCluster reconciler", func() {
  cluster.Spec.Replicas = 5
  cluster.Spec.ReplicationSourceSecretName = nil
  cluster.Spec.MaxDelaySeconds = 20
+ cluster.Spec.StartupWaitSeconds = 3
  cluster.Spec.LogRotationSchedule = "0 * * * *"
  cluster.Spec.DisableSlowQueryLogContainer = true
  cluster.Spec.PodTemplate.Spec.TerminationGracePeriodSeconds = pointer.Int64(512)
@@ -660,6 +669,9 @@ var _ = Describe("MySQLCluster reconciler", func() {
  for _, c := range sts.Spec.Template.Spec.Containers {
  Expect(c.Name).NotTo(Equal(constants.SlowQueryLogAgentContainerName))
  switch c.Name {
+ case constants.MysqldContainerName:
+ Expect(c.StartupProbe).NotTo(BeNil())
+ Expect(c.StartupProbe.FailureThreshold).To(Equal(int32(1)))
  case constants.AgentContainerName:
  Expect(c.Args).To(ContainElement("20s"))
  Expect(c.Args).To(ContainElement("0 * * * *"))

docs/crd_mysqlcluster.md

@@ -66,6 +66,7 @@ MySQLClusterSpec defines the desired state of MySQLCluster
 | replicationSourceSecretName | ReplicationSourceSecretName is a `Secret` name which contains replication source info. If this field is given, the `MySQLCluster` works as an intermediate primary. | *string | false |
 | serverIDBase | ServerIDBase, if set, will become the base number of server-id of each MySQL instance of this cluster. For example, if this is 100, the server-ids will be 100, 101, 102, and so on. If the field is not given or zero, MOCO automatically sets a random positive integer. | int32 | false |
 | maxDelaySeconds | MaxDelaySeconds, if set, configures the readiness probe of mysqld container. For a replica mysqld instance, if it is delayed to apply transactions over this threshold, the mysqld instance will be marked as non-ready. The default is 60 seconds. | int | false |
+| startupDelaySeconds | StartupWaitSeconds is the maximum duration to wait for `mysqld` container to start working. The default is 3600 seconds. | int32 | false |
 | logRotationSchedule | LogRotationSchedule specifies the schedule to rotate MySQL logs. If not set, the default is to rotate logs every 5 minutes. See https://pkg.go.dev/github.com/robfig/cron?#hdr-CRON_Expression_Format for the field format. | string | false |
 | restore | Restore is the specification to perform Point-in-Time-Recovery from existing cluster. If this field is filled, start restoring. This field is unable to be updated. | *[RestoreSpec](#restorespec) | false |
 | disableSlowQueryLogContainer | DisableSlowQueryLogContainer controls whether to add a sidecar container named \"slow-log\" to output slow logs as the containers output. If set to true, the sidecar container is not added. The default is false. | bool | false |