From f2ad97267f3bf73117738ebc6dc670d4e9cdc4d2 Mon Sep 17 00:00:00 2001 From: ymmt Date: Mon, 26 Apr 2021 01:47:03 +0000 Subject: [PATCH] Update documents and prepare for a new release --- .github/workflows/ci.yaml | 1 + .github/workflows/release.yaml | 47 +----- CHANGELOG.md | 9 +- DEVELOP.md | 45 ++++++ Makefile | 23 ++- README.md | 118 +++++++++++--- clustering/mock_test.go | 33 ++-- clustering/suite_test.go | 2 +- controllers/mysql_container.go | 2 +- docs/design.md | 248 +++------------------------- docs/images/overview.png | Bin 49163 -> 0 bytes links.csv => docs/links.csv | 0 docs/metrics.md | 6 + docs/setup.md | 47 ++---- docs/usage.md | 268 +++++++++++++++++++++++++++++-- e2e/lifecycle_test.go | 27 ++-- e2e/replication_test.go | 48 +++--- examples/anti-affinity.yaml | 44 +++++ examples/custom-mycnf.yaml | 35 ++++ examples/guaranteed.yaml | 30 ++++ examples/loadbalancer.yaml | 29 ++++ go.mod | 2 +- go.sum | 4 +- kustomization.yaml | 2 +- pkg/mycnf/generator.go | 1 + pkg/mycnf/testdata/bufsize.cnf | 1 + pkg/mycnf/testdata/loose.cnf | 1 + pkg/mycnf/testdata/nil.cnf | 1 + pkg/mycnf/testdata/normalize.cnf | 1 + version.go | 2 +- 30 files changed, 697 insertions(+), 380 deletions(-) create mode 100644 DEVELOP.md delete mode 100644 docs/images/overview.png rename links.csv => docs/links.csv (100%) create mode 100644 examples/anti-affinity.yaml create mode 100644 examples/custom-mycnf.yaml create mode 100644 examples/guaranteed.yaml create mode 100644 examples/loadbalancer.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a192e54eb..9f72b24c6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,6 +19,7 @@ jobs: - run: make test - run: make check-generate - run: make envtest + - run: make release-build test-dbop: name: Test pkg/dbop strategy: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dcfea8b85..0c6be3ecc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - run: docker build . -t moco:dev + - run: docker build -t moco:dev . - name: Login to ghcr.io run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin - run: docker tag moco:dev ghcr.io/cybozu-go/moco:${GITHUB_REF#refs/tags/v} @@ -25,44 +25,13 @@ jobs: - uses: actions/setup-go@v2 with: go-version: ${{ env.go-version }} - - run: | - make release-build + - run: make release-build - name: Create Release - id: create_release - uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref }} - body: | - See [CHANGELOG.md](./CHANGELOG.md) for details. - draft: false - prerelease: ${{ contains(github.ref, '-') }} - - name: Upload kubectl-moco for linux-amd64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./build/kubectl-moco-linux-amd64 - asset_name: kubectl-moco-linux-amd64 - asset_content_type: application/octet-stream - - name: Upload kubectl-moco for windows-amd64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./build/kubectl-moco-windows-amd64.exe - asset_name: kubectl-moco-windows-amd64.exe - asset_content_type: application/octet-stream - - name: Upload kubectl-moco for darwin-amd64 - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./build/kubectl-moco-darwin-amd64 - asset_name: kubectl-moco-darwin-amd64 - asset_content_type: application/octet-stream + run: | + tagname="${GITHUB_REF#refs/tags/}" + if echo ${{ github.ref }} | grep -q -e '-'; then prerelease=-p; fi + gh release create -t "Release $tagname" $prerelease \ + -n "See [CHANGELOG.md](./CHANGELOG.md) for details." \ + "$tagname" build/* diff --git a/CHANGELOG.md b/CHANGELOG.md index f6dae7ffb..dd791dd19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +## [0.8.0] - 2021-04-27 + +### Changed +- Everything. There is no backward compatibility. (#228) +- The older release must be uninstalled before installing this version. + ## [0.7.0] - 2021-02-22 Since v0.7.0, MOCO will no longer use CronJob for log rotation. @@ -131,7 +137,8 @@ The `MySQLCluster` created by MOCO `< v0.5.0` has no compatibility with `>= v0.5 - Bootstrap a vanilla MySQL cluster with no replicas (#2). -[Unreleased]: https://github.com/cybozu-go/moco/compare/v0.7.0...HEAD +[Unreleased]: https://github.com/cybozu-go/moco/compare/v0.8.0...HEAD +[0.8.0]: https://github.com/cybozu-go/moco/compare/v0.8.0...v0.8.0 [0.7.0]: https://github.com/cybozu-go/moco/compare/v0.6.0...v0.7.0 [0.6.0]: https://github.com/cybozu-go/moco/compare/v0.5.1...v0.6.0 [0.5.1]: https://github.com/cybozu-go/moco/compare/v0.5.0...v0.5.1 diff --git a/DEVELOP.md b/DEVELOP.md new file mode 100644 index 000000000..a898325d3 --- /dev/null +++ b/DEVELOP.md @@ -0,0 +1,45 @@ +# How to develop MOCO + +## Running tests + +MOCO has the following 4 kinds of tests: + +1. Tests that do not depend on MySQL or Kubernetes +2. `pkg/dbop` tests that depend on MySQL version +3. Tests that depend on Kubernetes and therefore run by controller-runtime's envtest +4. End-to-end tests + +To run these tests, use the following make targets respectively: + +1. `make test` +2. `make test-dbop` +3. `make envtest` +4. Read [`e2e/README.md`](e2e/README.md) + +## Generated files + +Some files in the repository are auto-generated. + +- [`docs/crd_mysqlcluster.md`](docs/crd_mysqlcluster.md) is generated by `make apidoc`. +- Some files under `config` are generated by `make manifests`. +- `api/**/*.deepcopy.go` are generated by `make generate`. + +CI checks and fails if they need to be rebuilt. + +## Testing with unreleased moco-agent + +MOCO depends on [moco-agent][] that is released from a different repository. +The dependency is therefore managed in `go.mod` file. + +To run e2e tests with an unreleased moco-agent, follow the instructions in +[`e2e/README.md`](e2e/README.md). + +## Updating moco-agent + +Run `go get github.com/cybozu-go/moco-agent@latest`. + +## Updating fluent-bit + +Edit `FluentBitImage` in [`version.go`](versoin.go). + +[moco-agent]: https://github.com/cybozu-go/moco-agent diff --git a/Makefile b/Makefile index 5c5ffbd32..6091c5b6c 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,11 @@ SHELL = /bin/bash # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS = "crd:crdVersions=v1" +# for Go +GOOS = $(shell go env GOOS) +GOARCH = $(shell go env GOARCH) +SUFFIX = + .PHONY: all all: build @@ -49,7 +54,7 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and .PHONY: apidoc apidoc: crd-to-markdown $(wildcard api/*/*_types.go) echo $(wildcard api/*/*_types.go) - $(CRD_TO_MARKDOWN) -f api/v1beta1/mysqlcluster_types.go -n MySQLCluster > docs/crd_mysqlcluster.md + $(CRD_TO_MARKDOWN) --links docs/links.csv -f api/v1beta1/mysqlcluster_types.go -n MySQLCluster > docs/crd_mysqlcluster.md .PHONY: check-generate check-generate: @@ -97,6 +102,22 @@ build: mkdir -p bin GOBIN=$(shell pwd)/bin go install ./cmd/... +.PHONY: release-build +release-build: kustomize + mkdir -p build + $(MAKE) kubectl-moco GOOS=windows GOARCH=amd64 SUFFIX=.exe + $(MAKE) kubectl-moco GOOS=darwin GOARCH=amd64 + $(MAKE) kubectl-moco GOOS=darwin GOARCH=arm64 + $(MAKE) kubectl-moco GOOS=linux GOARCH=amd64 + $(MAKE) kubectl-moco GOOS=linux GOARCH=arm64 + $(KUSTOMIZE) build . > build/moco.yaml + +.PHONY: kubectl-moco +kubectl-moco: build/kubectl-moco-$(GOOS)-$(GOARCH)$(SUFFIX) + +build/kubectl-moco-$(GOOS)-$(GOARCH)$(SUFFIX): + CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ ./cmd/kubectl-moco + ##@ Tools CONTROLLER_GEN := $(shell pwd)/bin/controller-gen diff --git a/README.md b/README.md index fbf817bff..d1e30e99a 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,117 @@ [![GitHub release](https://img.shields.io/github/release/cybozu-go/moco.svg?maxAge=60)][releases] -[![CircleCI](https://circleci.com/gh/cybozu-go/moco.svg?style=svg)](https://circleci.com/gh/cybozu-go/moco) +[![CI](https://github.com/cybozu-go/moco/actions/workflows/ci.yaml/badge.svg)](https://github.com/cybozu-go/moco/actions/workflows/ci.yaml) [![PkgGoDev](https://pkg.go.dev/badge/github.com/cybozu-go/moco)](https://pkg.go.dev/github.com/cybozu-go/moco) [![Go Report Card](https://goreportcard.com/badge/github.com/cybozu-go/moco)](https://goreportcard.com/report/github.com/cybozu-go/moco) # MOCO -MOCO is a Kubernetes operator for MySQL. -Its primary function is to manage a cluster of MySQL using binlog-based, semi-synchronous replication. +MOCO is a Kubernetes operator for [MySQL][]. +Its primary function is to manage MySQL clusters using [GTID-based](https://dev.mysql.com/doc/refman/8.0/en/replication-gtids.html) [semi-synchronous](https://dev.mysql.com/doc/refman/8.0/en/replication-semisync.html) replication. It does _not_ manage [group replication](https://dev.mysql.com/doc/refman/8.0/en/group-replication.html) clusters. -MOCO is designed for the following properties: +MOCO is designed to have the following properties. -- Durability - - Do not lose any data under a given degree of faults. +- Compatibility with the standard MySQL + - This is the reason that MOCO does not adopt group replication that has [a number of limitations](https://dev.mysql.com/doc/refman/8.0/en/group-replication-limitations.html). +- Safety + - MOCO only allows writes to a single instance called the primary at a time. + - MOCO configures loss-less semi-synchronous replication with sufficient replicas. + - MOCO detects and excludes instances having [errant transactions](https://www.percona.com/blog/2014/05/19/errant-transactions-major-hurdle-for-gtid-based-failover-in-mysql-5-6/). - Availability - - Keep the MySQL cluster available under a given degree of faults. -- Business Continuity - - Perform a quick recovery if some failure is occurred. + - MOCO can quickly switch the primary in case of the primary failure or restart. + - MOCO allows up to 5 instances in a cluster. -## Features - -TBD +## Supported software -## Supported MySQL versions +- MySQL: 8.0.18, 8.0.20, and 8.0.24 +- Kubernetes: 1.19 and 1.20 -8.0.18 and 8.0.20 +Other versions may work, though not tested. -## Supported Kubernetes version +## Features -1.20 and 1.19 +- Cluster of 1, 3, or 5 MySQL instances +- Replication from an external MySQL instance +- Manual and automatic switchover of the primary instance +- Automatic failover of the primary instance +- Errant transaction detection +- Different MySQL versions for each cluster +- Upgrading MySQL version of a cluster +- Monitor for replication delays +- Service for the primary and replicas, respectively +- Custom `my.cnf` configurations +- Custom Pod, Service, and PersistentVolumeClaim templates +- Redirect slow query logs to a sidecar container +- (planning) Backup and [Point-in-Time Recovery](https://dev.mysql.com/doc/refman/8.0/en/point-in-time-recovery-positions.html) + +## Quick start + +You can quickly run MOCO using [kind](https://kind.sigs.k8s.io/). + +1. Prepare a Linux machine and install Docker. +2. Checkout MOCO and go to `e2e` directory. +3. Run `make start` + +You can then create a three-instance MySQL cluster as follows: + +```console +$ cat > mycluster.yaml <<'EOF' +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + replicas: 3 + podTemplate: + spec: + containers: + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi +EOF + +$ export KUBECONFIG=$(pwd)/.kubeconfig +$ ../bin/kubectl apply -f mycluster.yaml +``` + +Check the status of MySQLCluster until it becomes healthy as follows: + +```console +$ ../bin/kubectl get mysqlcluster test +NAME AVAILABLE HEALTHY PRIMARY SYNCED REPLICAS ERRANT REPLICAS +test True True 0 3 +``` + +Once it becomes healthy, you can use `kubectl-moco` to play with `mysql` client. + +```console +$ ../bin/kubectl moco mysql -it test +``` + +To destroy the Kubernetes cluster, run: + +```console +$ make stop +``` ## Documentation -[docs](docs/) directory contains documents about designs and specifications. +- [`docs/setup.md`](docs/setup.md) for installing MOCO. +- [`docs/usage.md`](docs/usage.md) is the user manual of MOCO. +- [`examples`](examples/) directory contains example MySQLCluster manifests. -## Docker images - -Docker images are available on [Quay.io](https://quay.io/repository/cybozu/moco) +[`docs`](docs/) directory also contains other design or specification documents. -## License +## Docker images -MOCO is licensed under MIT license. +Docker images are available on [ghcr.io/cybozu-go/moco](https://github.com/orgs/cybozu-go/packages/container/package/moco). [releases]: https://github.com/cybozu-go/moco/releases -[godoc]: https://godoc.org/github.com/cybozu-go/moco +[MySQL]: https://www.mysql.com/ diff --git a/clustering/mock_test.go b/clustering/mock_test.go index c8c352476..2a65ad6b7 100644 --- a/clustering/mock_test.go +++ b/clustering/mock_test.go @@ -93,22 +93,25 @@ func (a *mockAgentConn) Clone(ctx context.Context, in *agent.CloneRequest, opts } func setPodReadiness(ctx context.Context, name string, ready bool) { - pod := &corev1.Pod{} - err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: name}, pod) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - if ready { - pod.Status.Conditions = []corev1.PodCondition{ - { - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }, + EventuallyWithOffset(1, func() error { + pod := &corev1.Pod{} + err := k8sClient.Get(ctx, client.ObjectKey{Namespace: "test", Name: name}, pod) + if err != nil { + return err } - } else { - pod.Status.Conditions = nil - } - err = k8sClient.Status().Update(ctx, pod) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + if ready { + pod.Status.Conditions = []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }, + } + } else { + pod.Status.Conditions = nil + } + return k8sClient.Status().Update(ctx, pod) + }).Should(Succeed()) } type mockAgentFactory struct { diff --git a/clustering/suite_test.go b/clustering/suite_test.go index cea5c1b1c..02b8d3083 100644 --- a/clustering/suite_test.go +++ b/clustering/suite_test.go @@ -35,7 +35,7 @@ var mysqlPassword *password.MySQLPassword func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) - SetDefaultEventuallyTimeout(10 * time.Second) + SetDefaultEventuallyTimeout(30 * time.Second) SetDefaultEventuallyPollingInterval(100 * time.Millisecond) SetDefaultConsistentlyDuration(3 * time.Second) SetDefaultConsistentlyPollingInterval(100 * time.Millisecond) diff --git a/controllers/mysql_container.go b/controllers/mysql_container.go index 243397afc..c071a932b 100644 --- a/controllers/mysql_container.go +++ b/controllers/mysql_container.go @@ -204,7 +204,7 @@ func (r *MySQLClusterReconciler) makeV1InitContainer(cluster *mocov1beta1.MySQLC Image: image, Command: []string{ constants.InitCommand, - "--base-dir=" + constants.MySQLDataPath, + "--data-dir=" + constants.MySQLDataPath, "--conf-dir=" + constants.MySQLInitConfPath, fmt.Sprintf("%d", cluster.Spec.ServerIDBase), }, diff --git a/docs/design.md b/docs/design.md index eb107045d..c8de54c53 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1,36 +1,31 @@ Design notes ============ -Motivation ----------- +## Motivation -This Kubernetes operator automates operations for the binlog-based replication on MySQL. +We are creating our own Kubernetes operator for clustering MySQL instances for the following reasons: -InnoDB cluster can be used for the replication purpose, but we choose not to use InnoDB cluster because it does not allow large (>2GB) transactions. +Firstly, our application requires strict-compatibility to the traditional MySQL. Although recent MySQL provides an advanced clustering solution called [group replication](https://dev.mysql.com/doc/refman/8.0/en/group-replication.html) that is based on [Paxos](https://en.wikipedia.org/wiki/Paxos_(computer_science)), we cannot use it because of [various limitations from group replication](https://dev.mysql.com/doc/refman/8.0/en/group-replication-limitations.html). -There are some existing operators which deploy a group of MySQL servers without InnoDB cluster but they does not support the Point-in-Time-Recovery(PiTR) feature. +Secondly, we want to have a Kubernetes native and the simplest operator. For example, we can use Kubernetes Service to load-balance read queries to multiple replicas. Also, we do not want to support non-GTID based replications. -- [oracle/mysql-operator](https://github.com/oracle/mysql-operator) takes backups only with `mysqldump` -- [presslabs/mysql-operator](https://github.com/presslabs/mysql-operator) does not restore clusters to the state at the desired Point-in-Time +Lastly, none of the existing operators could satisfy our requirements. -This operator deploys a group of MySQL servers which replicates data semi-synchronously to the replicas and takes backups with both `mysqlpump` and `mysqlbinlog`. +## Goals -In this context, we call the group of MySQL servers as MySQL cluster. - -Goals ------ - -- Have the primary replicate data semi-synchronously to the multiple replicas +- Manage primary-replica cluster of MySQL instances. + - The primary instance is the only instance that allows writes. + - Replica instances replicate data from the primary. +- Support replication from an external MySQL instance. - Support all the four transaction isolation levels. -- Avoid split-brain. +- No split-brain is allowed. - Accept large transactions. -- Upgrade this operator without restarting MySQL `Pod`s. +- Upgrade this operator without restarting MySQL Pods. - Support multiple MySQL versions and automatic upgrading. - Support automatic primary selection and switchover. - Support automatic failover. -- Support backups at least once in a day. -- Support a quick recovery by combining full backup and binary logs. -- Support asynchronous replication between remote data centers. +- Backup and restore features. +- Support point-in-time-recovery (PiTR). - Tenant users can specify the following parameters: - The version of MySQL instances. - The number of processor cores for each MySQL instance. @@ -41,32 +36,27 @@ Goals - Allow `CREATE / DROP TEMPORARY TABLE` during a transaction. - Use Custom Resource Definition(CRD) to automate construction of MySQL database using replication on Kubernetes. -Non-goals ---------- +## Non-goals -- Support for InnoDB cluster. -- Zero downtime upgrade. - Node fencing. - - Fencing should be done externally. Once Pod and PVC/PV are removed as a consequence of node fencing, the operator will restore the cluster appropriately. -Components ----------- + Fencing should be done externally. Once Pod and PVC/PV are removed as a consequence of node fencing, the operator will restore the cluster appropriately. + +## Components ### Workloads - Operator: Custom controller which automates MySQL cluster management with the following namespaced custom resources: - - [MySQLCluster](crd_mysql_cluster.md) represents a MySQL cluster. - - [ObjectStorage](crd_object_storage.md) represents a connection setting to an object storage which has Amazon S3 compatible API (e.g. Ceph RGW). + - [MySQLCluster](crd_mysql_cluster.md) represents a MySQL cluster. + - [ObjectStorage](crd_object_storage.md) represents a connection setting to an object storage which has Amazon S3 compatible API (e.g. Ceph RGW). - Admission Webhook: Webhook for validating custom resources (e.g. validate the object storage for backup exists). - [cert-manager](https://cert-manager.io/): Provide client certifications and primary-replica certifications automatically. ### Tools - `kubectl-moco`: CLI to manipulate MySQL cluster. It provides functionalities such as: - - Change primary manually. - - Port-forward to MySQL servers. - - Execute SQL like `mysql -u -p` without a credential file on a local environment. - - Fetch a credential file to a local environment. + - Execute `mysql` client for a MySQL instance running on Kubernetes. + - Fetch a credential file to a local environment. ### Diagram @@ -85,199 +75,5 @@ In this section, the name of `MySQLCluster` is assumed to be `mysql`. - `Service` for accessing replicas, both for MySQL protocol and X protocol. - `Secrets` to store credentials. - `ConfigMap` to store cluster configuration. - - `ServiceAccount`, `Role` and `RoleBinding` to allow Pods to access resources. Read [reconcile.md](reconcile.md) on how MOCO reconciles the StatefulSet. - -#### How to implement the initialization of MySQL pods with avoiding unnecessary restart at the operator update - -When initializing MySQL pods, the following procedures should be executed in their init containers. - -- Initialize data-dir. -- Create mysql users. -- Create `my.cnf`. -- etc. - -When upgrading the operator, we want to avoid unnecessary restarts of MySQL pods. -Creating mysql users and initializing data-dir are done in an init container of which image is the same with -the `mysqld` container, so the MySQL pods are not restarted at the upgrade. - -`my.cnf` is created by merging the following three configurations. - -- Default: This is created by the operator and contains default value of `my.cnf`. -- User: This is created by users and contains user-defined values of `my.cnf`. This overwrites Default values. -- Constant: This is created by the operator. This overwrites User values. - -The operator contains the Default and Constant configurations for all MySQL clusters, -and the User configuration for a certain MySQL cluster specified in the cluster's CR. -The operator merges these three files and creates a ConfigMap that will be mounted on MySQL Pods. -For instance-specific parameters such as `server-id`, an init container creates a supplementary config file. -The main `my.cnf` contains a stanza to include this supplementary config file. - -If users want to change their MySQL cluster configurations without restarting Pods, they should use `SET GLOBAL ...`. - -So, in short we prepare an init container that does the following two things. - -1. Initialize the MySQL cluster, for example, creating necessary users. -2. Create a supplementary config file for `my.cnf`. - -For this purpose, the `moco-agent` binary is provided by [cybozu-go/moco-agent](https://github.com/cybozu-go/moco-agent). - -Behaviors ---------- - -### How to watch the status of instances - -Fetch the following information with mysql-client from each instance: - -- SHOW MASTER STATUS - - Executed_Gtid_Set -- SHOW SLAVE STATUS - - Master_Host - - Executed_Gtid_Set - - Retrieved_Gtid_Set - - Slave_IO_Running - - Slave_SQL_Running - - Last_IO_Errno - - Last_IO_Error - - Last_SQL_Errno - - Last_SQL_Error -- select @@global.read_only, @@global.super_read_only; - - read_only - - super_read_only -- performance_schema.clone_status table - - STATE - -### How to bootstrap MySQL Cluster - -When all instances are the initial state, bootstrap the cluster as follows: - -1. Select the first instance as the primary. -2. Set the replication source using `CHANGE MASTER TO` for a replica instances. -3. Start replication using `START SLAVE` on each replicas. -4. Create Service resources to access to primary and replicas. -5. Update `MySQLCluster.status.currentPrimaryIndex` with the primary name. -6. Turn off read-only mode on the primary. - -### How to execute failover when the primary fails - -When the primary fails, the cluster is recovered in the following process: - -1. Stop the `IO_THREAD` of all replicas. -2. Select and configure new primary. -3. Update `MySQLCluster.status.currentPrimaryIndex` with the new primary name. -4. Turn off read-only mode on the new primary - -In the process, the operator configures the old primary as replica if the server is ready. - -### How to execute failover when a replica fails - -When a replica fails once and it restarts afterwards, the operator basically configures it to follow the primary. - -#### How to handle the case a replica continues to fail and restart - -If one of the replicas fails again after restarting, the `StatefulSet` controller restarts the replica again. -This means that the replica may continue to fail and restart again and again. -This loop can occur, for example, in the case that the data in the replica is corrupted. -Users must handle this failure manually by deleting the `Pod` or/and `PersistentVolumeClaim`. -Then, the replica `Pod` is scheduled again onto a different node and the operator configures it to follow the primary automatically. - -Users can keep the data with a [`VolumeSnapshot`](https://kubernetes.io/docs/concepts/storage/volume-snapshots/) and -create `PersistentVolumeClaim` from the snapshot even after this failure happens. -This feature is available only if the underlying `StorageClass` supports it. - -### How to execute switchover - -Users can execute primary switchover by applying `SwitchoverJob` CR which contains the primary index to be switched to. - -Note that while any `SwitchoverJob` is running, another `SwitchoverJob` can be created but the operator waits for the completion of running jobs. - -### How to make a backup - -When you create `MySQLBackupSchedule` CR, it creates `CronJob` which stores dump and binlog to an object storage: - -If we want to make backups only once, set `MySQLBackupSchedule.spec.schedules` to run once. - -### How to perform PiTR - -When we create a `MySQLCluster` with `.spec.restore` specified, the operator performs PiTR with the following procedure. - -`.spec.restore` is unable to be updated, so PiTR can be executed only when the cluster is being creating. - -1. The operator sets the source cluster's `.status.ready` as `False` and make the MySQL cluster block incoming transactions. -2. The operator makes the MySQL cluster flush binlogs from the source `MySQLCluster`. This binlog is used for recovery if the PiTR fails. -3. The operator lists `MySQLBackup` candidates based on `MySQLCluster.spec.restore.sourceClusterName`. -4. The operator selects the corresponding `MySQLBackup` CRs according to `MySQLCluster.spec.restore.pointInTime`. -5. The operator downloads the dump file and the binlogs for `MySQLCluster.spec.restore.pointInTime` from the object storage. -6. The operator restores the MySQL servers to the state at `MySQLCluster.spec.restore.pointInTime`. -7. If the recovery succeeds, the operator sets the source cluster's `.status.ready` as `True`. - -### How to upgrade MySQL version of primary and replicas - -MySQL software upgrade is triggered by changing container image specified in `MySQLCluster.spec.podTemplate`. -In this section, the name of `StatefulSet` is assumed to be `mysql`. - -1. Switch primary to the pod `mysql-0` if the current primary is not `mysql-0`. -2. Update and apply the `StatefulSet` manifest for `mysql`, to trigger upgrading the replicas as follows: - - Set `.spec.updateStrategy.rollingUpdate.partition` as `1`. - - Set new image version in `.spec.template.spec.containers`. -3. Wait for all the replicas to be upgraded. -4. Switch primary to `mysql-1`. -5. Update and apply the `StatefulSet` manifest to trigger upgrading `mysql-0` as follows: - - Remove `.spec.updateStrategy.rollingUpdate.partition`. -6. Wait for `mysql-0` to be upgraded. - -### How to manage recovery from blackouts - -In the scenario of the recovery from data center blackouts, all members of the MySQL cluster perform cold boots. - -The operator waits for all members to boot up again. -It automatically recovers the cluster only after all members come back, not just after the quorum come back. -This is to prevent the data loss even in corner cases. - -If a part of the cluster never finishes boot-up, users must intervene the recovery process. -The process is as follows. -1. Users delete the problematic Pods and/or PVCs. (or they might have been deleted already) - - Users need to delete PVCs before Pods if they want to delete both due to the [Storage Object in Use Protection](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#storage-object-in-use-protection). - - [The StatefulSet controller may recreate Pods improperly](https://github.com/kubernetes/kubernetes/issues/74374). - Users may need to delete the Nodes in this case. -2. The StatefulSet controller creates new blank Pods and/or PVCs. -3. The operator tries to select the primary of the cluster only after all members are up and running, and the quorum of the cluster has data. - - If the quorum of the cluster does not have data, users need to recover the cluster from a backup. -4. The operator initializes the new blank Pods as new replicas. - -### How to collect metrics for Prometheus - -To avoid unnecessary restart when the operator is upgraded, we prepare external Pods which collect mysqld metrics over the network, -and export them as Prometheus metrics. - -The detail is TBD. - -### How to manage log files - -The operator configures MySQL to output error logs and slow query logs into files. - -To avoid exhaustion of storage resources, the operator appends a sidecar container to the MySQL Pod. -The sidecar container rotates and deletes the log files based on the cron spec in `MySQLCluster.spec.logRotationSchedule`. - -The operator does not care about gathering the contents of the log files. -Tenant users can extract the contents by defining sidecar containers for `slow.log` and `error.log`. - -You can see an example with Fluent Bit in [Example Document](example_mysql_cluster.md). - -### How to delete resources (garbage collection) - -The operator sets an owner reference of MySQLCluster to the child resources. -By the owner reference, when the parent MySQLCluster is deleted, child resources are automatically deleted. - -The data volumes (PVCs) are also deleted automatically. -If you want to prevent the PVCs deletion, please edit the PVCs manifest and remove the owner reference manually. - -### TBD - -- Write merge strategy of `my.cnf`. -- How to clean up zombie backup files on a object storage which does not exist in Kubernetes. - -### Candidates of additional features - -- Backup files verification. diff --git a/docs/images/overview.png b/docs/images/overview.png deleted file mode 100644 index 799b1ff06eaea5f40e271e9c875395e64eee5a1f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 49163 zcmeFZ2UJsA*EXtvASy*fkq%NVw1kjEP%QK&q15RY3 z*fP0gixw>svmseIFIptf2md^m3d5D-swmS%i&l8Ctv%RboPeNE|3w;T%U|y_Q2K!k z7Fz>trGY~EL`3M)f_&(aK4C1~aDO&jg73o^w4i_>f7-8mQ2Ho+9es?BKF(DitARGx z$H5C4rHjU)ynfy9L-!B=-5^F+9~#hh@bL=@XERuV8fXjnY!lA*4~75XGQ2vvz>5d` zGtl2@V7Sw0J-juKhzRv}_xH66f-w*CJj~`s`2V^}XE5lY&?!m-Z4Dy*Ejzd++y}<; zr%atd96AQS9t`>|Lp%zPH^TbRbnxf^ypEB7fT2!+kFSvqhK9!BQHGcRBRoh3X%tIk zdxhc|D58@-JHX%44^Oua@PWAygE*Yfu#m72t{>ixXm4#0;7_Nx8`^L=9+uWbuAK(T zA{t}v7VQvh>1&P&b0eT)*-v+m8L%i826VKWgNGA^ zZo#6W4I&I2Q0|mS3y%oT7=k;ILf|r71EZ|bp+NyY{{9TE6*(w~8fw8Mu|i^)F8+@G z&fHj*4>#1!g=EMK#js+SWTVKKFq%UUIndG0%Q?iIf$<8640o`#jCHa=8Adp84Lzd> z`k~=aE6W<}DSB0Q+k<}UVLzV28L ze?%h)zLEZD0u~kELUW9eGNR(eJDL~&n(2GX4XHh*JUBhhAp)pu?FeTE~ zAPi#_!6XI+a;zE7Ax_qq5R_doH_F|BVe4eYCWhf6^%3kettm|e>$CR6wHcX5<=+uu8wTqXgjPs(}-mk7{qlAhH1Mv`?2)R!<`+i z^{G)f=P;w-5S)I9r!&RD-_M8W;>PkZ;CeVlb8MN0Se&Z|-US`thb597{75!|u|7-& z3T26rUEF(P3BiQVy7&h6+*ETZ9F4Dt_VPtK@ z@`|tuaSp-Q*+xf1Lbr}K+;FcD{iu+@Ffxwo6&T9&G_-Y!!UVCcY&~&qblac^jHfd< zB!X<`MGGcT$<}^$2DCtHHWj+j54B{uy7~mTMKUAwS z$bmE$93k30HYU`|&cNRXSt%5eWJy9fa0r&JPBx?nj71nKEYh44%ApVg>_m!#vs-=*w@oB z!r6_*3gHriBDffwGu4F~9PG;?g>yoT0xgWfU93La0(~J zL{X#YvB94996X&B7)Zo>#&|h$LphFkN3x+cD%>cD<>(pg>JsfvGRLEWyjZ9JuP8gJ zqXmub5fu?>9^vZYVB;PW5$Mb%A=@77W#wey#KcCsz;ak{>;gTaV5nAjSJzOwxjqI> zwj`jLgaCSwrMasQmrbIu%wv4~0)wbNk!TB7Yl??sFq6u5wZg?x%u&#ShYi;&$Tq?a zWA12R!FJ$=z%{NV!Oeoej0iFJwF)(|k2Rnf22$DhNSu45uWc;D-U06M4Advmqfnut zA$~y&2FIRd6>UH=CvojPEm$sStTi5MZSO=QqUgT(P^xnzJJ<{>MIqx1 zXrY$iH3){G4u1MkK^#}2evrGDk0HhtZRZ>iO7)L*@kF{H+l69C?iO~2Rs>8Co`y!- z*%@&07CFRZ>pAerhGP6%~GyM~Yqu@)2; z`%nu!m&(EWh2er-m|-+aeQQ)G#lhan-O`Y09{`UATBDg%qOEPPk&8Rti|S8vr&&aC zNDjURo}^%R%P>4U0&5-}70I-5W(T9^YBw(;s z(Ks?W2sXC0GX>>`^<>15Ndc%B2NE@$;p!BNW4W7iFyt8j5Jy*QLw65~IomCmv>0JD_4W)2rdy9s8995(+O@= zs*^`F^x@A%{Edr0#@F5@z!ghIMe3u1g5lLF%0J3E(kD73*gV`l#K*zVit8KgObswJ zx4`<4LKrN41~%G?iSaXvW>YXMw-_3QgZ6Mk`+7P-Y)3(yC)dX(mguh^L&y8;HN&;Gj{pp$anYG zuXj23_5Lf|N-34a@JE+A!7)bbDy$auu-NtJlr<{&srHN8iHVg9=6i(OX81$?orM_Z zs{x%tmw$BV-6lb6ZwH$W9ySW|vTM%sIG?Y(<8aeK8M}jPbkq}OzqZ{n+8a04Q#JYX zOP9!x_I$n4B!?GQ{QaMtb9h|K^usd;tz4hSetR8rmct)BY??S<_F%+D-An-f`ZUb= z)8V0bwA74D@^kMvhxsq13#}5SKTnu0Cw-fp`gk_@*&hd_uDb9XOEznGUJW>UnwS+F zu=$4Am&B=O?*88=hn;W5b61Z&I!)}jw{Q9I!;_Y=F3vUKJHCJEGCh6M^?;Pd+Z!Xt zkDPNTB1COi zI_mD7^JS#V?qhd3W+-Em!GpHL8V3V1=j&^%p4{keSbHp`oq^tDu zutN4b`S4ov&$kzL&uy6>8_F`xb3SW(VWKa(H!Sk!43j(ArZxP;(R6j|#w!yVc6qrQ z-2Ql9s(O$Y{(AF`*s0)@79Ry1y^qI@9JsgSLw9-63R#qu^BL00w@Ei>_m5GsKJ--i zO>{9$-n}|$DT0ri4t3(K8ZwCJH!}~2YtLPOC9KzoJ66?_`)sf1s;~0%W~bOBUE(^=n;}v|wF0N%$>N zOEn~0IoQx1C+~Q^wdWPufxHMIIaI%v)m_hfrYLTW34?R5>&qKj+@uoH(9N; zkS)u(H8vBEbi5G)irVHEAZIWKCINEA~ExwIQJ8!;Wb!~Zk(OqwD ziEP6AlWDH+nU+Z_e%3OFzG^OV&ie9(JEC~O<$R`VnUm_>A(c5XIm7JL8>y|1rgQ2$ z>Z08!i4%84_I&BBkE@a1NVOMNGSQBClMs_~&~KC%)JjWOTQCWGWhKvPK@beP|r#PYhQ2G|(DS#WoBFNW%Gv^>J>Q23=;oJnq%J ze3iQ6TjHk6n&h!7xsz{mKv2PX3d8zkiF&SO*#RE~&uZ4LY|YczHbD?Bh^7sH8+&nj zFyj*L0bT9aCJ=#5U^;l;^Kg=sS+o3Lebh(l>rTPLxPbd>I|H@U`y_o$Uaf8AoZi09 zGId4lCZ)!`dzJB#RvnuLYe4ux)3O4G($pTQ^hvUpYRRd81w)Ywz5QX>!gzAt@wutQ zerzMYYAD<0^Q#+I^D6qA1zz~)HtgM+EnDWc^Lp^nr?Bo1h6B2emwSGD=zL^Gc6^aU z?uq^Y!56`Xw42OTDUmr&{M7x*xCoU8QTj&vH#f?MW{dvQ;LV(PmdX{d9%s74#$VlV z2`N)8q(|dop^IA=?@MINlA|<&VMH^krkV2mGq@Ws#auMnf%=E zH4G~=^*s9lwvS`EIcoIG@M>-Oe^Q7UZ!D)*8H@YPzdxOIEnHE<@aRVO?hs~K-9uIwL+sf+$}X8VoEl*21%)nDFqUX)N1 zZj`?w+ZuYBCnP~QH85~5-#gYOF{?13Rj*phN_P2`o01NOOM`zN!tPbx_CqyJY4!v? zP;|xU6-qz~OSM^4G0lF<=xu{&k}hgyqkBc}w(uJopk_HnAol#*GBoozbaUm8DozirS|(u~B$f~)Gu`$bPE2zu^n^?6T?PZoCyPg!ADJ-j^Z_QHH! z-h~r1wU_y0yXs6$cFDFzrcGz9FsV40*TXC$n#RvmDy5~Nj|6Kw+`X>fWp-Osu~b^Z zWNkV_Y>=^Jzg#%&HAQoeh&5B8nOeE)u22C%K|zW1tM_2&-90U9kJg_%^-rHI6|o%l z>$Y`G%)@^dQPdVLYU+J9Fx3CcPW=iFURi*($|Et0f7$Y1uN9hR{OG6|9OtyqM7Dh2 zq1Im$Mdq3y(2yqaT=9DGW9>fj0il+as{|w&yoW-1wm!d1#V?o1(-+Se{PcP)D?l8) zLo_D|XYwnYkkUT@qfpNtOBFf={yZ~Jq3f^bVn8?QHhL?Ri-f3Q4KK9WQ^)U1f2iBq*8MgW|J$-~mMi1@VmE z4cV)k@r_n$)~!npgxz%dWHak`gmfnXVx!LoWr<@3AkC~LxldGoNi*3BZoFYgx)T7` ziv;g<9Qh?svqNy*PpB|oHMtSF!|1Pul%b*d{b>4!UGRX2NSftevs(pYaoF2dK~j+u zkRA^#<173YNQ5%1)eqn1m2rnaGJcDP*}vOTSqV+(ib+)#!1Z^FCMJI`k)#?lur~Rf zKS>1{ZH3tHh9>?p(wmRL19$HZJp5}=R&d)YpDP=s_NOjtGL{QAN&793S43F-TgRA< zN+7^#4>P>t@3u&LAo;2Gx;4`HtvT82q(C*QY08{QOKU(4gEBShO%(1 zJaI&IA|h~xzZ=refw5Q}8%q~Dr3$N+D%bT}vWveV%dHonP}Kqg+_J`r{MVqQ;Kp~4 z|7W@XXSx4pcmL1s{y$^PqYr+A^hP(VxB7a4fS$5ga@!FtyV9)*r|?HdY7gCkBt%6} zM4bJitRC|U!UxhHt`~Ti^$#@PAvrYh!Scgq+XcvyGC8*UHJPOgZ1bQPui@XG zOIE7T@iUGZ%`L_;Eh2~8HDu8N*Zfzs)oV8y-w>MPLR&kT=Q+LdjP#oNpY`mhyh5CQ zq2%W(zm|4Lmz5QE+=~DHb)e>oe@8nc%x5h2En^aPkCepN&dpBw-`gvCSk+Qkc56Im zqGH3>J1I++x6jZ0*pqlI_}PY_hbNMGUmx2&zP!_v`=t|m*81QYQD^Sk9OBzgHE;61 zet4oa^93?78vq5S=&Q8Ixkt1dM?J8NJ_Qq{p|OW2_pFdJB()p}V6RZ#J#%vGb)9?6 zV9%|DxJt&@=cJG4$R%E}(_`z;JC^1_*4!u?k}kdeZT+{tBShr{*M)dU(UCmzf=AUU z-p|R7yM9;wJ2bBcKYKhs5qE8JxTB@OuD>7Br;eu=Tr}dQ$0i34>e)i7zoL9YdX4IK zlbPFykM~+8&4?Rv;8*A0&aWWzqjwpu%yr+H;FhPiv*JqHq0jFho%SENw=eSjLsfi4 ze^NSjuTI#@YmsxaQ7^8Czq$n9l#}6Itik7 zoU@lrm{syiehS&EdyD9Y%dQs=-!DKDH^+RwG$OAkID0o^X=B{|V+OIEmv7D520yPT ze=cj_yZ7t+N0!-QEg4pUdpLe$SzD%$&GP{tiSkBbR|OM})0#~lD*1Xl*R;-Uyv$pd zI5(AVyEW5%lRn@(Hyb~EIad^Z_>$+#Ytu^6gEc=_6U+kUzFD>fNlb`81?np?4_(Toaelq9$kp;Ktk+vj&Xdsckpe zAB28vjUGEDy}^~#R0Fz=U_7!WoQR1YKp5VSGyPU}t|eqDTws|%;2lsS7}k`&dP7?+ z^~_)f6Q<+3??-En+T;V{aiJd#GpbK(zniU*AADU#QR9wYI4@O&oN@&ukz+N`pt^@QeAsXzG5Ov&K=I_>LJ9%|pHU6}td#7^G)*-Q-8ZtnCh+0-&@ zCU?Dy5K7_z!OoUP8O?z`x9bh3NTz+uLzQzSo1-2_X6_w5rP`!@;KuiNMH+<$X6%E~ z>(_BYb~+E%bNIgXy90kf-kCC(@!jWE86&MxUA?b0Vx;W|5mxI=f_XRPywXeWxKEh% zUe~nk92BNKCubwH`@FLvUVW$VK4rjic+Y#^+zhMo=;v_$PQ#1oBH|yPU6i4-Tg6Sk zKD4~+yy~g2+E7NCf$8_Rt5ye_*zdp3s&!tqR9)$s)ea-uZB9e+pI_k1=SxSguUn5X4-!|h zCW8Cs!K!N_WMn!Z16qoI#i)4{!{YiE)uw0p|(aO(bG=vV<0?vAYTJ8m!8KPO8g$-^r2;Xk{kl{O+;hYoL zlEmK)&g=2bsF2rQ1`o4?L<9|A)$VG^-CMn?v2*$8?Wz&Zt~bo5H_|>2WynQ8{Ub{b zv!T}7*Y^hp8~(~8CNP<^Fx|^wMyQ2L#;4Z^ng;4BOEvWlFE+%%NAq2p%4EKgY*|*_ zX^t^&rZsS=^Jun*$^O zr+3oA68{Q`i#WSAs#42MDUb79Z>+g>V6?BkZUEc-Qps$O;0cbrP{8DXG~-C}Y^`y6 zP1c^S3&wjO$&+P9^lBGx=ODXiI2LaW{^4Rok&&m?!EaDsnEG_3HDIX8 zQ~Ri8v%KEyJ?fd{OSo#&CYEH^)*qL);xh9i2L}92E+&iA^A9aK9&)$iN@d8Z z_eiiex|pORHqh=??bE8ZPUue-`AW}b^JYR`)iP&CS~G`Uh>e---BF@4 zCK*0BfGwCV(>S7A}UcVF9}VryXH%s?9A-4X_#0VaRC-~+6dH@O&hQXu0H&U=1|YRkcf z?5fl=ej8nRJsX^Yoi9>#+KTa4l@3a+UTS3PbeuCN8U!xp^F8UAmK6oeP%XKhf!4DF zbI&1qoF*Lp_p^@3E+^Qu%Iu6+6_Yn|m2IvQd~kfrMMib~aNahy@@`((8rI$A7oDgn zn>g|O<~otj5LP9IC8vJ`0Q#_E&rW5jzB0;b+_-k-77xu85V;C8(%68m3!5C5OG8Rm zDStrro)+6?j1H(I@pQXwFa;oPxTiPqE235wC9PcL8^6nm(WstsHFI2Shs-+t7bsd%xj(5$s1Pv@fEy}P|!N@j{_ zOIN^P;qgi*NfQp+21wk)GGQGZh*1mWu{w~plverhT_#GL zqKzEYFWisiHH6)#M6%d7Ir8SaN1v|05&!)Pt?T%X34+&HYRa3_1N7EIesN~4KQG^& z|3kv}94Y5stH$=kO(DEbVyb`hnRbufZr^UBvSu*j@-Q(|p+H5M*F%Hg^Ge{ObuE*| z3A5MoR;hfE4ea?Kz#xK?IUhjU=vpR8|Jj<}eR=0jP5WaivM5?MWI>}9vdO9WHP=F4 zIH#_VJyqaYb3I}(L*|)~H+87`29vjdq$x3OPJcy8_0kRxnvM!xTEpP&6?&ZUJlwLe z5EHT{?Fgb5PGipwiHtf47VGWY3gyPzdHVgsn|d&6Dt)^OptRUo`1$2^JG)hcETwXd z<@=Vce!S&mm7uVd(hoDoA|r>1{yxY!w%0^Utr`a?do(ZMP2xg5WIxt1;yx@>ShRYB zGyh;5J#_k#Qs%O&cF5nlLQKWE-YL=TRgbB`0WEI0|m)swP}+*432 z)#a=hJy2C7+;?g}>&Mp*tLR1eu`9{^wH)AvKy(F z9%8k9u^CYxyFWZoAcrpIH%G^}Mu_Z{_iNhtvg!wM7b8R~v(yhofMFcUe?fHNb!v{j znTG+bi+p`s)h71c%(4<4lf_R3)#Tf!O$EyQQnOR54TB!H9y9PW^c#CB$Dm!KEK8X@ z+~{S2>BFtj^S-BcQ@ zJ^1H#&BvLWOSWoUk&r{>mob_eqR5Q<94$pv+9kfEdi4oQ>gI=)uN7=bNcrRNA?H<< zR>Ja98P5;a1U(_-ZSq{F<8_VX#B)4uav?II|IT8aPP5lDPzjC-F&@(r``}o*wO%b> z65W@%;-LSYbhqXokO4jZ!)@c;tw~uS7yse{jDeQR4!o18Yim5^Q#uXB?vJeXPKLjY=Or%I2v?y=@)cC>e5`ix^f=R%I=c=fTiqNt%s;DOtBkmBRE>QI|XaA>>k zdYN6YwUA}-<|^GUdv6%!xZzPlGy9~DKnl8C$-YUfc$ItzuVn4&kImA{`AA-~o2-=Z zm6Fi6xNA?phV_=%uMZhb6Zg%U56PAny-kd3e&gP?2EF-51hOfeh=(7zX9j+Wf41n|n-1a+0vE!=UN* z1MWF~eVhdRUV+Op#B$@&m;8tCRymreB2~NcWl-4Nq@3_I$6F9H!8x&NVPa^lRy7HD zx+Sxa5)|X&Guu>>J@#Y6o%j(3;Q^Ei>A+-0s?)8sF6phr3^~l>E^)c7JE4~T(jMm0mZ9;Dp(2|hCE_B! z_++3$8sogMnA{m`L(ii!`vO#~i#~I(ahIuZ6(JtiWbEr|qQ3%m{K34XO~#YED7>L7 zkw3Qfl|5=Qt-eStUq)WBur^B`OSZEU7);;~HsS6iCALC@R%U$aRor+zbDF;!-0{@O zj#~*of=g?K=W~efd&vc6<+6&|36p7;dgyAKQcmA?(OWPP;UM}6wO^CsISM&xdV zO3j;kPe`olsZSdW8yKASd=An3lHx%{Zl!i#&eg~x9OBlED`Ac z>1&ts39_QMz9+k;8RZ5Hyb#HaHjh;8&^B!=#9UNLU4c;wClrV(xr&Fil7`F!p$r)* zZi41RW)_?$H!hR(aH!}URBd-0m~e1=e|^G1e6*8)U!+m~apu^~XBmTG7@vi&0KHKz zt?$bbIXI|@-w3!{CUYCtDqt|kF;az!EEhe_k?;z&MkjJhW zbJ`m_aXsX@E$l$IOHAE9rRDhBQ{CI97L%VQsx80e(;%!vJjvO1`!i?B7f2pD>A{{^ zpi+;Vd1`bpZ^cKCWS-#|ub1o2(t!BEbL_=c;?;KTfu)8f5BG2Rl9M359~*pyAE7QJ zzinZ_noNICOvxe48ObT#0shgA?nq1CBZ=vD;&R$^=2k0H03~O$4cncwqguLD`8|&( za)?Tzx1wZ5{9;zqUaIzSBIJzMc)mWyuP&@ioIMYAw)?l8EneMZ97!c_K0Ns8OrVEp zp2vZhU7BT6go35jv&$2*@hXtiU!`}Au1RbS7LnV#R`1HaJ?%@DuX&1X(Cp6kY3XHy z4itauW!(NS{%8W7TYAR!sO_73>0284cUUJXUd0YFFGJ{sVUBxe4SqV+1M`luQ_tcXU@Y{r|&?gHgxv6DEwyI>;dDqfDxt%-4kp;oWlNpJ=itp!I@v;H$UG! zH`|th?F*c1RE%7bUSlaRkn95qo3s^TumL)IkdEl5mZA+m+S%}u@L|W`)iv_>>xB)!Wy!*yYtw#BK8RcmLMbm1Wp0|lW1iR_5@_1_V#xVYdKwj|X zYXcub#vBVD=2?y3`cV=7vJ|0ibqiJ#?seO8HorPV0CidS&mzqei0K&(g4{o7uS zpA@G@@W;NvPhOI*fu!|u+cj~;&1ZfjTJTi+B_|(F0~hMTAz+*IoKXI8#3<~g;U80# z%pt0zbEFws)(x{&PP}ruJ;1b8J1aiN^0y|=^AK(n?B?@~ zTg$Ei@K|OT^ZubZb`HCt&oWK?DCIPvtt6&zJ+Qg*fp8T_Rh0j>0Ejn|=pT*Cz21NH zI#6kHp>*)!Vu+SQq55$9(IZa$=8LQ}3FQjL^u2Ah%^C7qQy-sWuRVaF4E^)kEpO_L zZ3aY0_Q&vOg$#ptuG*u@H*%ui{OHHMUMHS5lA7Zd=I4N)rx^6+R=q8zojm(({PJYR zM{}O)RCe5UnZ(qZuyW6<1Wm`|mkOf4u59Hv7UL-0<=f6*xdp_qEfRW5NJlgs8bd%P z1{+TPlk)uc0gQJRVMDm0eU+;@LvBC&t=0P8t6h8g7QmNWgs=u!==C4K-&)$=d3?*Z zo%Dl|QoPwUVtT910u)|MCUL%vKC`P1fBna>9^m+^56NGCC6=J(QEQ$5RQ}Grl8-f| zBds}>e!ISITVjkG!0)j=x5j0n#|#w8LQx0a5F`d4h~b z(Dk3ZA9AQllf;&=r34qb)uYY-xUo_&H9Urfa60c2h59pfX-xn$fXRi;Lt0 zNSWjL!Eay^)$Q^&J-)%BmzRrQGs~0rv>J=`z1D1ei*vdehHxMA5{eZ5fEs&w3k(p& zVxTki?eD5LaG_8TVcf)V#)z2apwUS0wSH-D(xk z9_clH7Aub=1DPNN|LEZF+z1hX)S=B|Py-3tF%1=$GoWo=v6Fpq<4bB*Aw(Mu_loUz zX9h-O^t|oa?yG2*fIze1_06a!H$MT{@?%4ewyJq`9jNk#L4*h?x=OR`=Vi!l3$YB{ z^W>7ldbeiQY4;&8f4IZ`G^47HWe;B5N~p`-={B99d)Tj;Cw+2dO7 zh4B#S10GxXDlCvp2!S_7KuU4G!)J2-&Mq;X-FG$riTi?SGM1DKRo#zuA1ptLeG`9m zJ=5Zy+*vhkHS-TkXK(YH2>C}2r-m*pj&bmocO89ZqY+>1mVVbCraUN^U9A)Y^a1RP@1`g6Y z5&i%224gC(Ug*XB8^FkZegAV$z97g+QBp`eSxAArXmATKXVbDC_cqi;4yQwi(B6CG z?bs%*eiHu{x$c_D1TTpN_m`|%p1D_uq!KGILydlUEeI9+H4qQ~zAEzVr6%;j&ZO<> zxNrl3Ocj}f+CauS48X{{Wva;&P*)s~>1f=3D(An`f_EtU6y+7^K zjuZ!e2(W(^OD%N-=;`>%ES1G}5Txlebv%GJeXZc4+C$!|SBs+Y8WfRN|`!ZQ;X00}jq z2Un`^S`UkJmh%EHn*BF^7ZU^5E;h8>jZ22XE~mUz08$lnC-^_z{hcsl9{5a;y?A^* z#2qUBwFp>+J8)1&&lXBc8ZPJTCjq~{$%<5nx(}lE%Pf{*tAbRQ?`c+zI|x$U#%DhX z4d^V?00(viQmi&8ib*QW{usZl2m11S0XZH~bgr@LB`jE&TDUJ7Gq18VXm(_Zj}K=UUUg}>4U(ncPXvupumd^`H=;_S%#%cPujvuFj= zz=tYOxleoxFMskzuh81o0p0xCwZKRBzU$*HPI6c)v&}dFyi6S82#I>w!tmF7M;{kj zy#HGW-;9evY=`K#p1A2E@XPc4E7Fj4Crhlv0QFjJHo0K8oAqND#ymkTV^_Fr* zq1wY8`S}p6bpYD>_HEk{gh25E@tGCdJ(sQCaAx5XFw-|d0ekE0d$kvk8LFJ_gQ&H@ zIHqFR){0(0vOIa5s&?#|xSun84uj2xNqI2mno-H)4u^C@)iQ2$%mE_ZF;wVo&w4^_N~I; zj;jF>^4AUumsc88cUBPEo~~gnDkfmVC$`TGagU{SZ&7) zP6t#|G!LWcEhP6YiFz}Q5}tt$}&7F*`;Bntw2Lv%HxcsX6$>mxnK+ zR%~DY><_mmrRK9aONcu|AAqNbj*iD|rUS**SWR4EUl#1B1cmsb^GX}~$}Xm=z%huD zNyt_@v`XWpq#3y>Z##?Jff4hEg#D@TI>wt@C^$@^gRvp;_;=K8T{4xnY=8jK?MCzT zJ~)%|{Eb>&<|d=Pbr3-pY7jGGAhK4E`TWXjbP~#fwqVY$Ned5e{7>T$U)g*aVU~RR zNGO>?cbS})Fsj&j+gtf~V`<_~e#_%CHmejrh1I-17u9>2H<=#)<5P{7^WkicT9H`6 zJe2Idn(UvIXYfv(P)zs2So6c@r<2FItHmF+0MzyE;H4@>>7OsJ(zCYeADGO6ydf}Q2Fh6%>&i~}blihfFzAISY9jPAu#cmO#W>qIQT-txNO+eB zc?@-8`i%w3R=3Q*YE3zvrw}!~GoHV%d-v5F`C!lLtA<+rfLW_(+|82`1-^?cLwCjP zm9^2I3gZDv{F69vs~I><-~wzMN#jx0hW zO=sJK&7}#v{h*$`9p1f_vi}($z$^FbIBo-L0U?t?9_v@6dgYu;?+wX>A~b^cvd!k{$^bI0m(Nk}XAo!*MZ@qMEzs z0VEg(ADVI>3+Gjmt_#gXoV_0Zq7I7$TPukV?lmE6xTI@*a7#${CdA7X8^=7@+zKO_ zw_7w=4=M8n&#RgR5cH{o26vwQRTJMPvnc)BrWe&=MfZ#GPk!ItpDG`o4K+(F^1oYD4`H&(Vav8eSv`G^m^2{>(9{nAN4@?JUSidL^q!` z9bf;ibyU$}k%aq|=H8oAO~;22O?5~5_(x6s-N(O^WdXlB+`MW;9nrTX7XJ`{MsW)G z0nh=Aq2OPtI&(9eD$LyuAihff?$DmwaV?pO2nu3KE@fpJEDqrLttifCAx2_jL$i4t za^B)*pfhiMUw66g^!vWgvO#fscD&Y3dsvE#Vh8K;2!ubwt5WAifsWe20s7hy7583RSq451LzkP_yaxQu2{KtNG8H zuBlgka1QZ}dxXVkP(|z3Su?C*Sv|i5hK?f4>cJE6c1iC4# zz{tN&c+&EJe^39n=J!A8z71bowg2cRA%28L2(SD{@v&!UJBR!0+II7Rf&6LsePnnA zc*1I3`~SQS4{#+J5IH|6HPw@hFg4=gM(W0IvF4VukMW~hM!|?9N$)4bSmv8vCJO+@ z9KO;0noPN)Pi+B^1Y1DEb2RVFSI z(%w#t`9JNR^}oMA_pb~YjE?eqC)oeX%_Ti9DZr^cjDfwNH?5G5nA&y!yz1!ReFXd& zavTCqu|hC0EYEKlE}g}57C$$xT_9fz>UX8U)$@z9eOaBz3{)eV-w z;%|O_88ynM!`~Fc?#tS8c}xkq_|}-NHl>Z&yX2`l&QKE`&Lh5C&c3wGcMOjl-04Ub zx(8!Qqc^IN?l{8(A19o7JC6cv-FhwP$)!CCdyap9*S^-wXaGXQRVr7Ocfs9pTN{f{ zJOPa>KPdTssrCO!k^aA7L|XuZr*|D+O}jPM;}JaAQr3&5r`IHmas9tR>|CcsRk%Pz zCcAu7Q_)#XWHA&jct3Zs6bU(AcXRb% z48Qpx3o_9TnfD$wjXkWTi-E@m)aWzYYQhf#o8|jJnsUnN-2mdAg_6(g!t={J=_WRJ zB4L?2R(?Uind%TRCs5Mp@-3#`&p!-q;ZD-(SH8Z%Ixnr~z1{L@l^^jt)K~q1$8OFi z1ieHEdjJRL0AHyc4sxrz{oy(}Gn8cqC(w~I?=KzUaQ<81#dep5y}7jmSORMD*zjD$ z_CWG+)?(kX$f;4cIo4NqI5|Eio2hIVm<sQ=Sz2cd0h(}XgND{ z5%0oFSRGF8J^+TkT6cr=c+eHbo2}g8qwc>nt9|BA%_7T;L(zZ)6z<{Ab>q~5ImqWV zfc4-Ni=2gDJr3uBZJ_YDM0T=>am%yNC~N~d;C^|fo1CF^4~aQuLba%ocRgOcfP z-uBuE^OrRY<*}~45tYl+z`7e7>MP_W{%PGY|E1@+9Xv3*>mllY7Wx%(Mz_qkZ=ZE$ zzQ|#M%Q!=XssG(H7chR+|J#Y~_U%fL*iA=7;4v|voJI0lv{T{rQ`Ap}w>ZjM<2^!5 z>R(M{B>=IVonFQl?}TK@{gQ{la|hGi&BpA^ilRelB4fh*c~~(Gzve@QF9-!BIOcH{ zeodfoIF*B(#6}od7KcQ1A3-HaS(-tLP7_!B+69Nu%clk)`jQ$`5jr^wr!T&aul>1W zPgJ3nqf!BualGC5B^+e9Zog!BWB=;gC*z0VaQxSa4FgYqh44sHz+GI+{%{z6o51!e zL60|)(y#0ps?`H&9x(1J6*jX;w`NB#o;X$%<3I9s?{~E$sfj&&H*BN4#8Gq2Xkb%e z%3@we<+Fr~TS>>c?_@izZp{oHEVQu-+cfy?v7O$9{!QC!ogjDlFvFb*N(z@8hgygY zsK?n|cT-Xd8M)~IrD$sRwK@UIQZ|%?wyF($bjqCn87kxYmWq;$pEAQ6J&xf1Dbv>7 zr*)iTK`z;R_UxwG#U3{z&$?cuIwT5zXrap_RB@3+uC8kM{&pmx3O_NiOS*c6UCHL_ z%ch|`bWLfM1Zp6}pLT=y3Q4Y~{s?+(ZANOYbsO>ynG|a_aI^wE$qA ze#L&`5G%N3J&=y)9h#qZCGdeNRD{5AB$0JrgKo&MUo$qE)HBf*t>86F!*Nk-J{) z25Y2S-ah`K3vfbp_L+ZQRuqFo5%Sv}+}D68;+^oa6tRJ=F`u^=XR3xlXY>Oc@OK|b6GyB zdxpB?zvuR|VzP2ui8IwpB(xI$AE)&>hP%G}^^+Snz?t>_G&@-x2j^u9nHHAa(hE3p zp<&z4gZN4!?PH~%K(wuy-4V@*4|M=cui z_IZqG8=DIs@B9Ix>&r5rzmEVj{Y6lSV%cpxp}E=AzfRE5@@ilB{b-SQD+{Lrs(XKS zr59|8wdQrwQoNXBT^XBj!EF@x`6ij2fuFENrmz-svM~={nj;aNkl;?$ECsyt)w8?X zqeTLRuk@9DiuS8E*3Wu)348T*cRcm}=+VJB#$k@;K!b#7E`W=!-q` zO65ms4Ua0V&$@V}e~f2bbn(5*Y)k@~$+gZblUaCp@|D2#*JYQ#Zfn@%-FoFldA0HG zVgXNv$=Z&W;Ug8k=`}3Bh8#Tba(nSr{iJv#)!Wg}U#)jW{&L8)U&CkJ9?AgNgy^ZS zIaAHy5>OLmPMPqwa?)#_tKI)zfKV8Pt*6Ze+8{7o^ZLdhl+AOHsMlI{XV>Pc0rQQy z#wHVY1aAtOL(S1%Rx=uvq;PtAl|&StJ71>Q@)IfKKB?YfIabvq|1L{W$w_JGX)0X= z`9TyNS+nUauLNVrC26aN1dg~}qfK(1|bl1JxkVFnGOWCs)n3$f7H4`s`cxA%d4p={VDB-Ph zTA;KgcCFXRj(olRH9B6c5Zt+Ke%(X8>|OpP&15J)CspKuQrwt90!6g%(ud|$dE%jZ zz^rEjT#cmki(syyr>4IV`PHVv3dvofGZY>+?ByA~{5uCX0+(ZCbam46D?>-SBm;R< zx`$Kk_4=a0C2qa|HFWj!xani?lPkhIr5{7^x4-$o^B%wXR=J@i>%zmeGcF)NF7opW zuS`~FzD@?M6klPBO0)VK0Mbw7>u#&Q6kuD`St9ciep%$}K8$2LNzO{~@aR#+%@ft8 zjbGqs;ZvE+y&+{wDgc7bljWWqAV^^rj+wTn_K@yuf-Q5EwE4U;=$r4|oSEFDH;wYc zvORwCQ=Q(y*=H;9NUOg#R&WSi0& zIM3zRK@bQ4Ot{GUF@Ek>v90mE&`-@i$y|W(ZO8lSx2$IJPKrw?tou>o+Q9qrw(T_I z>KEq*vDKs3;g?s6-d=M$oU=J97&-Kof9|rs3&NG$%4z#I&1%@@W&=~GEI#gnBDhhd z-7`K&G4Jq`4KC*n*%#v;OP|M#;M1C8nKch9S)I3P{18 zy^@pU3BnD?VgE1M-aDS^|Nk2=l%1WuclIWGWD{v1d&{OccFN8slqeagsEmvp$tEFG z_TiW*at<;gevjAt{rO(k^}TM__51zx`{(n=`=i%+jpukg?(1o)PVcwUd50G-e6lqu zB#lkU1y1y;6T6KlB_d<|wLWCu zp!Ut-2Sj+xXmqS;u6qHhnsakWawNMzD3n6rQc;2WPDu}8J+(iKI%I}XBoF?$JHK9u z@|RVE%5+EfGtyoG{jJt-`VjJ|42C7$Md|&56l)l16QKZBzHu|{d?xgvUHJO(L zeMy4f0W8Lx)8HaS{QP>7r1Zjii)5g#FM@#wZ67HF!~b0{zn(f&lkKL7h@w6*lgnV1 z{aqyZsDZT$`SP9Z77zS|2|oCX-U!{@VpKR!$|tyylt-CJl*A01svBCnq-UL|sql${ z+Q8rgcp#-}NiV)nFubzyin|{Wbxnm!Ev49iE_}V__J4i|v-kO(!`XjqR&&%KS%|^= z|FbA5pMMGhlpH|f``AOKX|50>Z3t)dp*c>&;XgKG^WSueeV}IoYA*|BT@#dwi-U#N zebM-C7h6_A`_?F1aF~cdk6{pOKOF4vwc~zY>aDshsW1*1jBXNs&mZ0lmZWK81=`-O z6y6;s80givIbbe==9kY8jgK#~1kz>o%kyjsHbt{saoclZ*Xt3-u;I5#tGnz7FZgF? zT?E3F%&_dr!Idfyx77A{r9bZ?tiyaDv5}L{&nV$#*Z}#T?a`l~5=u~66(RP&2eO`7 zH^CVA!z+4!DGw2XheC!_8L)(#0Mqn?;3s=+12Mirsr0;ZVK+uT>JYW047#*cU~-D3 z9*AKKx+Xt=c&dnkUhPhPF6>M5wZa&hlKB`KD2UZ^fE_!f3{@tYUdT*idFZjkp3w&2Cij6FBtJUr zOrk^d2C6HdTt^s1XTm=l!`bL zB5)*Q`-fmkLkz`GYCvr2i>sMcQ6Fxf zyko5FtHI+#zoBn{Z`6m$#mm1`v;dhZNep^F%wbGA$AEaOnat6nn}sJ5*0AR&q-X15 zqfHKmqPA&kP}y~T^6e$CFC&cJrvinG=Y!Xn(2~+vWPQE~0=;BL314rG)13%6wFzxV z#W-mA6i|gpbb>#Q5l0>3A$qWkbO+FL%CV+Z{sc9_EpegredvY}RY6$o?ITHFeJ9|P zWq+n#I{PT(&$cHLvW9e<=ME9+5W=K#gf7wy1Tu48Gf?UsJYXpD7aeY(2)nh zV3SYr1_QQXo_N))D5Q;PB4K9(`AEgPiBx%yk?<%xIt861xkT0Xwu!Mpjh3A1Q8p{P zh+#DKbj@#2M9}BZVBK5*eGG&TVVKULisVIwEu4y)LqSl0&?sy${P!)Wf37Pd-Yj;db<6EGdh|75@%Xb+~Xv<5O z7T}nI`0L!-%moO!b(e2-Cel2tZe1w~znqMJ2~dVL%N^rgBZ-#M2Y!j`(6tj;O(CIe zuCxzs9V(5hg2D3~2BOPGouMneC>9!HjAZbgD;Jv3JO(EHk`)r7mb*Du5C?AQp95{c zw#&+-IZK|&iGLzd+o}BAvZVZPaRSH4!yWgcP+cQ0&{RH~PiMW2AQ7OaetMklr*no= zt#Zkrv90}G+rvGm>dT;;`?GEUiu!FgimlGq9~5XCg_Cbq0mbuIzgmU{b^t%-|7FGP zCOB>N5Xf;f48J=glyyk6?^67Vb(8F*)>$@YW0wb+r{Z#Ww#d!(%IPOA%UoB)>z+OP zdr8Pu^RK_wKdi+78?g@4!d)(2nR^ANdu4T4D4o*Y)`F{LrCr->#4Nkm-KxiE^Hfe(2ukXcnb#Tj~YHK(mfb*IEH;Cu*g zj1ACklsf%Igs(Xc6iB12W|RCUv=i9!>4;PB8QP#Gy_)eh!G%*f*qF222U>$}XIOgA z6Tj(bI@sN~CN3%`wMEzd{@?VShqXh+C}m|ewn8e{iFv7{7tdZwHkM~G)B&@1J&#dR ziWrOZ+ulhnOK#_0N$qp6?lFp0X$YU}6s^;Cvw#Y*p&D0)<^F{2oj_9;LE4IUFtm$U-XoJ~m1f>^nv({dx1vi++O zJ4P9h*2nM8yvJ&sX4|5wvh9hfSek7Pd5x)j!fg>f_aR0xl4~nn4phe=5#X(YxQ+U> zDqB?$#qkXN%=!?GtWfr3y*f{S?I&RGnw`H|~r1lA1;)W-nq= z){%B!_M~#+lUE9#KXZO0&A2cCo6900nUXXu-uFvI=C>ke4tSvMSg-jQBk$BiRs#KC zke0~ayO(f*ds#?1CZD*b7Td|HT=1km68wg31*C}r(v!vWsjM_P^>dtrU+&%IcHz8} zM$FW9>g{5NjsCe#kgWGUCGNg^{8W}mLEFs*-OVaQ={O45boShzZ|;g_w(k?M^H5*y-vzRMJ+E z^2l%_$5Es7$Jp!F&G}E;s!{KF+Ms2wTIA*4fk>f;$RwV09YT}-&d zW?QE+JaatJ0=F3n_l;EUZ$eudo+RC56S_?lzD)i>w~4h16#QOU{mkUmW1;m^aY&-2 z$a$qPf*&|>BYz?9;oM5hrwy;yE#;yF^Qp895#%SZJ&lN2Rh4sz4%VUPN02|1K@EuI>mw7Xc}ZtT?} ziE~Tr3TMV2oN_g)D9~djf6tIuSKY0A>5_^#py04KHLWQASC0mZr=h@UNdKD$^?Y8HNPpHlxd>Sm|3qT?fQ7qT=FXf#Up0<}GBuwkGtZ~~ zjJqe-Jqs1OLofA4+g;+HCKStY(k%X+sb_p`a6xM1f%ErQb2h@i_c3<0-L-rPe%PTr)=X^WL7;r=lkAo`=C#OEJ{|AoXr2A8txD_ot- zR^Ggn&G)AR`^BuAQ;9k7a z`hXAa)Lx_#&Sa^lZnOl*FotR8(JLf3WM1iV=0%diVzZjlivMRGTno)*UU2Sj&`y5s z^rr2#ETJcwrGD~@o!MVy`~lkosuY??Z8ZMuXN_7d^nja_Yc#!|$m*Y3XjOLSP4PQ+ z#!9?-NZ^>LLlVVU64+8nv5p2grCd{~VR(<(Pt4Ji<3p_c{Nt}O=l@2-+rsRLDtki7 z3l@S$U?CX5_S)%lLDb$x#jw&39l_*~%9BzHkp&Maij>ZVwigmJ`q!{m-VrlBc5# zdL`lik6Ze;8LCJx6+Eqt-+?{J7`l*_{6JlCSp0&mKIG#~WgQ|Kv#73xEsU-`vihfY zuGQ7T%irTW;EoZ3rJ;mq<;`e5kk19oehHC6s*-HU4a9et{J>(e(Ul_;$PA@-KS4Lr zdL+TEhcDEwl_J-nrwrm=}obH`L5*U??6phHM(=+6G%Zs?yM@{91Wl_Uo>;2jtL zM0&rwA5e;<`Oq-=$eV}%aYk;%4ZXg<7P|ie?&3n;|MpD(|6T%m=t5W%C#szjra>y^ zR#Nx271E$c?6`t}qu}ewRF96bY7VYi?9UtehPMvDL$UmEqSl=scEF_I=>N3pWjaZ; zS1H=Qsrdn!3nt;%t>S;BG_`c!eRT;@R>sfDR$!P*7G?JbagDH#_@)aTnjbpVd@5x#yUIutpA4P#`G3k`vyahoNonLH| z_e2dyP{-b6{mA8y5#4>FdDjK>jWm=^0E0mvW3dX9fKHF{-zVwnN0tlwP`?;LGF1B+ zMg`&f=dNS{>yx1Rjgr`Vvd#d+mK9riO{A=(I@%|q6H-_e#0Q^4td{=y1DF)G3mwDz zw{LTHu|B;78)@6vmhbCZh!anv6v-0+R_mTem+w@)2{Z{}y;0Y|((p9sQP^i#yybE$ zuJ;V+@IjHEe+(H5}l8G~dAR;gx$6;Vq#l0^E9PksMyc%KnC#(6x>1}$=rTA6k3Qa ztwNa)`{3lEDo}dH!oW8KzFTfO19=3&W|E5Z(X_aiDBekG#vInUE?$Va$^@Ly5Xvb& z@bsnGPyof5XCM9d%Fj_e6;I7&IsxMhXrA@t?6&W+tCv9O;Zg315?9W>JVB4 zU-AMc--W{S%d-&hT(O+~Nj;X`CSc3F%#Z;|fDl?wzfZOVZV&fQX z!WWpx?a;c5L~nRU0FGu>0U;>GotZ#rwRpkw;~}ZiM~ z9*KW}BrV(0TRn{-`0*va<=qCr*P4arZ)_JvWIHf+0HtUULvU*Coyuz?0vftT@PeOO z3as{~->@86eyh)Q61rn?axi5`(A-9;Z}##^F|stsq&&u(f*pOX?6Q!Ol=a7@;Iq|C zt+qdN=GpIk-(=y-AOd+)9x+I*@<_({WqsWfS^QMC0-<-T@TX!wFT0mfHnp9enH8RCOCU(Ah1AbFqnB!jZ}=2!`WNmKn!1U)UM ze&baLAv~m<%G@#Ek$u3uM+U@)kMxk=;~-7z?IT*Veg=E$YqGQfM%ZNElgJ1f5WMH6 zJ|~-Z)U;L*`m9Ont>5yGxTH(yW5zD}F}(aKd0HGP>^m0;20XKFUnJoZz&tzJ)4o({ zER6K)OMIZ>5nLT98)er@kzM0WZ~1{=)-M5TZ%%$o53+0Sywb3nk%vc``n=T+VzIN- zi4vH{5jA#pkM^Ec+P9j%zFO{W75ZoO8FY{D{=n+Fz56dM!1juan{nO)8Mj{0MQ`|^3$Zm^J!y}ro1$z37aMJ(4p)y206V5_`Lb&9jOY9>(B5SUJD=X) z9vxc5O1sjo`)!q*Y>qppf7}o}rq)LR_~Y^$g8JV_oI-@*hwgj+`vCX2j4q2An6_>c zmbrYMb1wt-gWj)%ylZ&@DMTY)4YtA%tm>ni5&jNR$kpy}=>q*CD`JWY`ixYTb1nA1F4 z$nVK3tH+RKKH>h@2u$~`+xb#~uwNQ%xtW}@`i>~_*OY?F!W&*~e%oxg^5YnAx#zYo z%h9_SGT1_+&CrZ3I}dNJZZ={_h(0>twwC&5A#kv0RB9m~kUK9NaHv3df*gF48ug)cBX;?!UY>b>BwSVc8>LSEzx9LC+dw1#;axVdQH zIrggM@~cZ1;G6Wr$>Bn*YL1Yw)T zH#b8M16IE@)k4~XF^7+t{bZew$V~ICyhloOL*uO29%pCJo0;I#1`HRjupw!`jT!zL zFb(>+s9U=a8fHy<5&4J_cx(VzEK}WG-6~CCRks@~Qlfhn>jD5ELA7ZnPaL}S#{HXX z!IKwGNywS1am@y3?ZZ0E)OKpWCX=pK|4c1fX^`?aG~9g&ZGrtKa5IMvE9** z@}*ba6N>Yx#B7i;QTF>QOx61UO5^S4TY%^D1fyzeyxOfFA|tON4|KW?(z6AZ(v$;_ zL>5vvY1X2UgDckQXw}@dg{;7&p8NiC)ujs%*uk_CMjiAe0o0|In$@e>|Ce`qP8M+5 zEO1Ml4xu8wS)gujw>tIuah>N3C=@FGzJI3g2xg2TS>F*IS4JmR^@SGqAzv}&{rz4V zc#iDgq#6Cq#X+$LzVBAyZrwN3`W~ydA`XV z>gv?4!E?*vrz*-WiAPNQA`onp2c6gB`NEW2PP@+$wr0jFW2^zufePTzqOfQ1Bg1^E zaMqyjz_7QI(LM*dZcn$+^~vaJBVj^X;8+YFQ@@|+pH$=fE1e`jQRgbey zZLJ&|q4Dm3^^M{$S6Ujpks&iQ&`o11wp!1#i-7apu2$OcgV`XbNHy#`2llmCnC(ZE z!;G814ja{kKCTmqS()a--cydcUf(!gT??B_AvP>#m zeb7x|d;VARv){)YDa$d)R1sqEsJmKqy`S|S<4e>rYVVE89}hx?pD(Yy{%CRW_h=-V zkh1wHMdVTFj29d8ud@RM{m&lze&%#;xX*7in*&+)eOuZIAIT+B^1A#)Gp~wWXCmw> z@d4c+hOMpD0F|t6EQ;;FHLerN3ULsoDn!Acmo zsHh}F`z9O4n&}^Oa4Br^fV+&5rEf8vRKK?iIB{w8@loKEPlEv@s+Iy-Z><2I-B3@L z*iFM!=WlJdqD2BNt^|doa>#VCpTFtvaXE$33m|rBu!uz_5yR~j!UHUX{CdHM zW5EE9{Qpd5SLBsqcOfwy>Kt3Kx zL85|ZA`@RjN?$ z|D`0RPU<5nS3TF!CT?qhdCGr5`o0QdT}q4I1X^M==F~ z8SNYv``AI16q82Hu3dPPS7A$Z_^YO6zE6vwwoSvbqtJM1 zI@8ThGDEShxmE(*@k4TBHuxMEnw{+`Ilg*GVu+Mz{kRi;qgTIuR;+RE4wl@0PwVg) z&ZOe@uIUqy>SlFJ%UC}4Z2i0L*Ae!!it+diHgZsDWt$`dJrBsw#YPx*^q8Zk{GW+q zo)V?r<&KdhN`Cl^Y(utWv6{B`_U(4%*xJGKB5SonYEn5R)o%u0HUDC9d+1rt6rPIX zkaE&V`HBdkOB!`6B%0T10xsipUN_DsJ3=Rr8m))kYbTeO=0$@>a!{x|Tm6|g!U(ch zFwQg2LS^a9rbJn65}`EDf;B;h&|0^v2F#SCoj1f5wr#W-U45)!ey9A;Vjb5-f(ru{ zaUn$|-uATL@4`KGGWoP?z*cm2p=Jfb=)O1Ux*$`;F^=KtUJmZ`0*nKeWtap0ABCTF z`e&8pTA!PS)0U)M$|SG@x2TspZ}2|VOJiexX^er}km_8@Ydv&b3;O=K*!&5Qzspi3 zWbFmgyKt0@SH7g9)~C8VT=uuVUSUAq)RUemB{OpZvW<(Uf5{xP2(@0jKs+&(t6@(> zk2x-H*EMi@ZjZvoaTt4t8P{aa;oS)N?>yK6YkVxF!n9QcNldBCWGdy=A3ap)v$>KcM`fuJL%-r1|CWiqr;DCKu zIKyrBMMdsQsq(_g6~52%C*wk(WU>6GBT1{(=@ zJz*)&+M78RDi{jsgbY95j9RgCl%x^5QFIPXJSsZ+?GoI4UZCZr*_?J|Iq8cP!tiq@ zCIL6D>i## zr9wK)1Q)lN0@g(7H+<`cRRJA&tt`huL+FFz?g=6TrOO!8yLN|)dQ z=|kxnT+v6PCl#SDaN-+}aD%5wW8RVaoilh3AzIZL1C9yo@~IyNwNRWP(*f0+{K6-m z!DrEOOKWeYUJJI?LM}o36qjpg{O=|3rMPdWiIGc?4V%puMogB@mhdcr{@VGYx^+XG zLYzL;l-KJl#snU; zH+u=Sn01lT^Lw028<-k;pkcms8C$reF;z$uMpsdQ>weFlYsmvrHmQf_U3xP(PDXv! zB<`A*ouQB`UOb?R-s`aZ^Y@Fz#nd?8Goa=adf)oz>uX(Xs-*hPxU=)`e*8jDg6hsV zufN1SiT9{6fzto`FWNSddV}x>3n}-ayCzo;f8kUtD#{C_Fgn-Ct5)E*vl>MRL}A=l4eh*jtcIocnJ4*ol7C&?aDlCE|hlmh}Y1{^bT z$m@K?JQsKRRLs-m=k4KU#uBOC_fM3yafYu|OG4Aq5N6oZbAM)GCN1W8|MBsJ#UFO5 zKk3njyz!6bl609uMi&PD%8_jhMIUWpvy8+^3%qL#(7$Job{ZU8et#o(QPo8GHLuq@ zf@j~ar!s12cwy#NsgLcO6p6R>6wh%6@cU%ETi?jJV8n|p$o!Bwo~QK8);I~%tx&sl zt){;nQp_2IR*Fb|b(`eQq~U*wp~tM;3*Ue-xf|s>skHKl?iCP-{!kuYz{06_iEluh zeaqM5wTKY9IjydWJQrW~Eti`??K#XEM;A=kNo2IUM(7Ho$@6aq@B6`cA9UBEu`*#% z#kfBT&~>=G0;ncO{26wZmKi{sLlx(46T;n4`K`V->&~-YExxsX`W~&Ut$auS*GDii z>QaV;;z9-l!ygP-phDEnUs%NJBGC#;-9g7g$A=eF@KJ!2l}LFn^9{_YY<;4OQ`mnG zYvDvwaTvOJ@C_@Y=z>udYw(p-7I+)yq&DsD3I6*w46(L{5o~oKPjG`#-)l+aRwKt^ zBj3bi?`=v%@AqS69`743$1~r^Qku^_F9g>pV|_iW3D;oZ{*uE+0+Y0VFXa+yMH0LA z6nV@wcU_Kl<#3Vmbxo`-(QnTK>BuALzfLMz`Mn}1HWaaJ?b@;cGSy-QyG8Dx(y z=RA2XDec#C0#>R~X|)o1%qe4y=0gQo(RO5t8L%eL_nF-vNgWidI#I75XA?FH zdjJ#j0GFVC$c+f(u#k96row3l=REA$9sRUFp0tp-P5A}wrG79m)E%-p9W=xamEC*G z=efIhm@CY=w!ss7?r=o8>hH^S>v4>Bi3M%;aPXza6rkf@omqz%ytVxtrl1nwI9gCY z;+@R&p!9~O(Xy_IBJ~etk`D;1*QzO1<9rZSe-9`x!g_vdvZ?pDI0>C8TK2bK3Q0hA zV$p`_zyRasnRiMTN2jkdKQ?b7*?_`59$AI}T9;5c#C2dH)dR>K0ZH|dxU<$mVd}zO zsTFRY-3xDhQ`~&~Rta8mr< z!2;(oQx{kJt+!vV-rJamlxp>{b6ud-if#mUqQvKW+66{i+4CzE+xL9Fu+Qx)!lJ1& z<*(v!y##{-h^WLz@$@iDlFvo8eU~~{5L;2v(@Z(srFaMfqb5kF+Va)xttUHIsUwRz=t7(`#u$@$}lSfhZ=m2a!saK?0e^>d%fel7U z?E}6v;`m^e3K0IFLsibO@5T78Ng=e-1{j>uPX(u`Z1%nxke~XeB(e*6Rv4UojB9_g z$^X~2=CM88t0M%LNl~fK2YkJ9EtR0T zAoYq5n=eThj8i!^&ict4FWnhw^pAMbGQf4q3v4j-1D12q?;s83JHeY#JOw@IJ@m@EFvgj&qC$U$0Sa%c+eNJ`*cXfxVMZ3f3tG$a+e%5^ww^r-pRVg`|&mvv)VV3 zuM<$gc0jfrdq?DG-PIM(hhFu8-Y~Om=`cBIYH-l%!e)O+>%q($IHR+^oo~!pylF8D zchCNiN4yGGG7g6^2CdHys+k}IECt?Mk1arF9P(@zevEAPOK^&#aqW9=OjTA0N&T)qC!ujBEx^?RanQ|B<7NWa zo%SqCkbeKgx@qG)yG+It$E74H7cIrm4g9dp&60UBt$la(1FP*u&cI&ofZrlM9W?%M zR7|(~X&$wo)D2gErH`Y$J7R91aguw(e)1^$@nPwTQi@OPPx_hh0^*=GW!RRV`c%bl zosqp*TT=zIEG6^?+7PnYrltYa15e*f?LpTW_v3F_5d!48~e zlb`-<+1_b{R1Duwt=yHfDpv*DO-WB2N^8dwS3EOLJ$_LIuXD)TsXRnv?T2lF!R-`FMAn{1EiCHGMl*Jikq|lBwwChyfNfx9P`Rm#%xp zZd9}MawvLtWSZCOsvJYS4HIZN89Qj8gB&{PXO8_#`m{byp>6+zo5ccDV+(svUs>}; z2|t*N=;CiLxeMpnWFQL`jsmR`gRhd+tT6NxqvDZ7F-YiM1il z(7+|K23xLcWh~J2^moK_Mb1Gxy`7T&YN*1l!qkiYrFWgmRlazS+*52j!5r?CQPL;TA3bmVv*n2 zLEWzEg+lJ&g#K-6hrW-wR7mOY=sE)im7b=wobM0CeV3Wk2suyb1ou#uu${NOFg{~R zQOnTM$>qWBq}S_I39U{UM`2VA{uO@qu!tJ3B0*w+Ez=?)lgdbKXB|2-c43isI*DIr zQNasF$BLr8^wZFsX9*sjMW1tE_h3+v5Ig z|DR>)Ki|odUUsdDi)aog`E<^KE8gCGE?xKhX^ZR)7KZkE5jbrAg(U;4BXh)MhwIL6 z3PmBM4(QeL0*G@(yI<9!nH}&pLxpgmLqj*EyE?8k=ssdrC*iQJUxQA#)?Fn7=Zhw6 zH_1Z3?t7fQbw3^O0?Tv!&&2j`7zig47qoVScg#|LX*}EHOzLW2ed$Ajx*+dKg2y&( zgmQ7a8nK)E;c+2qj}CtF)5yB;@o4u#pThEf=k05TXlM56liZA*6})wxil_VuGQ=BS zaMF{WdBoxpIZt@E`y=O$6DxPv6<_ZUg2XD;FG4I(4Z>(V*%l#}LWnJ^(RbAGkHrCr zpO(VtTFwWuFk71+)FXqWcBj!J*COd( zHS6DE%NkFppO6o)oA3I0we>FkJhJq(thE=IG03~s4Bat&TO}VC^0Kr>)f8JOHhc9| z+7rzW)nr|2_uk2BN$XE*c*{C^dT+ln>X9N<}#D7D0ii@GYPIG}u^`>-+A9p2%8$?)&m`XGyI=w#|I~JTwFMoM(m5 z!fHWdwjo`{TDNX=YHx8I{oROu?De)=g=v0{mz;$4>uBZ964&=`)PHFK0{jYh+En0HCtC5rr!!mLz3g-l5)d4kAA z7;br8=DMJVA1=lDTA(Z*q)B+nTb{}7?*7>3;zzUa0ymiO=f%rcEHVUNE|ktMmCuV_ z3G(Rzz=pB^F$Od^^Z6l8o((_Kb^`*ur&v5p?ZRflcA`Huxs&Y`*c$Sl(x45S#GQDyEjVnMme; zf)meGZ^_<15o1{X_?_IQR`zeg4bK^t`$`6_dZ#nq$tAzUn-4dR+&JHMQCI0cPM|v+ zO5m4MY!A9QvUX}ys&IqB6|miG`k>FV=HSAzm4l(g>3_W-zN0fvA#djW)Rg(myF{T~ zBVlxqo|;KQgqg*7TKXlYmY-kr*Nsluo}0-!Rce~*Ee83^Wb|iWUU%l$yy*N;CoZI* zUAa%ms=;7nibKT{5wEl>FT`9$xl=TbEhX`nHlBW|q`dY<8!4xqzxKgmB|gjQS*@TJ zfhWPrVLM{@Cy?T}yLR%UCVZenMJ{w-hUO>9>bJLUsJz52ZdIp7-tsiKZfDAnoRuxbXT!ZUMV0a*5g}$ekP%E`jl%Iwwiy*46C|cQBEcoI-exy z6tM(Nvw8o#G(GA3%ZJ3wN$ZqyeLGTs66C(VmZ$LzX1NvJ(eCdFV`GM%_beMC$t~=+ zzL*b(8(z@4TJvHWPSs&;nVvns?6!joid|csRH;o~y7EdBw=RMxH_k3!jQ(R`>5_S2 zzLhkfDvVV%?52hwg0c)hFTwo^*Rde=ixq0Vyq~#sWpqu`tgEl%?)pcO*A$6E-G?24`^ta2WA!HaVp52yjH>k4*KeoMW1|7XEd!MYJ?2oot8A~i zssVGJ4G!O|7P!DL36}YVgf6zAPc#=+?~sc#4Z;&~U{fh0E}J>AA2>*EeUiD>R%iL0 zd@9$DiYMs>;^1eN*ADTDmU=t&4ZfAu_a^U#iU%7hI4&c&%p6|J} zi_xAmjnlYVj7JHk=6kRif*B~RlB4%1u_i1DAr8K;cImQ(rKX`V?W_m57e)T&QYLK1m9^7uspaEHkRsg?lqqY>C7AYtRs^H3OHHTtcD~5Y$>r)r&&a0WYja5b z(Cbyxe1cX<{3X<`x)VF;#^Q)5F`m~UDdx3T%^gV+RA&DOZg8tu!_ON~-cyJc!Iz3pczaMG~ob~>>UblFf z0i#5+(JhVu0{O`sY0AG6?aL9kgGns{^~v~yuh$5n>kPzNu!NA-b{qiZLYDKz7Dx#~ zgBgP)eU`T|58K1JviN5SK7r|i^_lmdTm6K;D!OYMQOXsz0=UzgZ`NKHrmk9hmQ{Gw+=!)jW;Hds z$$sXl8efHBUt=a&wUq^GMHJ22%-dMRhAtIbNYFda_5Sldk1OFxBca6CkdmoD7U3kjcAF(RO4vLQ;u3$QsO;Le?b@pYGM*A1qPtG$Q#^k@oL2SeyP zNM3+}lE0gC%U7Vkn}A%DdiinmQR9jy0_pLPS6@sC&D!O}A3=@W86q9}+{z#djw)Mt zi{eC>y5*bQKTY$=H|AlLQzeVS{I06I8W*zgwkTq=&(e)!3bF&LEh7@B$ca!TK1s_S zb*Z~*lv6l6=P9tYvioRzIj~;v8z&*vK4P1?YUO&LW6FBq-N*}^Z{0fN5vB1kO;LAE zJD2#l)N1~e{qYWM| z$G1^b8qYgY4F^>_q57-hV)px*A&qXG2`SQOsXueE6NMgWt zt@84gZ+Ad^5IHuv_TKI$EUon9lenlw-rFIY-n9c~-QebgKI3tlHa*YdgHp$T0$TWD zX6m6^kgGByPpeXW8_m>bt|T8t9E%Sxg+O&G)uT@N`Fy&n7@vD$^_F#2bbevV?ad8; zJp{cMc!{U>BtOV+)7NvqqM1lR-o&|Tv0?KRR_yuM8iy%>JSQ?0Y|}%IW#mGzg*yq^ zBpY=rx5Civ%@%hEAHT+NA;W#-Y}K%(5fm-6rjz5z7V)3|WG>V#4Um zP}hPY7%=-Ou)cB*+OA%}&M>6>wbQ{Csyc_mWqts7VmpApf?S4dBSJBxj4>|cn&dfj zZI1^PZN>*K0pw@qM9^Ez3P@|yJaVtSS%e7+CFeR~_*~Tg`~vQO{{`q1q$agZNX@hg zF3lR0G>!`tc+(C7Z;(H#`!E-c~Yf`}>!8{`P`9JoL)dUxDCl zez;J98+4zh$L#z6GL#_Oc;)bAp-@#fyh8Y7tisY`z^K1?aG#PC`iayr3C`)N@?1}- zp!V`dAU8m{RDi1ycWjsmzSm=#FUh_3cFYb7)asepKi?YW@j02gy+Itu;H$ms{M8|l zAEGG#;LqofKAU7;Dn|HUe|#5X&&MbP3Kz9#!N#-3KR()iRvq@HiVez{C8mVBkYb#v zQDw^lGTcQLMWM+V2OI)E)(fK(*uOxJ#j*#Fa~}dCOaefk|6Z3w6b*vUTgHdC{@o!* zz??l;SlAHcIv8S#ae&9_`pLUiAymDdR1|oc6Oekq>+KF*7K9J-HbTqa4U#Ikor?+Ex96q>PT}G5~+Nnr4btmrcTF_RC%&!peWiaYX@2E zqV|{kL8n6eF^%^Pf)(NV?T@5Y_7}4qp-iGQATgA|4(!86x{gi&?}W-D1fqmh>4oa? z3#JO(tB?ATf7FlxIrRjn^}n;n?DfE=;Xo4o+suu5>*+o5WC;q6MU(}qZQB=Clba}@ zOxh9h?+b5de?neA{5h+=nkUQ{vFhy}5%j3cR!W=plaLD1T)K_O)v`R@*T4kMva1fo zW>bLXUQL9?5m<~AtT0$#51%>k6DaR%7xmd6OpcY)U{IZ;Mg}Sv6?J^>DDE8n~$%=i63SzQKrv+oi+2ZrQQ6DRL1!LeR1WVtf&Gcw)mTszDQlnJ<^Y zX+iTDzTnLtG)^l=HbDan_8i*<^{^q)8bzbql~rYim{f@V&|*4m>9n-icCNkOy#EH; ziFoHUC4sOSIO%nV>DfRn`ARn!h}f>t9v`l#CqRXDeywiQ6yG@e_g9E{zB;^dEYvNj zWAj5aYsdl_A-|w{F;-#B;1TEH0q`?a$hej4wKrl39JaGDcR?b8xH6@qz85ESn`odyaGOYMJQ)Nn z*sIxXLI0R6LFh!+00YK<#6YmiM1!sW_b(8Fd~Y)th0Vg_Z3rxys=h^XiX>cU;5xha zyh~IO_k?_SAAeqoNL;7SR?4)~J+XO#UfYlvIV8;6aX@a20*)CEiO@wZg#}p!MKll% z9ngae*EVf}AU<6rZRJsrY08vtU9EW#qC*+7dR0K;clFFfWnYUo@W76l7tSXAo%p8- zOAF8sXC&LG>zq`M`b_FKPm#2`jKwtn9gF^ZjlRqN_|F)j1XH^??XU_QuH2ZD%(a4z zZac-;xq$n~h9JdC_bD$Rt}q)r{?l27J8NqzWdVzsdfj;=3a$GCOjLcZjvlx@6#iNv zjJCku_~{TAa(!IIGlTlONc!$+R&VOZw}2Y*qfpiq&7KVhI)if5M8TF~A{@r$f=m#2 zdlE~L8i{GLXQN6gU17-V5?#$^uMCSGX{yP?Jexp+beur$qwzqNA{=CHS65x>)Rgva zDtL|?WDmV|{xT}$)WU5>=9}%T67rreP@3DmXpKZ88@S`AIDsST4F!BMs?O?k8H%XA z;4{4w4XK5dM{%5!wV0g>jIv-GdE~U*b|^Dhn?E!|jNxJ&j-_3;m+_zqFnp-tW!um0Gc~m)yYo!8ls169p&t~virvft3&3eGA0_Z( zG}?XeNLJ+wOp>rhh(MU{=5%7R`c+z8oRvXy$&scc5vO~T_kGfqo7H{YKb3qbaW?gx z38UlBFYfhOdsUV7C_fIJ35JcRemL|gQ6Ts2t-F7fepc3+ivOK32qP#Y^BdVeTD=&Z zykXIP|Kk^-U!75e82oyj^i(;{*Ij4yvCQ{a1v_^w?L>7xS1$E6^+PNb(!iud4-(!r z(TaU&13i08qhH`a4f7U^9Fr6!9TeTY-er91r49@%Yi|}OVs#0U;C;Hu_S4+x zb^4gO{NS3WObdlkcb=(gV+(WhdyJrB*^`Y4j#GHqzxKdsOO)HNl5GpacWV);ux5MK zT3D0KKK7jADKFw z$^80DP<`KE$Wif%L0HWD{#wfOSV|jtgZb)i?Lyh8PxLqU*4))u;}kgjE?L9oP?DNx z=kxY!Sge9@Wz^rF_C1fL4V*1LBRUv7I|qd~$J59a}VGyT5r{)iGStY=l)%%7H@3AY1)tSZ#Rbyh4sdY!LX|fW@76?wD zwNn(t`WOoqGx!fmJwOBrD5Vk(C{CTn(6f*9ZaFNtcOO<*7=C1vxEbZi3Akh`Qp`37pqlr~t zepKIUqjGxq5wcn?xFbE^*}@w(>E6zsB9|LP&@UaXTVn@4s(RDR*71qvt@*o1YN2-J zOp&(k(+NFIEw;i^7XIYhIRktRPn^Vr4Gj9HMSWzPQ+C>u`k1oF-nEwF#FKEQ2Oh;{ zM2`H6<*M_Eydy;zJf3VLjphvMWZdrUL=_kv85*rblfYQS(R9Y%pmk~|lj0G7u0Q!5 zUrvnyhJqc*rxL zpOzmL;34_qIlU)3#Tn1 z$@~WSg-?2h(>SBYaxI;b{Qud(gG+F_-dp0!EhuQVdTcfly>zDTU$bN80K*7MOzVn% zK2o!E<>Ou6KGX{9-p*bar^B)u(Q7+Rj`8i(+=p|*lOGP+nXm8i?2XZ7*^7E zy0P)RCW*Mx{sU!$3R35Bc;~By`-wch&G#uA$u^S7HZHdgQk?l^gxYd&YDzVTyLOv6 zy($s}E;|#Uqwy}hV`ol#RB%DGL%_c;ky{+ZTF5N9Vy>8Ef!6fTZO(OuPO zp+uLF$PmdD{GrD1NqyM*s1zH|LqTeVKn}*M1+lr*&c=t57k-b_1;>Rjczfxg=oU=M zeAVsoz&d13KvKVQt9w>or@*?memfG)FX$XJm@~YF?gpUhoCu?QEHlf(%EO^v-@Ywd$eLwj9lKLx8A5g$vP3D0>{PZ% z7;C91WXTpH9m!tFmXwi9gJesnVNSw0iqse_Djogq@94b0b6xNM?{&S`b^K$lndkd_ z@B7*B&;7aYM`SG?2&${x;j<6Z{!%I^ijiH2+W}lziP6FMs~J*1Bz@GesyryQ2@X6g z$Yi{~{?Eak{hBieJHH>OV?A*Yf3T}Hnt)mOnmbMaew;zlGMZGfToy>PQa0?#!FA<4 zG1&Qquqko1{dJ~a`YO?VGLV!iI#pX0*2xv4caulyB33AMS-Bc*>YZdjj%K`0u#R!* z>4*>9lAuJu7$wH4?#rEck;sNOo$g=p_sS$z;(aND;d#&%&QZvqH+}{HK%#_cQi5nN zcN5}#)ArjmHG;l`A!X=xppDDqCDJylC~{`$l2MMLBv}f6k0qJ0J~ydrCGW2D#G|0< zE^~QlCzVKq8&6?b!CkD#6AO8fn<9%&w z1$Qt74^uIIv4k%Flg{Xr6Dxiy`kB}++|spqE=N^vuP=eMIFBoaf5j5vtTyy^Tf>1% zFxBJy=C{EHHptVWLCn)ocxb!Yzn1qbCot0>Y%0;cswmd#*E{ zfz&49selIenbI4}RT>YkrkPlLZCWen5%AE*HT=jY6feh+HIxk146~YQG$Z-J4&e>A zOK);Nqa|kiC;HujW-|ZS!10}e*N?9Q zm{&}j_k;#aWZLq9V2(@Pn=H%-THv{ZQM(vSh!fmnWoCL^fvliJLOJQh%B!jUuDhOu zoI{L$$L8c!$oSSS0s$&r1~>ama6Z4y8s8j$Mf$q5lcLpm2NqF|-~y>2ur#G3$b&Q! zxpF%hz!c^hVn<<95rGx&xFc-fyJ>FmGY)NBK63aD6v{^d9iifn7MddDPnRkW3TjV~ z&&|{-=a}NCnWQzuBR&=O{0%}08_^h=7a(95c1KDM;iTGcGLX!=Lk_OAhRB$*z(1cR zb07Ff7n>TF#?U?3grCtos|~@&5Ttuz2a%yG67Qg=VPR%F;IyPKcIT^lwTp0Uhnk-K z4}FCnb9qhz5Q>qpz=hZ%s>KK!$%%*V>SUtV6Vcq4+o@s|P<18nVlVCkB;TG5JF=eT zP`b3#Z8W8+C$MVZ?26xFRa1QW4%pFcu`gkbA+2Kgv|CosQ zafI)Z<3A2VJZ4E%&#{0uv8mh6byM2B^u^38F`BGAx+>l^SiPVWlF{$Dn=Q-p^Sv38 zqrz${v92y#%^ftO={9(*%SbMNVSZ_4#o>Qp8qFdBA!MR!OLRSf_q7STCW(K|LTDp%t7)z`w zl)uS?-FnFS&S6XZ4WF8LmwPsDNIoIDYjMu9Z@HMnu<_x_^O#uKzIKPtwL9-3#(b5u zzp&ygkG4EV<>iRn!07DJlb~u=`75_DHpE_K|GBLF!VVnO7JYuON!AIrK~kF8r#tAK z5A^DMa|y{iKS!Maj^F8lAJO2r9Up8dnq%8$?UGTUMl_Y2FWCIndHTW+>fF?JSIblT z3JOl0!;e6oW?B)7m}hfqSBYWY=CG0imhLpX_oI$Rp=*}MCLHu9D%4tA#2Pqfol=Fq z&0HzDt$RL3Bj4Bt4xGHKi(bn~$~Pp^co)ZPEi>>pX?r%UO&_{l1OKnkRP&3_9o~-W zM5^GFXzg`^rZy!iyh3G;%iQqD;n$?K+Ujc;C4=KOhtGb&{z?;o?KbnC@A4TA zP90Vbeg&>!HEY`97-hJ-{VbU`@%(MK2{-Be6$g|(S2A>l4h6VY-H(}6pR5JVz#NhMmAZqQ$&al6IE;L|fp+NC0~ECem9%9}I)QMn^gt!M2W zBkY8#2^N~b0OTCDwGhvO%f~Jr{e++uebM$I@@u~Kkf$Q=;@)lx(Q=2 z1+_vb4Mpk?8r6_C2?;~cn;V6ZVE*5kJQDmZsW$a~7bl85Yf5mWR4Dc>Z=ejXo`;fIf#;8}&sqZP}7nUDD@p*CikU^^qvMsagT0( zMdlwMIU7nDq`RT|0z(baIe$;&008&-9D=%#QX-5`Cl=m*;xUVg>_oo+(CZ|^kjyJD zds-nGzUjl!nu907y0(7f>M2E^hq0pApzX$_E=bb`7v@Hl3%G&TL;NEYxVufDdp z$-*L{g|#x@FO-1g`93K%ne20$H(dXIebZC*Mg!q%n_Y}&yKL|@o@#Fwg_ir<2nGk7 zup(&`2MDJ$KaunS_=9H9IRIvELIZWV6L0gqOoOm-aJ|T}_~~4&!2)Hzb)d39xnTa@fwYTEKC> z&SN?yPjloa-#M&ENS2;WH9s85hOZHN++ZhVA4IN|zlHsl+FPP&Ac`%QFVAxpB4ETE zv1=4O--5b3Zu;>D;PPg9g+9EB)x*ZYsqy|~T9-=xc^m18b6a-TEJu^<%-GK5f?>@l zR(J@Ro<|ZQ$>9 zy-WogQVjFGyfO}B>9AWq*8~62`mwUT1ME}p z#|QudVvZ1;Q8$!#ZB0h+_AbUIlq# zt()C}eIy%hF?k!*oxQ@e!3)AGl_L5pFFV)ucw?gHn?-x;gX>CWwO_-z>V~t5z*Vh$ zfCzPa;+j!cD|Zzf6q_3I0GwV+`0a1Ohc=<9BB*EJjr!A0er9T73;E(A1TtX;J4EP% z*W|x&3zhGy?CE%&@NrN(kB8nnl_{EVYiT9PFW{bCW=k^hXX9;mc_1{TkV-~{%=d~K zmxi2qB;tNI8u4lm#*!5nWhC|!Co)-Pn)swYOB9W8RnQJA|H%KR%N2ffLCLBP{d5TJ)hpoYwaXFn?3bf#G`AnE=3|-a zZNhNKntHj+YL5tIw|?T_mhXBz$kh)`^D;8o3ZJ(3n6ZdPcZ6JUwUJ;)3}_r z>q6ceEjXL@yo;3nnSC}5e{x73&Asn;rq^)g5Xis8QOqOdTY-yqet-(saLj7bd;Yj7 zD7DCYx1cSCYNyDziOOy06A%T6Wyqf`M=o_;x5D!cDX+^bg=z4oxXucaTXzQL0POS9 z*H&bvj4(V+*aj$JyDCV}INx)@fGlVo{j`X^P&CecZ|WWwAqYCKLPPODQ*FI5J| z-D?9+Z!f3O8hLHLv-cQ12-n#} zQk%1>yEFU}%hB%%1R*2S(Y-o%V{$TvT<}30FZy9&IWHJ|;MuZ(l~$vN?V_80?W)+G z(n=w6ShFctDH1eeZH`qga6}ceQzO1OZ(*Zy#}C}haXyo1vlOJjzPXi=GnzI1A#r%_ zao}UM679y!IrdmLMgl-Dj}aD0(&w8;yAwSKS~*vgLZ#zwer%!o>8lcA*g^rb!8N3A zLc-nrj%@EYpPR2}u%l&7g(WL<{B-M!!0NmFaR8>pT?cZ1<5Q8K`&ns|Ow-4mUHTQ0 zMq~&(Trx5rNCP2L6DLRI`O`ee8?{!Fn{k*>A9#I(eZE`0A(KNqSpAD(RI^aK1Bpjh zw8R}(*5qLF5Ay)1cQFNkmLSo+~#{1=m-p} zT~#QHHu7T>2;Xv&mN=S^Br}?R_YPuo?Drnt^I|i{ZuY0jPD$K98q_8&hJp(MJv=!V zUdY9lo&>9*Fg}53#A7SBEg7OAvg(XJv?*THly+MhpLw-+dm!}7K1}h6x5MG&9isJNi$cIB^K2KVthL@b_g#<&&qJ?m;O$fOhVu$$9^ zk9sF>nq8Bqmhx@DjV6u^d=q@6-;^Pzi~GX|wGbgfwGBuQeajz0d(8S~TQu5(om!)j zJQxt1vFAo;f}A%hj+aU~l1}MqwX*j?rSUd^hdMRX<-2fFe)n-J%$s9UAI`m?m2&3x z+R532g{8R=7Yf}Ijzr24-?wxg7hy(J-ZTj(u1N_x<6n9U!i8GGZ{?ly%W&N2>-(AS6>XuwmQ5TxJ2!zJ*zbmuZM)bEEb*u;%uiwOJ9iwPZ#$ zI_lWTXe9Fi#-ej(Q3(z(5Co+Z%&SAm(!eqc)X$w_RK2qT|8w=o*j88?Rm~M+c`A^w z(F6EuNu^7d%{T!&l{EXD3sw7YTol$lFB#H%8OWmnfq~0XR|u#a zuOc*y+JLuFJ(^jdJ$JObq+SFxE46Vw4E7TQZ_g?H+ui?{-7TmKaA2Dp+BuF5@vasb z;C$;>!SN<`uN3R;!NwaNbROVefUdnGVCHSf!o7GZj4ZME2%;Y1MS!^=4wQ*@?ooSe z*)|vm(KWGI8&8zQ!KfNX>CoE(;e#H^c@TydQkO1}MWT<}mU%<*vs{lpC?WNmu^GYy zc*8LnhJKcDhD5zD&-_wY0wF7$^SgSti3}fMR1fQ5%pm9fNTvd2Yz{C-N~MHVi=g;4YeW1m`H{hInB0ou(NzSw z!k9{d!OMkgml8p@HMOCZG_Kym2$%_yp2~s8)3rMCIVM4mu$1CAzn-W=2+JMXAG1!_ zz=vkmap_+#08}Zw`T_?pkXN?jS(dyuq%s7 zU~TgJTQcT2n0YV#w?IfKGXbP~%vzZ~tH0j}?Zy80XXN20$-tcFdJ_FlF_ArR<2AqJ zGK2yAhGk;=KgEO&7x?Gf<+xx_Ny|}x4f>7eAx*)5+XG~rLl#7K|Mq{1i8Rv2Fy4^s m-$wekk^UV?|1YQ8@=q3h+dI==Jxiik;16r CREATE USER 'clone-init'@'localhost' IDENTIFIED BY 'yyyyyyyyyyy'; mysql> GRANT ALL ON *.* TO 'clone-init'@'localhost' WITH GRANT OPTION; ``` -You may change the user names and should change the passwords. +You may change the user names and should change their passwords. Then create a Secret in the same namespace as MySQLCluster: @@ -98,9 +158,22 @@ spec: containers: - name: mysqld image: quay.io/cybozu/moco-mysql:8.0.24 # must be the same version as the donor + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi ``` -You can stop the replication from the donor by setting `spec.replicationSourceSecretName` to `null` afterwards. +To stop the replication from the donor, update MySQLCluster with `spec.replicationSourceSecretName: null`. + +### Bring your own image + +We provide a pre-built MySQL container image at [quay.io/cybozu/moco-mysql](http://quay.io/cybozu/moco-mysql). +If you want to build and use your own image, read [`custom-mysqld.md`](custom-mysqld.md). ### Configurations @@ -115,8 +188,8 @@ metadata: namespace: foo name: mycnf data: - long_query_time: "10" - innodb_buffer_pool_size: 10G + long_query_time: "5" + innodb_buffer_pool_size: "10G" ``` and set the name of the ConfigMap in MySQLCluster as follows: @@ -138,9 +211,103 @@ If `innodb_buffer_pool_size` is not given, MOCO sets it automatically to 70% of ### `kubectl moco` -### Connecting to the primary instance +From outside of your Kubernetes cluster, you can access MOCO MySQL instances using `kubectl-moco`. +`kubectl-moco` is [a plugin for `kubectl`](https://kubernetes.io/docs/tasks/extend-kubectl/kubectl-plugins/). +Pre-built binaries are available on [GitHub releases](https://github.com/cybozu-go/moco/releases/latest). + +The following is an example to run `mysql` command interactively to access the primary instance of `test` MySQLCluster in `foo` namespace. -### Connecting to read-only replicas +```console +$ kubectl moco -n foo mysql -it test +``` + +Read [the reference manual of `kubectl-moco`](kubectl-moco.md) for further details and examples. + +### MySQL users + +MOCO prepares a set of users. + +- `moco-readonly` can read all tables of all databases. +- `moco-writable` can create users, databases, or tables. +- `moco-admin` is the super user. + +The exact privileges that `moco-readonly` has are: + +- PROCESS +- REPLICATION CLIENT +- REPLICATION SLAVE +- SELECT +- SHOW DATABASES +- SHOW VIEW + +The exact privileges that `moco-writable` has are: + +- ALTER +- ALTER ROUTINE +- CREATE +- CREATE ROLE +- CREATE ROUTINE +- CREATE TEMPORARY TABLES +- CREATE USER +- CREATE VIEW +- DELETE +- DROP +- DROP ROLE +- EVENT +- EXECUTE +- INDEX +- INSERT +- LOCK TABLES +- PROCESS +- REFERENCES +- REPLICATION CLIENT +- REPLICATION SLAVE +- SELECT +- SHOW DATABASES +- SHOW VIEW +- TRIGGER +- UPDATE + +`moco-writable` cannot edit tables in `mysql` database, though. + +You can create other users and grant them certain privileges as either `moco-writable` or `moco-admin`. + +```console +$ kubectl moco mysql -u moco-writable test -- -e "CREATE USER 'foo'@'%' IDENTIFIED BY 'bar'" +$ kubectl moco mysql -u moco-writable test -- -e "CREATE DATABASE db1" +$ kubectl moco mysql -u moco-writable test -- -e "GRANT ALL ON db1.* TO 'foo'@'%'" +``` + +### Connecting to `mysqld` over network + +MOCO prepares two Services for each MySQLCluster. +For example, a MySQLCluster named `test` in `foo` Namespace has the following Services. + +| Service Name | DNS Name | Description | +| ------------------- | --------------------------- | -------------------------------- | +| `moco-test-primary` | `moco-test-primary.foo.svc` | Connect to the primary instance. | +| `moco-test-replica` | `moco-test-replica.foo.svc` | Connect to replica instances. | + +`moco-test-replica` can be used only for read access. + +The type of these Services is usually ClusterIP. +The following is an example to change Service type to LoadBalancer and add an annotation for [MetalLB][]. + +```yaml +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: foo + name: test +spec: + serviceTemplate: + metadata: + annotations: + metallb.universe.tf/address-pool: production-public-ips + spec: + type: LoadBalancer +... +``` ## Deleting the cluster @@ -152,20 +319,80 @@ If you want to keep the PersistentVolumeClaims, remove `metadata.ownerReferences ### Cluster status +You can see the health and availability status of MySQLCluster as follows: + +```console +$ kubectl get mysqlcluster +NAME AVAILABLE HEALTHY PRIMARY SYNCED REPLICAS ERRANT REPLICAS +test True True 0 3 +``` + +- The cluster is available when the primary Pod is running and ready. +- The cluster is healthy when there is no problems. +- `PRIMARY` is the index of the current primary instance Pod. +- `SYNCED REPLICAS` is the number of ready Pods. +- `ERRANT REPLICAS` is the number of instances having errant transactions. + +You can also use `kubectl describe mysqlcluster` to see the recent events on the cluster. + ### Pod status MOCO adds mysqld containers a liveness probe and a readiness probe to check the replication status in addition to the process status. A replica Pod is _ready_ only when it is replicating data from the primary without a significant delay. +The default threshold of the delay is 60 seconds. The threshold can be configured as follows. + +```yaml +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: foo + name: test +spec: + maxDelaySeconds: 180 + ... +``` + +Unready replica Pods are automatically excluded from the load-balancing targets so that users will not see too old data. ### Metrics +See [`metrics.md`](metrics.md) for available metrics in Prometheus format. + ### Logs +Error logs from `mysqld` can be viewed as follows: + +```console +$ kubectl logs moco-test-0 mysqld +``` + +Slow logs from `mysqld` can be viewed as follows: + +```console +$ kubectl logs moco-test-0 slow-log +``` + ## Maintenance ### Increasing the number of instances in the cluster +Edit `spec.replicas` field of MySQLCluster: + +```yaml +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: foo + name: test +spec: + replicas: 5 + ... +``` + +You can only increase the number of instances in a MySQLCluster from 1 to 3 or 5, or from 3 to 5. +Decreasing the number of instances is not allowed. + ### Switchover Switchver is an operation to change the live primary to one of the replicas. @@ -175,15 +402,36 @@ MOCO automatically switch the primary when the Pod of the primary instance is to Users can manually trigger a switchover by annotating the Pod of the primary instance with `moco.cybozu.com/demote: true`. You can use `kubectl` to do this: ```console -$ kubectl annotate mysqlclusters moco.cybozu.com/demote=true +$ kubectl annotate mysqlcluster moco.cybozu.com/demote=true ``` ### Failover +Failover is an operation to replace the dead primary with the most advanced replica. +MOCO automatically does this as soon as it detects that the primary is down. + +The most advanced replica is a replica who has retrieved the most up-to-date transaction from the dead primary. +Since MOCO configures loss-less semi-synchronous replication, the failover is guaranteed not to lose any user data. + +After a failover, the old primary may become an errant replica [as described](#errant-replicas). + ### Upgrading mysql version +TBD + ### Re-initializing an errant replica +Delete the PVC and Pod of the errant replica, like this: + +```console +$ kubectl delete --wait=false pvc mysql-data-moco-test-0 +$ kubectl delete --grace-period=1 pods moco-test-0 +``` + +Depending on your Kubernetes version, StatefulSet controller may create a pending Pod before PVC gets deleted. +Delete such pending Pods until PVC is actually removed. + [semisync]: https://dev.mysql.com/doc/refman/8.0/en/replication-semisync.html [GTID]: https://dev.mysql.com/doc/refman/8.0/en/replication-gtids.html [CLONE]: https://dev.mysql.com/doc/refman/8.0/en/clone-plugin.html +[MetalLB]: https://metallb.universe.tf/ diff --git a/e2e/lifecycle_test.go b/e2e/lifecycle_test.go index b1f8baa67..f54c0f092 100644 --- a/e2e/lifecycle_test.go +++ b/e2e/lifecycle_test.go @@ -57,12 +57,16 @@ var _ = Context("lifecycle", func() { }) It("should update the configmap and restart mysqld", func() { - cluster, err := getCluster("foo", "single") - Expect(err).NotTo(HaveOccurred()) - - cluster.Spec.MySQLConfigMapName = nil - data, _ := json.Marshal(cluster) - kubectlSafe(data, "apply", "-f", "-") + Eventually(func() error { + cluster, err := getCluster("foo", "single") + if err != nil { + return err + } + cluster.Spec.MySQLConfigMapName = nil + data, _ := json.Marshal(cluster) + _, err = kubectl(data, "apply", "-f", "-") + return err + }).Should(Succeed()) Eventually(func() float64 { out, err := kubectl(nil, "moco", "-n", "foo", "mysql", "single", "--", "-N", "-e", "SELECT @@long_query_time") @@ -154,10 +158,15 @@ var _ = Context("lifecycle", func() { if err != nil { return err } - if len(cms.Items) == 1 { - return nil + + for _, cm := range cms.Items { + switch cm.Name { + case "kube-root-ca.crt", "mycnf": + default: + return fmt.Errorf("pending config map %+v", cm) + } } - return fmt.Errorf("pending config maps: %+v", cms.Items) + return nil }).Should(Succeed()) Eventually(func() error { secrets := &corev1.SecretList{} diff --git a/e2e/replication_test.go b/e2e/replication_test.go index 2d516b790..020b9aaae 100644 --- a/e2e/replication_test.go +++ b/e2e/replication_test.go @@ -138,14 +138,16 @@ var _ = Context("replication", func() { }) It("should be able to scale out the cluster", func() { - out := kubectlSafe(nil, "-n", "repl", "get", "mysqlcluster", "test", "-o", "json") - cluster := &mocov1beta1.MySQLCluster{} - err := json.Unmarshal(out, cluster) - Expect(err).NotTo(HaveOccurred()) - - cluster.Spec.Replicas = 5 - data, _ := json.Marshal(cluster) - kubectlSafe(data, "-n", "repl", "apply", "-f", "-") + Eventually(func() error { + cluster, err := getCluster("repl", "test") + if err != nil { + return err + } + cluster.Spec.Replicas = 5 + data, _ := json.Marshal(cluster) + _, err = kubectl(data, "apply", "-f", "-") + return err + }).Should(Succeed()) Eventually(func() error { cluster, err := getCluster("repl", "test") @@ -169,10 +171,16 @@ var _ = Context("replication", func() { }) It("should detect errant transactions", func() { - kubectlSafe(nil, "moco", "-n", "repl", "mysql", "-u", "moco-admin", "--index", "0", "test", "--", - "-e", "SET GLOBAL read_only=0") - kubectlSafe(nil, "moco", "-n", "repl", "mysql", "-u", "moco-admin", "--index", "0", "test", "--", - "-e", "CREATE DATABASE errant") + Eventually(func() error { + _, err := kubectl(nil, "moco", "-n", "repl", "mysql", "-u", "moco-admin", "--index", "0", "test", "--", + "-e", "SET GLOBAL read_only=0") + if err != nil { + return err + } + _, err = kubectl(nil, "moco", "-n", "repl", "mysql", "-u", "moco-admin", "--index", "0", "test", "--", + "-e", "CREATE DATABASE errant") + return err + }).Should(Succeed()) Eventually(func() int { cluster, err := getCluster("repl", "test") @@ -296,12 +304,16 @@ var _ = Context("replication", func() { }) It("should be able to stop replication from the donor", func() { - cluster, err := getCluster("repl", "test") - Expect(err).NotTo(HaveOccurred()) - - cluster.Spec.ReplicationSourceSecretName = nil - data, _ := json.Marshal(cluster) - kubectlSafe(data, "apply", "-f", "-") + Eventually(func() error { + cluster, err := getCluster("repl", "test") + if err != nil { + return err + } + cluster.Spec.ReplicationSourceSecretName = nil + data, _ := json.Marshal(cluster) + _, err = kubectl(data, "apply", "-f", "-") + return err + }).Should(Succeed()) Eventually(func() error { _, err := kubectl(nil, "moco", "-n", "repl", "mysql", "-u", "moco-writable", "test", "--", diff --git a/examples/anti-affinity.yaml b/examples/anti-affinity.yaml new file mode 100644 index 000000000..31b93b1a2 --- /dev/null +++ b/examples/anti-affinity.yaml @@ -0,0 +1,44 @@ +# This example shows how to schedule Pods on different Nodes. +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + replicas: 3 + podTemplate: + spec: + affinity: + # The anti-affinity for Pods + # cf. https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - moco + - key: app.kubernetes.io/instance + operator: In + values: + - test + topologyKey: "kubernetes.io/hostname" + containers: + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + resources: + requests: + cpu: "10" + memory: "10Gi" + limits: + cpu: "10" + memory: "10Gi" + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi diff --git a/examples/custom-mycnf.yaml b/examples/custom-mycnf.yaml new file mode 100644 index 000000000..bd6f311a7 --- /dev/null +++ b/examples/custom-mycnf.yaml @@ -0,0 +1,35 @@ +# This example shows how to set MySQL server system variables. +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: default + name: mycnf +data: + # key-value in data field will become server system variable names and values. + # https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html + # https://dev.mysql.com/doc/refman/8.0/en/innodb-parameters.html + long_query_time: "5" + innodb_buffer_pool_size: "70G" +--- +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + replicas: 3 + # ConfigMap name in the same namespace. + mysqlConfigMapName: mycnf + podTemplate: + spec: + containers: + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi diff --git a/examples/guaranteed.yaml b/examples/guaranteed.yaml new file mode 100644 index 000000000..7d2f8568e --- /dev/null +++ b/examples/guaranteed.yaml @@ -0,0 +1,30 @@ +# This example shows how to assign Guaranteed QoS class to Pods. +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + replicas: 3 + podTemplate: + spec: + containers: + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + # By allocating the same CPU and memory for both requests and limits, + # the Pod gets assigned Guaranteed QoS class. + resources: + requests: + cpu: "10" + memory: "10Gi" + limits: + cpu: "10" + memory: "10Gi" + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi diff --git a/examples/loadbalancer.yaml b/examples/loadbalancer.yaml new file mode 100644 index 000000000..7a1299160 --- /dev/null +++ b/examples/loadbalancer.yaml @@ -0,0 +1,29 @@ +# This example shows how to change Service type to LoadBalancer +apiVersion: moco.cybozu.com/v1beta1 +kind: MySQLCluster +metadata: + namespace: default + name: test +spec: + replicas: 3 + # serviceTemplate allows you to specify annotations, labels, and spec + # of Services to be generated for this MySQLCluster. + serviceTemplate: + metadata: + annotations: + metallb.universe.tf/address-pool: production-public-ips + spec: + type: LoadBalancer + podTemplate: + spec: + containers: + - name: mysqld + image: quay.io/cybozu/moco-mysql:8.0.24 + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi diff --git a/go.mod b/go.mod index 42719945b..39af7b512 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cybozu-go/moco go 1.16 require ( - github.com/cybozu-go/moco-agent v0.6.0 + github.com/cybozu-go/moco-agent v0.6.1 github.com/go-logr/logr v0.4.0 github.com/go-logr/stdr v0.4.0 github.com/go-sql-driver/mysql v1.6.0 diff --git a/go.sum b/go.sum index 052967e02..0f2dbee41 100644 --- a/go.sum +++ b/go.sum @@ -106,8 +106,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cybozu-go/log v1.5.0/go.mod h1:zpfovuCgUx+a/ErvQrThoT+/z1RVQoLDOf95wkBeRiw= github.com/cybozu-go/log v1.6.0/go.mod h1:2iAEvn2cL5dy/1uP5Jfb0Ao9+DUnDr//V0Bk3WDJX1U= -github.com/cybozu-go/moco-agent v0.6.0 h1:3anuGMTMUh5BbBp7yRvGpWhLi+kKmL7O2zke0lfNnWk= -github.com/cybozu-go/moco-agent v0.6.0/go.mod h1:emhcINL6P81LZ4o355DETRXs8S63HUAXJN7SK01e7MY= +github.com/cybozu-go/moco-agent v0.6.1 h1:Cn9//cBSsZoXNzTfeoaNyyWVKmp+m1LJdV2hNvKdGrA= +github.com/cybozu-go/moco-agent v0.6.1/go.mod h1:emhcINL6P81LZ4o355DETRXs8S63HUAXJN7SK01e7MY= github.com/cybozu-go/netutil v1.2.0/go.mod h1:Wx92iF1dPrtuSzLUMEidtrKTFiDWpLcsYvbQ1lHSmxY= github.com/cybozu-go/well v1.10.0/go.mod h1:OQdjEXQpbG+kSgEF3t3IYUx5y1R4qeBGvzL4gmi61qE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/kustomization.yaml b/kustomization.yaml index 2ee3bdb33..90c9acc2d 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -3,4 +3,4 @@ resources: images: - name: ghcr.io/cybozu-go/moco - newTag: 0.7.0 + newTag: 0.8.0 diff --git a/pkg/mycnf/generator.go b/pkg/mycnf/generator.go index e28d90ecc..4c22400de 100644 --- a/pkg/mycnf/generator.go +++ b/pkg/mycnf/generator.go @@ -88,6 +88,7 @@ var DefaultMycnf = map[string]string{ "innodb_flush_neighbors": "0", "innodb_random_read_ahead": "false", "innodb_read_ahead_threshold": "0", + "innodb_log_write_ahead_size": "512", } // ConstMycnf is the mysqld configurations that MOCO applies forcibly. diff --git a/pkg/mycnf/testdata/bufsize.cnf b/pkg/mycnf/testdata/bufsize.cnf index b6c361fc5..84caa3992 100644 --- a/pkg/mycnf/testdata/bufsize.cnf +++ b/pkg/mycnf/testdata/bufsize.cnf @@ -31,6 +31,7 @@ innodb_flush_neighbors = 0 innodb_lock_wait_timeout = 60 innodb_log_file_size = 800M innodb_log_files_in_group = 2 +innodb_log_write_ahead_size = 512 innodb_online_alter_log_max_size = 1073741824 innodb_print_all_deadlocks = 1 innodb_random_read_ahead = false diff --git a/pkg/mycnf/testdata/loose.cnf b/pkg/mycnf/testdata/loose.cnf index 7b03cd428..4b95f0552 100644 --- a/pkg/mycnf/testdata/loose.cnf +++ b/pkg/mycnf/testdata/loose.cnf @@ -31,6 +31,7 @@ innodb_flush_neighbors = 0 innodb_lock_wait_timeout = 60 innodb_log_file_size = 800M innodb_log_files_in_group = 2 +innodb_log_write_ahead_size = 512 innodb_numa_interleave = OFF innodb_online_alter_log_max_size = 1073741824 innodb_print_all_deadlocks = 1 diff --git a/pkg/mycnf/testdata/nil.cnf b/pkg/mycnf/testdata/nil.cnf index 93917b5bd..6b582a81e 100644 --- a/pkg/mycnf/testdata/nil.cnf +++ b/pkg/mycnf/testdata/nil.cnf @@ -31,6 +31,7 @@ innodb_flush_neighbors = 0 innodb_lock_wait_timeout = 60 innodb_log_file_size = 800M innodb_log_files_in_group = 2 +innodb_log_write_ahead_size = 512 innodb_online_alter_log_max_size = 1073741824 innodb_print_all_deadlocks = 1 innodb_random_read_ahead = false diff --git a/pkg/mycnf/testdata/normalize.cnf b/pkg/mycnf/testdata/normalize.cnf index 5c7027687..2fdc4662d 100644 --- a/pkg/mycnf/testdata/normalize.cnf +++ b/pkg/mycnf/testdata/normalize.cnf @@ -32,6 +32,7 @@ innodb_flush_neighbors = 0 innodb_lock_wait_timeout = 60 innodb_log_file_size = 800M innodb_log_files_in_group = 2 +innodb_log_write_ahead_size = 512 innodb_online_alter_log_max_size = 1073741824 innodb_print_all_deadlocks = 1 innodb_random_read_ahead = false diff --git a/version.go b/version.go index 274ba95ab..7057d7d82 100644 --- a/version.go +++ b/version.go @@ -5,5 +5,5 @@ const ( Version = "0.8.0" // FluentBitImage is the image for slow-log sidecar container. - FluentBitImage = "quay.io/cybozu/fluent-bit:1.7.2.2" + FluentBitImage = "quay.io/cybozu/fluent-bit:1.7.4.1" )