Merge pull request #228 from cybozu-go/refactor-1

Reimplementation

.dockerignore

@@ -1,4 +1,5 @@
 /bin
+/testbin
 /config
 /docs
 /e2e

.github/workflows/ci.yaml

@@ -3,124 +3,61 @@ on:
  pull_request:
  push:
  branches:
-  - 'main'
+ - 'main'
 env:
  go-version: 1.16
  cache-version: 1
 jobs:
- validation:
- name: Validation
+ test:
+ name: Small tests
  runs-on: ubuntu-20.04
  steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-go@v2
- with:
- go-version: ${{ env.go-version }}
- - name: Cache Tools
- id: cache-tools
- uses: actions/cache@v2
- with:
- path: |
- bin
- key: cache-${{ env.cache-version }}-go-${{ env.go-version }}-${{ hashFiles('Makefile') }}
- - run: make setup
- if: steps.cache-tools.outputs.cache-hit != 'true'
- - run: make validate
- test:
- name: Small Tests
+ - uses: actions/checkout@v2
+ - uses: actions/setup-go@v2
+ with:
+ go-version: ${{ env.go-version }}
+ - run: make test
+ - run: make check-generate
+ - run: make envtest
+ - run: make release-build
+ test-dbop:
+ name: Test pkg/dbop
  strategy:
  matrix:
- mysql-version: ["8.0.18", "8.0.20"]
+ mysql-version: ["8.0.18", "8.0.20", "8.0.24"]
  runs-on: ubuntu-20.04
  steps:
-  - uses: actions/checkout@v2
-  - uses: actions/setup-go@v2
-  with:
-  go-version: ${{ env.go-version }}
-  - run: make test MYSQL_VERSION=${{ matrix.mysql-version }}
+ - uses: actions/checkout@v2
+ - uses: actions/setup-go@v2
+ with:
+ go-version: ${{ env.go-version }}
+ - run: make test-dbop MYSQL_VERSION=${{ matrix.mysql-version }}
  e2e:
  name: End-to-End Tests
  strategy:
  matrix:
- mysql-version: ["8.0.18", "8.0.20"]
- k8s-version: ["1.19.4", "1.20.2"]
- runs-on: ubuntu-20.04
- steps:
- - uses: actions/checkout@v2
- - run: |
- swapon > swapon.txt
- sudo swapoff -a
- cat swapon.txt | tail -n+2 | awk '$2=="file" {print $1}' | sudo xargs --no-run-if-empty rm
- - run: sudo mkdir /mnt/local-path-provisioner
- - run: cd e2e && make setup
- - name: Prepare
- run: |
- VARIABLES=read_only,super_read_only,rpl_semi_sync_master_wait_for_slave_count,clone_valid_donor_list
- ./e2e/script/watch_mysql.sh e2e-test mysqlcluster $VARIABLES > watch_mysql_e2e.txt &
- ./e2e/script/watch_mysql.sh e2e-test-external mysqlcluster $VARIABLES > watch_mysql_e2e-external.txt &
- ./e2e/script/watch_pod.sh e2e-test > watch_pod_e2e.txt &
- ./e2e/script/watch_pod.sh e2e-test-external > watch_pod_e2e-external.txt &
- - run: cd e2e && make test KIND_CLUSTER_CONFIG_SUFFIX=-actions MYSQL_VERSION=${{ matrix.mysql-version }} KUBERNETES_VERSION=${{ matrix.k8s-version }}
-
- # Print status and log of each component
- - name: Watch MySQL status (e2e-test)
- run: cat watch_mysql_e2e.txt
- if: always()
- - name: Watch MySQL status (e2e-test-external)
- run: cat watch_mysql_e2e-external.txt
- if: always()
- - name: Watch pod (e2e-test)
- run: cat watch_pod_e2e.txt
- if: always()
- - name: Watch pod (e2e-test-external)
- run: cat watch_pod_e2e-external.txt
- if: always()
- - run: |
- top -b -n 1
- df -h
- if: always()
- - run: ./e2e/bin/kubectl -n moco-system get all
- if: always()
- - run: |
- ./e2e/bin/kubectl -n e2e-test-external describe mysqlclusters.moco.cybozu.com
- ./e2e/bin/kubectl -n e2e-test-external get all
- if: always()
- - run: ./e2e/bin/kubectl -n moco-system logs -lcontrol-plane=moco-controller-manager --tail=-1
- if: always()
- - run: ./e2e/bin/kubectl -n e2e-test-external logs -lapp.kubernetes.io/name=moco-mysql -c agent --tail=-1 | grep -w -v "grpc.health.v1.Health"
- if: always()
- - run: ./e2e/bin/kubectl -n e2e-test-external logs moco-mysqlcluster-0 -c err-log
- if: always()
- - run: ./e2e/bin/kubectl -n e2e-test-external logs moco-mysqlcluster-1 -c err-log
- if: always()
- - run: ./e2e/bin/kubectl -n e2e-test-external logs moco-mysqlcluster-2 -c err-log
- if: always()
- e2e-upgrade:
- name: End-to-End Upgrade Tests
+ mysql-version: ["8.0.18", "8.0.20", "8.0.24"]
+ k8s-version: ["1.19.7", "1.20.2"]
  runs-on: ubuntu-20.04
  steps:
- - uses: actions/checkout@v2
- - run: |
- swapon > swapon.txt
- sudo swapoff -a
- cat swapon.txt | tail -n+2 | awk '$2=="file" {print $1}' | sudo xargs --no-run-if-empty rm
- - run: sudo mkdir /mnt/local-path-provisioner
- - run: cd e2e && make setup
- - name: Prepare
- run: |
- VARIABLES=read_only,super_read_only,rpl_semi_sync_master_wait_for_slave_count,clone_valid_donor_list
- ./e2e/script/watch_mysql.sh e2e-test mysqlcluster $VARIABLES > watch_mysql_e2e.txt &
- ./e2e/script/watch_pod.sh e2e-test > watch_pod_e2e.txt &
- - run: cd e2e && UPGRADE=1 make test KIND_CLUSTER_CONFIG_SUFFIX=-actions
-
- # Print status and log of each component
- - name: Watch MySQL status (e2e-test)
- run: cat watch_mysql_e2e.txt
- if: always()
- - name: Watch pod (e2e-test)
- run: cat watch_pod_e2e.txt
- if: always()
- - run: ./e2e/bin/kubectl -n moco-system get all
- if: always()
- - run: ./e2e/bin/kubectl -n moco-system logs -lcontrol-plane=moco-controller-manager --tail=-1
- if: always()
+ - uses: actions/checkout@v2
+ - uses: actions/setup-go@v2
+ with:
+ go-version: ${{ env.go-version }}
+ - run: |
+ swapon > swapon.txt
+ sudo swapoff -a
+ cat swapon.txt | tail -n+2 | awk '$2=="file" {print $1}' | sudo xargs --no-run-if-empty rm
+ - run: sudo mkdir /mnt/local-path-provisioner0 /mnt/local-path-provisioner1 /mnt/local-path-provisioner2
+ - run: make start KUBERNETES_VERSION=${{ matrix.k8s-version }} MYSQL_VERSION=${{ matrix.mysql-version }} KIND_CONFIG=kind-config_actions.yaml
+ working-directory: e2e
+ - run: make test
+ working-directory: e2e
+ - run: make logs
+ working-directory: e2e
+ if: always()
+ - uses: actions/upload-artifact@v2
+ if: always()
+ with:
+ name: logs.tar.gz
+ path: e2e/logs.tar.gz

.github/workflows/release.yaml

@@ -4,65 +4,34 @@ on:
  tags:
  - 'v*'
 env:
- go-version: 1.13
+ go-version: 1.16
 jobs:
  image:
  name: Push Container Image
  runs-on: ubuntu-20.04
  steps:
-  - uses: actions/checkout@v2
-  - run: docker build . -t moco:dev
-  - name: Login to ghcr.io
-  run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
-  - run: docker tag moco:dev ghcr.io/cybozu-go/moco:${GITHUB_REF#refs/tags/v}
-  - run: docker push ghcr.io/cybozu-go/moco:${GITHUB_REF#refs/tags/v}
+ - uses: actions/checkout@v2
+ - run: docker build -t moco:dev .
+ - name: Login to ghcr.io
+ run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
+ - run: docker tag moco:dev ghcr.io/cybozu-go/moco:${GITHUB_REF#refs/tags/v}
+ - run: docker push ghcr.io/cybozu-go/moco:${GITHUB_REF#refs/tags/v}
  release:
  name: Release on GitHub
  needs: image
  runs-on: ubuntu-20.04
  steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-go@v2
- with:
- go-version: ${{ env.go-version }}
- - run: |
- make release-build
- - name: Create Release
- id: create_release
- uses: actions/create-release@v1
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- with:
- tag_name: ${{ github.ref }}
- release_name: Release ${{ github.ref }}
- body: |
- See [CHANGELOG.md](./CHANGELOG.md) for details.
- draft: false
- prerelease: ${{ contains(github.ref, '-') }}
- - name: Upload kubectl-moco for linux-amd64
- uses: actions/upload-release-asset@v1
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- with:
- upload_url: ${{ steps.create_release.outputs.upload_url }}
- asset_path: ./build/kubectl-moco-linux-amd64
- asset_name: kubectl-moco-linux-amd64
- asset_content_type: application/octet-stream
- - name: Upload kubectl-moco for windows-amd64
- uses: actions/upload-release-asset@v1
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- with:
- upload_url: ${{ steps.create_release.outputs.upload_url }}
- asset_path: ./build/kubectl-moco-windows-amd64.exe
- asset_name: kubectl-moco-windows-amd64.exe
- asset_content_type: application/octet-stream
- - name: Upload kubectl-moco for darwin-amd64
- uses: actions/upload-release-asset@v1
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- with:
- upload_url: ${{ steps.create_release.outputs.upload_url }}
- asset_path: ./build/kubectl-moco-darwin-amd64
- asset_name: kubectl-moco-darwin-amd64
- asset_content_type: application/octet-stream
+ - uses: actions/checkout@v2
+ - uses: actions/setup-go@v2
+ with:
+ go-version: ${{ env.go-version }}
+ - run: make release-build
+ - name: Create Release
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ tagname="${GITHUB_REF#refs/tags/}"
+ if echo ${{ github.ref }} | grep -q -e '-'; then prerelease=-p; fi
+ gh release create -t "Release $tagname" $prerelease \
+ -n "See [CHANGELOG.md](./CHANGELOG.md) for details." \
+ "$tagname" build/*

.gitignore

@@ -18,6 +18,4 @@
 # ignore binaries
 /bin
 /build
-
-# ignore tmp files (created by make kubebuilder)
-/tmp
+/testbin

CHANGELOG.md

@@ -5,6 +5,12 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.8.0] - 2021-04-27
+
+### Changed
+- Everything. There is no backward compatibility. (#228)
+- The older release must be uninstalled before installing this version.
+
 ## [0.7.0] - 2021-02-22
 
 Since v0.7.0, MOCO will no longer use CronJob for log rotation.
@@ -131,7 +137,8 @@ The `MySQLCluster` created by MOCO `< v0.5.0` has no compatibility with `>= v0.5
 
 - Bootstrap a vanilla MySQL cluster with no replicas (#2).
 
-[Unreleased]: https://github.com/cybozu-go/moco/compare/v0.7.0...HEAD
+[Unreleased]: https://github.com/cybozu-go/moco/compare/v0.8.0...HEAD
+[0.8.0]: https://github.com/cybozu-go/moco/compare/v0.8.0...v0.8.0
 [0.7.0]: https://github.com/cybozu-go/moco/compare/v0.6.0...v0.7.0
 [0.6.0]: https://github.com/cybozu-go/moco/compare/v0.5.1...v0.6.0
 [0.5.1]: https://github.com/cybozu-go/moco/compare/v0.5.0...v0.5.1

DEVELOP.md

@@ -0,0 +1,45 @@
+# How to develop MOCO
+
+## Running tests
+
+MOCO has the following 4 kinds of tests:
+
+1. Tests that do not depend on MySQL or Kubernetes
+2. `pkg/dbop` tests that depend on MySQL version
+3. Tests that depend on Kubernetes and therefore run by controller-runtime's envtest
+4. End-to-end tests
+
+To run these tests, use the following make targets respectively:
+
+1. `make test`
+2. `make test-dbop`
+3. `make envtest`
+4. Read [`e2e/README.md`](e2e/README.md)
+
+## Generated files
+
+Some files in the repository are auto-generated.
+
+- [`docs/crd_mysqlcluster.md`](docs/crd_mysqlcluster.md) is generated by `make apidoc`.
+- Some files under `config` are generated by `make manifests`.
+- `api/**/*.deepcopy.go` are generated by `make generate`.
+
+CI checks and fails if they need to be rebuilt.
+
+## Testing with unreleased moco-agent
+
+MOCO depends on [moco-agent][] that is released from a different repository.
+The dependency is therefore managed in `go.mod` file.
+
+To run e2e tests with an unreleased moco-agent, follow the instructions in
+[`e2e/README.md`](e2e/README.md).
+
+## Updating moco-agent
+
+Run `go get github.com/cybozu-go/moco-agent@latest`.
+
+## Updating fluent-bit
+
+Edit `FluentBitImage` in [`version.go`](versoin.go).
+
+[moco-agent]: https://github.com/cybozu-go/moco-agent

Dockerfile

@@ -1,20 +1,17 @@
 # Build the moco-controller binary
 FROM quay.io/cybozu/golang:1.16-focal as builder
 
-WORKDIR /workspace
-
 # Copy the go source
 COPY ./ .
 
 # Build
-RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o moco-controller ./cmd/moco-controller/main.go
+RUN CGO_ENABLED=0 go build -ldflags="-w -s" -o moco-controller ./cmd/moco-controller
 
 # stage2
 FROM scratch
 LABEL org.opencontainers.image.source https://github.com/cybozu-go/moco
 
-WORKDIR /
-COPY --from=builder /workspace/moco-controller ./
+COPY --from=builder /work/moco-controller ./
 USER 10000:10000
 
 ENTRYPOINT ["/moco-controller"]