diff --git a/.dockerignore b/.dockerignore index 0f04682..f784a1e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,7 @@ # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file # Ignore build and test binaries. +.git/ +.github/ +.husky/ bin/ testbin/ diff --git a/.earthlyignore b/.earthlyignore new file mode 100644 index 0000000..4807936 --- /dev/null +++ b/.earthlyignore @@ -0,0 +1,6 @@ +.git/ +.github/ +.husky/ +.vscode/ +bin/ +kind-logs-* \ No newline at end of file diff --git a/.github/workflows/_docker-build.yml b/.github/workflows/_docker-build.yml deleted file mode 100644 index 443e501..0000000 --- a/.github/workflows/_docker-build.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: docker build - -on: workflow_call - -permissions: - contents: read - pull-requests: read - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - IMAGE_TAG: ${{ github.sha }} -jobs: - build-and-push: - runs-on: ubuntu-latest - permissions: - packages: write - - steps: - - name: Harden Runner - uses: step-security/harden-runner@9b0655f430fba8c7001d4e38f8d4306db5c6e0ab - with: - egress-policy: audit - - - name: checkout repository - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 - - - name: log in to ghrc.io - uses: docker/login-action@1edf6180e07d2ffb423fc48a1a552855c0a1f508 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: build and container image - uses: docker/build-push-action@9472e9021074a3cb3279ba431598b8836d40433f - with: - context: . - push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} diff --git a/.github/workflows/_gocilint.yml b/.github/workflows/_gocilint.yml deleted file mode 100644 index 50a5108..0000000 --- a/.github/workflows/_gocilint.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: golangci-lint - -on: workflow_call - -permissions: - contents: read - pull-requests: read - -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f - - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 - - name: golangci-lint - uses: golangci/golangci-lint-action@c3ef0c370269e2a25b67c7f8e03d37e6cb106cb9 - with: - version: latest - args: --timeout 300s diff --git a/.github/workflows/_gosecscan.yml b/.github/workflows/_gosecscan.yml deleted file mode 100644 index d5f1d1b..0000000 --- a/.github/workflows/_gosecscan.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "gosec" - -on: workflow_call - -permissions: - contents: read - pull-requests: read - -jobs: - build: - name: scan - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - name: checkout repo - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 - - name: run gosec scan - uses: securego/gosec@a64cde55a4499d951566243783f204e94b9197ed - with: - args: "-exclude-dir=bin -exclude-dir=drivers -exclude-generated ./..." diff --git a/.github/workflows/_kuttl.yml b/.github/workflows/_kuttl.yml deleted file mode 100644 index 55bb297..0000000 --- a/.github/workflows/_kuttl.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: kuttl test - -on: workflow_call - -permissions: - contents: read - pull-requests: read - -env: - IMG: local/discoblocks:e2e - KUTTL: /usr/local/bin/kubectl-kuttl - KUBECTL_STORAGEOS: /usr/local/bin/kubectl-storageos - PLUGIN_PATH: ./kubectl-storageos - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - IMAGE_TAG: ${{ github.sha }} -jobs: - test: - name: test - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 - - name: Set up Go - uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f - with: - go-version: 1.18 - - name: Determine latest version - run: echo "LATEST_VERSION=$(curl -s https://api.github.com/repos/storageos/kubectl-storageos/releases/latest | grep tag_name | head -1 | cut -d'"' -f4 | tr -d v)" >> $GITHUB_ENV - - name: install storageos plugin - run: | - sudo curl -sSLo kubectl-storageos.tar.gz https://github.com/storageos/kubectl-storageos/releases/download/v${{ env.LATEST_VERSION }}/kubectl-storageos_${{ env.LATEST_VERSION }}_linux_amd64.tar.gz - sudo tar -xf kubectl-storageos.tar.gz - sudo chmod +x kubectl-storageos - sudo mv kubectl-storageos $KUBECTL_STORAGEOS - - name: install kuttl - run: | - sudo curl -Lo $KUTTL https://github.com/kudobuilder/kuttl/releases/download/v0.11.1/kubectl-kuttl_0.11.1_linux_x86_64 - sudo chmod +x $KUTTL - - name: pull and tag image - run: | - docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} ${{ env.IMG }} - - name: run kuttl - run: kubectl-kuttl test --config tests/e2e/kuttl/kuttl-config-1.23.yaml - - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 - if: ${{ always() }} - with: - name: kind-logs - path: kind-logs-* diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml deleted file mode 100644 index 10239fa..0000000 --- a/.github/workflows/_test.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: "test" - -on: workflow_call - -permissions: - contents: read - pull-requests: read - -jobs: - test: - name: Test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 - - name: Set up Go - uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f - with: - go-version: 1.18 - - name: Execute test - run: make manifests generate test diff --git a/.github/workflows/_trivy.yml b/.github/workflows/_trivy.yml deleted file mode 100644 index 5f39c77..0000000 --- a/.github/workflows/_trivy.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Vulnerability Scanning - -on: workflow_call - -permissions: - contents: read - pull-requests: read - -jobs: - build: - name: Build - runs-on: ubuntu-latest - permissions: - security-events: write - steps: - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@f39d29766a1eb7432c47f6bb7b64ed70b2241524 - with: - image-ref: ghcr.io/ondat/discoblocks:${{ github.sha }} - format: 'template' - template: '@/contrib/sarif.tpl' - output: 'trivy-results.sarif' - severity: 'CRITICAL,HIGH' - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@1fc1008278d05ba9455caf083444e6c5a1a3cfd8 - with: - sarif_file: 'trivy-results.sarif' diff --git a/.github/workflows/e2e-on-pr.yaml b/.github/workflows/e2e-on-pr.yaml new file mode 100644 index 0000000..fd8e6f2 --- /dev/null +++ b/.github/workflows/e2e-on-pr.yaml @@ -0,0 +1,60 @@ +# in test don't use yet + +name: end-2-end build + +on: + pull_request: + branches: [ main ] + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + actions: read + security-events: write + packages: write + +concurrency: + group: ci-e2e-${{ github.ref }}-1 + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + IMAGE_TAG: ${{ github.sha }} +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + security-events: write + packages: write + steps: + - name: harden runner + uses: step-security/harden-runner@9b0655f430fba8c7001d4e38f8d4306db5c6e0ab + with: + egress-policy: audit + - name: checkout repository + uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 + - name: log in to ghrc.io + uses: docker/login-action@1edf6180e07d2ffb423fc48a1a552855c0a1f508 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: run golangci-lint + run: make lint + - name: run gosec scan + run: make gosec + - name: run test + run: make test + - name: run e2e test + run: make e2e-test + - name: run image scan + run: make scan-image + - name: generate bundle manifest + run: make bundle + - name: upload Trivy scan results to GitHub Security tab + if: always() + uses: github/codeql-action/upload-sarif@1fc1008278d05ba9455caf083444e6c5a1a3cfd8 + with: + sarif_file: 'trivy-results.sarif' \ No newline at end of file diff --git a/.github/workflows/e2e-on-pr.yml b/.github/workflows/e2e-on-pr.yml deleted file mode 100644 index 00b857a..0000000 --- a/.github/workflows/e2e-on-pr.yml +++ /dev/null @@ -1,41 +0,0 @@ -# in test don't use yet - -name: end-2-end build - -on: - pull_request: - branches: [ main ] - workflow_dispatch: - -permissions: - contents: read - pull-requests: read - actions: read - security-events: write - packages: write - -concurrency: - group: ci-e2e-${{ github.ref }}-1 - cancel-in-progress: true - -jobs: - test: - uses: ./.github/workflows/_test.yml - - golangci-lint: - uses: ./.github/workflows/_gocilint.yml - - gosec-scanning: - uses: ./.github/workflows/_gosecscan.yml - - image-build: - uses: ./.github/workflows/_docker-build.yml - needs: [test, golangci-lint, gosec-scanning] - - image-vulnerability-scan: - uses: ./.github/workflows/_trivy.yml - needs: image-build - - kuttl: - uses: ./.github/workflows/_kuttl.yml - needs: image-build diff --git a/.github/workflows/go-lint-scan-pull_request.yaml b/.github/workflows/go-lint-scan-pull_request.yaml deleted file mode 100644 index 507c10d..0000000 --- a/.github/workflows/go-lint-scan-pull_request.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: golangci-lint - -on: - workflow_dispatch: - -permissions: read-all - -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f - - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 - - name: golangci-lint - uses: golangci/golangci-lint-action@4b237a63e5299c390fb934f06848da715fdde07c - with: - version: latest - args: --issues-exit-code=0 - only-new-issues: true - skip-cache: true - skip-pkg-cache: true - skip-build-cache: true - diff --git a/.github/workflows/gosec-scanner-on-pull_request.yaml b/.github/workflows/gosec-scanner-on-pull_request.yaml deleted file mode 100644 index 3e3a9b2..0000000 --- a/.github/workflows/gosec-scanner-on-pull_request.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: "gosec" - -on: - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -permissions: read-all - -jobs: - build: - runs-on: ubuntu-latest - env: - GO111MODULE: on - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - name: checkout repo - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 - - - name: run gosec scan - uses: securego/gosec@89dfdc0c972655dfaa4eec7a115742a28e0bc216 - with: - args: "./..." diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 001ded1..6abd255 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -21,43 +21,6 @@ env: IMAGE_NAME: ${{ github.repository }} IMAGE_TAG: ${{ github.sha }} jobs: - test: - uses: ./.github/workflows/_test.yml - - golangci-lint: - uses: ./.github/workflows/_gocilint.yml - - gosec-scanning: - uses: ./.github/workflows/_gosecscan.yml - - image-build: - uses: ./.github/workflows/_docker-build.yml - needs: [test, golangci-lint, gosec-scanning] - - job-image-build: - run: | - ORIGINAL=$(grep -r "var JobImage" pkg/utils/ | head -1 | cut -d\" -f2) - docker pull ${ORIGINAL} - docker tag ${ORIGINAL} ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-job:${{ env.IMAGE_TAG }} - docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-job:${{ env.IMAGE_TAG }} - needs: image-build - - proxy-image-build: - run: | - ORIGINAL=$(grep -r "var ProxyImage" pkg/utils/ | head -1 | cut -d\" -f2) - docker pull ${ORIGINAL} - docker tag ${ORIGINAL} ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-proxy:${{ env.IMAGE_TAG }} - docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-proxy:${{ env.IMAGE_TAG }} - needs: image-build - - image-vulnerability-scan: - uses: ./.github/workflows/_trivy.yml - needs: image-build - - kuttl: - uses: ./.github/workflows/_kuttl.yml - needs: [image-build, job-image-build, proxy-image-build] - auto-pre-release: if: startsWith(github.ref, 'refs/tags/v') && (contains(github.ref_name, '-alpha.') || contains(github.ref_name, '-beta.')) runs-on: ubuntu-latest @@ -65,7 +28,11 @@ jobs: contents: write packages: write steps: - - name: Checkout + - name: harden runner + uses: step-security/harden-runner@9b0655f430fba8c7001d4e38f8d4306db5c6e0ab + with: + egress-policy: audit + - name: checkout uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 with: fetch-depth: 0 @@ -75,29 +42,22 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Tag and push image - run: | - docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} - docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} - - name: Generate bundle manifest - run: make bundle - env: - IMG: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} - - name: Compress Kustomize manifests - run: (cd config ; tar -czvf ../discoblocks-kustomize.tar.gz .) - - name: Set since tag + - name: run e2e workflow + uses: ./.github/workflows/e2e-on-pr.yaml + - name: push images + run: make docker-push + - name: set since tag run: echo "SINCE_TAG=$(git describe --tags --always $(git rev-list --tags) | grep -e '^v[0-9]+*\.[0-9]+*\.[0-9]+*$' | head -1)" >> $GITHUB_ENV - - name: "Generate release changelog" + - name: generate release changelog uses: heinrichreimer/github-changelog-generator-action@6f5b9494dd265d6fb7243a10c53dc0169c55f247 with: token: ${{ secrets.GITHUB_TOKEN }} base: .github/RELEASE_TEMPLATE.md sinceTag: ${{ env.SINCE_TAG }} simpleList: true - - name: Fix version in changelog + - name: fix version in changelog run: sed -i "s/#VERSION#/${{ github.ref_name }}/g" CHANGELOG.md - - name: Create pre-release + - name: create pre-release uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 with: prerelease: true @@ -105,7 +65,6 @@ jobs: files: | discoblocks-bundle.yaml discoblocks-kustomize.tar.gz - needs: [image-vulnerability-scan, kuttl] auto-release: if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-alpha.') && !contains(github.ref_name, '-beta.') @@ -114,7 +73,11 @@ jobs: contents: write packages: write steps: - - name: Checkout + - name: harden runner + uses: step-security/harden-runner@9b0655f430fba8c7001d4e38f8d4306db5c6e0ab + with: + egress-policy: audit + - name: checkout uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846 with: fetch-depth: 0 @@ -124,34 +87,26 @@ jobs: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Tag and push image - run: | - docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - docker tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} - docker push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} - - name: Generate bundle manifest - run: make bundle - env: - IMG: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} - - name: Compress Kustomize manifests - run: (cd config ; tar -czvf ../discoblocks-kustomize.tar.gz .) - - name: Set since tag + - name: run e2e workflow + uses: ./.github/workflows/e2e-on-pr.yaml + - name: push images + run: make docker-push + - name: set since tag run: echo "SINCE_TAG=$(git describe --tags --always $(git rev-list --tags) | grep -e '^v[0-9]+*\.[0-9]+*\.[0-9]+*$' | head -2 | tail -1)" >> $GITHUB_ENV - - name: "Generate release changelog" + - name: generate release changelog uses: heinrichreimer/github-changelog-generator-action@6f5b9494dd265d6fb7243a10c53dc0169c55f247 with: token: ${{ secrets.GITHUB_TOKEN }} base: .github/RELEASE_TEMPLATE.md sinceTag: ${{ env.SINCE_TAG }} simpleList: true - - name: Fix version in changelog + - name: fix version in changelog run: sed -i "s/#VERSION#/${{ github.ref_name }}/g" CHANGELOG.md - - name: Create pre-release + - name: create pre-release uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 with: prerelease: false body_path: CHANGELOG.md files: | discoblocks-bundle.yaml - discoblocks-kustomize.tar.gz - needs: [image-vulnerability-scan, kuttl] \ No newline at end of file + discoblocks-kustomize.tar.gz \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6e3fbec..ab2c3d6 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ testbin/* *.test kind-logs-* kubeconfig +trivy-results.sarif # Output of the go coverage tool, specifically when used with LiteIDE *.out diff --git a/.golangci.yaml b/.golangci.yaml index 5434010..cb2b7c5 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -19,7 +19,6 @@ linters-settings: require-specific: true linters: - disable-all: true enable: - bodyclose - depguard @@ -53,10 +52,6 @@ linters: - unparam - whitespace - unused - # Deprecated - - varcheck - - structcheck - - deadcode run: issues-exit-code: 0 diff --git a/.husky/hooks/pre-push b/.husky/hooks/pre-push index e0d0e98..35a3fa3 100755 --- a/.husky/hooks/pre-push +++ b/.husky/hooks/pre-push @@ -13,6 +13,7 @@ if git status --short | grep -qv "??"; then trap unstash EXIT fi -make manifests generate test +make earthly manifests generate +./bin/earthly -P +all git diff --exit-code --quiet || (git status && exit 1) diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index e53e86e..0000000 --- a/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# Build CSI drivers -FROM tinygo/tinygo@sha256:65dc1c3e54f88aabe1efe073c3aadb1393593a56355a6ac03df5f18e6c3855dd as drivers - -COPY drivers/ /go/src - -RUN cd /go/src/csi.storageos.com ; go mod tidy && tinygo build -o main.wasm -target wasi --no-debug main.go -RUN cd /go/src/ebs.csi.aws.com ; go mod tidy && tinygo build -o main.wasm -target wasi --no-debug main.go - -# Build the manager binary -FROM golang@sha256:5b75b529da0f2196ee8561a90e5b99aceee56e125c6ef09a3da4e32cf3cc6c20 as builder - -WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum -# cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN go mod download - -# Copy the go source -COPY main.go main.go -COPY api/ api/ -COPY controllers/ controllers/ -COPY mutators/ mutators/ -COPY pkg/ pkg/ -COPY schedulers/ schedulers/ - -# Build -RUN GOOS=linux GOARCH=amd64 go build -a -o manager main.go - -# Use UBI as minimal base image to package the manager binary -FROM redhat/ubi8-micro@sha256:4f6f8db9a6dc949d9779a57c43954b251957bd4d019a37edbbde8ed5228fe90a - -LABEL org.opencontainers.image.title "Discoblocks" -LABEL org.opencontainers.image.vendor "Discoblocks.io" -LABEL org.opencontainers.image.licenses "Apache-2.0 License" -LABEL org.opencontainers.image.source "https://github.com/ondat/discoblocks" -LABEL org.opencontainers.image.description "Discoblocks is an open-source declarative disk configuration system for Kubernetes helping to automate CRUD (Create, Read, Update, Delete) operations for cloud disk device resources attached to Kubernetes cluster nodes." -LABEL org.opencontainers.image.documentation "https://github.com/ondat/discoblocks/wiki" - -WORKDIR / -COPY --from=drivers /go/src /drivers -COPY --from=builder /workspace/manager . -COPY --from=builder /go/pkg/mod/github.com/wasmerio/wasmer-go@v1.0.4/wasmer/packaged/lib/linux-amd64/libwasmer.so /lib64 -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt - -USER 65532:65532 - -ENTRYPOINT ["/manager"] diff --git a/Earthfile b/Earthfile new file mode 100644 index 0000000..11b4580 --- /dev/null +++ b/Earthfile @@ -0,0 +1,202 @@ +VERSION --use-cache-command 0.7 +FROM golang:1.18 +WORKDIR /workdir +ARG --global KUBE_VERSION=1.23 +ARG --global REGISTRY=ghcr.io +ARG --global IMAGE_NAME=discoblocks +ARG --global IMAGE_TAG=latest + +all: + WAIT + BUILD +go-lint + BUILD +go-test + BUILD +bundle + END + WAIT + BUILD +go-sec + END + WAIT + BUILD +scan-image + END + +go-lint: + FROM earthly/dind:alpine + COPY . ./workdir + WITH DOCKER --pull golangci/golangci-lint:v1.51.0 + RUN docker run -w /workdir -v /workdir:/workdir golangci/golangci-lint:v1.51.0 golangci-lint run --timeout 300s + END + +go-sec: + FROM earthly/dind:alpine + COPY . ./workdir + WITH DOCKER --pull securego/gosec:2.15.0 + RUN docker run -w /workdir -v /workdir:/workdir securego/gosec:2.15.0 -exclude-dir=bin -exclude-dir=drivers -exclude-generated ./... + END + +go-test: + FROM +deps-go-build + CACHE $HOME/.cache/go-build + COPY . ./ + WITH DOCKER --pull tinygo/tinygo:0.23.0 + RUN make _test + END + +e2e-test: + FROM earthly/dind:alpine + RUN apk add make bash + WORKDIR /workdir + ENV KUSTOMIZE=/usr/local/bin/kustomize + ARG TIMEOUT=240 + COPY --dir +deps-tooling/* /usr/local + COPY Makefile ./ + COPY --dir config ./ + COPY --dir tests ./ + WITH DOCKER --load local/discoblocks:e2e=+build-image --load local/discoblocks:job-e2e=+build-job-image --load local/discoblocks:proxy-e2e=+build-proxy-image + RUN kubectl-kuttl test --timeout ${TIMEOUT} --config tests/e2e/kuttl/kuttl-config-${KUBE_VERSION}.yaml || touch /failure + END + + IF [ -d kind-logs-* ] + SAVE ARTIFACT --if-exists kind-logs-* AS LOCAL ./ + END + + IF [ -f /failure ] + RUN echo "e2e test run failed" && exit 1 + END + +build-drivers: + FROM tinygo/tinygo:0.23.0 + COPY --dir drivers /go/src + WAIT + RUN cd /go/src/csi.storageos.com ; go mod tidy && tinygo build -o main.wasm -target wasi --no-debug main.go + END + WAIT + RUN cd /go/src/ebs.csi.aws.com ; go mod tidy && tinygo build -o main.wasm -target wasi --no-debug main.go + END + + SAVE ARTIFACT /go/src/csi.storageos.com /drivers/csi.storageos.com + SAVE ARTIFACT /go/src/ebs.csi.aws.com /drivers/ebs.csi.aws.com + +build-operator: + FROM +deps-go + CACHE $HOME/.cache/go-build + COPY main.go ./ + COPY --dir api ./ + COPY --dir controllers ./ + COPY --dir mutators ./ + COPY --dir pkg ./ + COPY --dir schedulers ./ + RUN GOOS=linux GOARCH=amd64 go build -a -o manager main.go + + SAVE ARTIFACT manager + SAVE ARTIFACT /go/pkg/mod /go/pkg/mod + SAVE ARTIFACT /etc/ssl/certs /etc/ssl/certs + +build-all-images: + WAIT + BUILD +build-image + END + WAIT + BUILD +build-job-image + END + WAIT + BUILD +build-proxy-image + END + +build-job-image: + FROM DOCKERFILE -f +deps-job-image/Dockerfile . + + SAVE IMAGE --push ${REGISTRY}/${IMAGE_NAME}:job-${IMAGE_TAG} + +build-proxy-image: + FROM DOCKERFILE -f +deps-proxy-image/Dockerfile . + + SAVE IMAGE --push ${REGISTRY}/${IMAGE_NAME}:proxy-${IMAGE_TAG} + +build-image: + FROM redhat/ubi8-micro@sha256:4f6f8db9a6dc949d9779a57c43954b251957bd4d019a37edbbde8ed5228fe90a + WORKDIR / + LABEL org.opencontainers.image.title="Discoblocks" + LABEL org.opencontainers.image.vendor="Discoblocks.io" + LABEL org.opencontainers.image.licenses="Apache-2.0 License" + LABEL org.opencontainers.image.source="https://github.com/ondat/discoblocks" + LABEL org.opencontainers.image.description="Discoblocks is an open-source declarative disk configuration system for Kubernetes helping to automate CRUD (Create, Read, Update, Delete) operations for cloud disk device resources attached to Kubernetes cluster nodes." + LABEL org.opencontainers.image.documentation="https://github.com/ondat/discoblocks/wiki" + COPY +build-drivers/drivers /drivers + COPY +build-operator/manager /manager + COPY +build-operator/go/pkg/mod/github.com/wasmerio/wasmer-go@v1.0.4/wasmer/packaged/lib/linux-amd64/libwasmer.so /lib64 + COPY +build-operator/etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + USER 65532:65532 + ENTRYPOINT ["/manager"] + + SAVE IMAGE --push ${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG} + +scan-image: + FROM earthly/dind:alpine + WITH DOCKER --load local/discoblocks:trivy=+build-image --pull aquasec/trivy:0.38.1 + RUN docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy:0.38.1 image -f sarif -o trivy-results.sarif -s 'CRITICAL,HIGH' local/discoblocks:trivy + END + + SAVE ARTIFACT --if-exists trivy-results.sarif AS LOCAL trivy-results.sarif + +bundle: + FROM +deps-go-build + ENV KUSTOMIZE=/usr/local/bin/kustomize + COPY --dir +deps-tooling/* /usr/local + COPY Makefile ./ + COPY --dir config ./ + RUN sed -i "s|$(grep jobContainerImage config/manager/controller_manager_config.yaml | awk '{print $2}')|${REGISTRY}/${IMAGE_NAME}:job-${IMAGE_TAG}|" config/manager/controller_manager_config.yaml + RUN sed -i "s|$(grep proxyContainerImage config/manager/controller_manager_config.yaml | awk '{print $2}')|${REGISTRY}/${IMAGE_NAME}:proxy-${IMAGE_TAG}|" config/manager/controller_manager_config.yaml + RUN make _bundle + + SAVE ARTIFACT --if-exists discoblocks-bundle.yaml AS LOCAL discoblocks-bundle.yaml + SAVE ARTIFACT --if-exists discoblocks-kustomize.tar.gz AS LOCAL discoblocks-kustomize.tar.gz + +deps-go: + COPY go.mod go.sum ./ + RUN go mod download + +deps-go-build: + FROM +deps-go + COPY Makefile ./ + RUN make controller-gen envtest + +deps-tooling: + SAVE ARTIFACT /usr/local/go / + SAVE ARTIFACT /usr/local/go/bin / + + WAIT + ARG LATEST_VERSION=$(curl -s https://api.github.com/repos/storageos/kubectl-storageos/releases/latest | grep tag_name | head -1 | awk -F'\"' '{ print $4 }' | tr -d v) + RUN curl -sSL https://github.com/storageos/kubectl-storageos/releases/download/v${LATEST_VERSION}/kubectl-storageos_${LATEST_VERSION}_linux_amd64.tar.gz | tar -xz + RUN chmod +x kubectl-storageos + SAVE ARTIFACT kubectl-storageos /bin/kubectl-storageos + END + + WAIT + RUN curl -sLo kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v0.15.0/kubectl-kuttl_0.15.0_linux_x86_64 + RUN chmod +x kubectl-kuttl + SAVE ARTIFACT kubectl-kuttl /bin/kubectl-kuttl + END + + WAIT + RUN curl -sLO https://dl.k8s.io/release/v${KUBE_VERSION}.0/bin/linux/amd64/kubectl + RUN chmod +x kubectl + SAVE ARTIFACT kubectl /bin/kubectl + END + + WAIT + COPY Makefile ./ + RUN make kustomize + SAVE ARTIFACT bin/kustomize /bin/kustomize + END + +deps-job-image: + COPY config/manager/controller_manager_config.yaml ./ + RUN echo "FROM $(grep "jobContainerImage" controller_manager_config.yaml | awk '{print $2}')" > Dockerfile + + SAVE ARTIFACT Dockerfile + +deps-proxy-image: + COPY config/manager/controller_manager_config.yaml ./ + RUN echo "FROM $(grep "proxyContainerImage" controller_manager_config.yaml | awk '{print $2}')" > Dockerfile + + SAVE ARTIFACT Dockerfile \ No newline at end of file diff --git a/Makefile b/Makefile index 01cb50c..fad9476 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,15 @@ # Image URL to use all building/pushing image targets -IMG ?= discoblocks:latest +REGISTRY ?= ghcr.io +IMAGE_NAME ?= discoblocks +IMAGE_TAG ?= latest +IMG = $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG) + +E2E_TIMEOUT ?= 240 + # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.23 -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - LDF_FLAGS = -X github.com/ondat/discoblocks/pkg/drivers.DriversDir=$(PWD)/drivers # Setting SHELL to bash allows bash commands to be executed by recipes. @@ -59,12 +58,31 @@ fmt: ## Run go fmt against code. vet: ## Run go vet against code. go vet ./... +.PHONY: lint +lint: earthly + $(EARTHLY) -P +go-lint + +.PHONY: gosec +gosec: earthly + $(EARTHLY) -P +go-sec + .PHONY: test -test: manifests generate fmt vet build-drivers envtest ## Run tests. +test: earthly ## Run tests. + $(EARTHLY) -P +go-test + +_test: manifests generate fmt vet build-drivers envtest KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -ldflags "$(LDF_FLAGS)" ./... -coverprofile cover.out +.PHONY: e2e-test +e2e-test: earthly ## Run e2e tests. + $(EARTHLY) -P +e2e-test --REGISTRY=$(REGISTRY) --IMAGE_NAME=$(IMAGE_NAME) --IMAGE_TAG=$(IMAGE_TAG) --TIMEOUT=$(E2E_TIMEOUT) + ##@ Build +.PHONY: tidy +tidy: + go mod tidy + .PHONY: build build: generate fmt vet ## Build manager binary. go build -ldflags "$(LDF_FLAGS)" -o bin/manager main.go @@ -74,18 +92,22 @@ run: manifests generate fmt vet ## Run a controller from your host. go run -ldflags "$(LDF_FLAGS)" ./main.go .PHONY: docker-build -docker-build: test ## Build docker image with the manager. - docker build -t ${IMG} . +docker-build: earthly ## Build docker image with the manager. + $(EARTHLY) -P +build-all-images --REGISTRY=$(REGISTRY) --IMAGE_NAME=$(IMAGE_NAME) --IMAGE_TAG=$(IMAGE_TAG) .PHONY: docker-push docker-push: ## Push docker image with the manager. - docker push ${IMG} + $(EARTHLY) --push -P +build-all-images --REGISTRY=$(REGISTRY) --IMAGE_NAME=$(IMAGE_NAME) --IMAGE_TAG=$(IMAGE_TAG) .PHONY: build-drivers -build-drivers: ## Build CSI driver WASIs +build-drivers: ## Build CSI driver WASIs. docker run -v $(PWD)/drivers:/go/src -w /go/src/csi.storageos.com tinygo/tinygo:0.23.0 bash -c "go mod tidy && tinygo build -o main.wasm -target wasi --no-debug main.go" docker run -v $(PWD)/drivers:/go/src -w /go/src/ebs.csi.aws.com tinygo/tinygo:0.23.0 bash -c "go mod tidy && tinygo build -o main.wasm -target wasi --no-debug main.go" +.PHONY: scan-image +scan-image: earthly ## Run image scan. + $(EARTHLY) -P +scan-image + ##@ Deployment ifndef ignore-not-found @@ -93,15 +115,21 @@ ifndef ignore-not-found endif .PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. +install: manifests kustomize _install ## Install CRDs into the K8s cluster specified in ~/.kube/config. + +_install: $(KUSTOMIZE) build config/crd | kubectl apply -f - .PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. +uninstall: manifests kustomize _uninstall ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + +_uninstall: $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. +deploy: manifests kustomize _deploy ## Deploy controller to the K8s cluster specified in ~/.kube/config. + +_deploy: cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - cd config/manager && $(KUSTOMIZE) edit set image controller=discoblocks:latest @@ -111,11 +139,14 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: bundle -bundle: manifests kustomize ## Generates Kubernetes manifests +bundle: earthly ## Generates Kubernetes manifests + $(EARTHLY) -P +bundle --REGISTRY=$(REGISTRY) --IMAGE_NAME=$(IMAGE_NAME) --IMAGE_TAG=$(IMAGE_TAG) + +_bundle: rm -rf discoblocks-bundle.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default > discoblocks-bundle.yaml - cd config/manager && $(KUSTOMIZE) edit set image controller=discoblocks:latest + (cd config ; tar -czvf ../discoblocks-kustomize.tar.gz .) .PHONY: deploy-prometheus deploy-prometheus: ## Deploy prometheus to the K8s cluster specified in ~/.kube/config. @@ -137,32 +168,48 @@ deploy-cert-manager: ## Deploy cert manager to the K8s cluster specified in ~/.k undeploy-cert-manager: ## Undeploy cert manager from the K8s cluster specified in ~/.kube/config. kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.0/cert-manager.yaml -CONTROLLER_GEN = $(shell pwd)/bin/controller-gen +CONTROLLER_GEN ?= $(shell pwd)/bin/controller-gen .PHONY: controller-gen controller-gen: ## Download controller-gen locally if necessary. $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0) -KUSTOMIZE = $(shell pwd)/bin/kustomize +KUSTOMIZE ?= $(shell pwd)/bin/kustomize .PHONY: kustomize kustomize: ## Download kustomize locally if necessary. $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.7) -ENVTEST = $(shell pwd)/bin/setup-envtest +ENVTEST ?= $(shell pwd)/bin/setup-envtest .PHONY: envtest envtest: ## Download envtest-setup locally if necessary. $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) -KUBEBUILDER = $(shell pwd)/bin/kubebuilder +KUBEBUILDER ?= $(shell pwd)/bin/kubebuilder .PHONY: kubebuilder kubebuilder: ## Download kubebuilder locally if necessary. - curl -L -o $(KUBEBUILDER) https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.3.0/kubebuilder_$(shell uname | tr '[:upper:]' '[:lower:]')_amd64 +ifeq (,$(wildcard $(KUBEBUILDER))) + curl -sL -o $(KUBEBUILDER) https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.3.0/kubebuilder_$(shell uname | tr '[:upper:]' '[:lower:]')_amd64 chmod +x $(KUBEBUILDER) +endif -HUSKY = $(shell pwd)/bin/husky +HUSKY ?= $(shell pwd)/bin/husky .PHONY: husky husky: ## Download husky locally if necessary. $(call go-get-tool,$(HUSKY),github.com/automation-co/husky@v0.2.14) +EARTHLY ?= $(shell pwd)/bin/earthly +earthly: +ifeq (,$(wildcard $(EARTHLY))) + curl -sL https://github.com/earthly/earthly/releases/download/v0.7.1/earthly-linux-amd64 -o $(EARTHLY) + chmod +x $(EARTHLY) +endif + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) define go-get-tool diff --git a/PROJECT b/PROJECT index 2b2ebcd..c1a38c5 100644 --- a/PROJECT +++ b/PROJECT @@ -16,4 +16,11 @@ resources: webhooks: validation: true webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + group: config.discoblocks.ondat.io + kind: OperatorConfig + path: github.com/ondat/discoblocks/api/config.discoblocks.ondat.io/v1 + version: v1 version: "3" diff --git a/api/config.discoblocks.io/v1/groupversion_info.go b/api/config.discoblocks.ondat.io/v1/groupversion_info.go similarity index 76% rename from api/config.discoblocks.io/v1/groupversion_info.go rename to api/config.discoblocks.ondat.io/v1/groupversion_info.go index 16e9832..97588f1 100644 --- a/api/config.discoblocks.io/v1/groupversion_info.go +++ b/api/config.discoblocks.ondat.io/v1/groupversion_info.go @@ -1,6 +1,6 @@ -// Package v1 contains API Schema definitions for the config.discoblocks.io v1 API group +// Package v1 contains API Schema definitions for the config.discoblocks.ondat.io v1 API group // +kubebuilder:object:generate=true -// +groupName=config.discoblocks.io +// +groupName=config.discoblocks.ondat.io package v1 import ( @@ -10,7 +10,7 @@ import ( var ( // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "config.discoblocks.io", Version: "v1"} + GroupVersion = schema.GroupVersion{Group: "config.discoblocks.ondat.io", Version: "v1"} // SchemeBuilder is used to add go types to the GroupVersionKind scheme SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} diff --git a/api/config.discoblocks.io/v1/operatorconfig_types.go b/api/config.discoblocks.ondat.io/v1/operatorconfig_types.go similarity index 100% rename from api/config.discoblocks.io/v1/operatorconfig_types.go rename to api/config.discoblocks.ondat.io/v1/operatorconfig_types.go diff --git a/api/config.discoblocks.io/v1/zz_generated.deepcopy.go b/api/config.discoblocks.ondat.io/v1/zz_generated.deepcopy.go similarity index 100% rename from api/config.discoblocks.io/v1/zz_generated.deepcopy.go rename to api/config.discoblocks.ondat.io/v1/zz_generated.deepcopy.go diff --git a/config/crd/bases/config.discoblocks.io_operatorconfigs.yaml b/config/crd/bases/config.discoblocks.io_operatorconfigs.yaml index 2b91296..7ab84de 100644 --- a/config/crd/bases/config.discoblocks.io_operatorconfigs.yaml +++ b/config/crd/bases/config.discoblocks.io_operatorconfigs.yaml @@ -5,9 +5,9 @@ metadata: annotations: controller-gen.kubebuilder.io/version: v0.8.0 creationTimestamp: null - name: operatorconfigs.config.discoblocks.io + name: operatorconfigs.config.discoblocks.ondat.io spec: - group: config.discoblocks.io + group: config.discoblocks.ondat.io names: kind: OperatorConfig listKind: OperatorConfigList diff --git a/config/crd/bases/config.discoblocks.ondat.io_operatorconfigs.yaml b/config/crd/bases/config.discoblocks.ondat.io_operatorconfigs.yaml new file mode 100644 index 0000000..7ab84de --- /dev/null +++ b/config/crd/bases/config.discoblocks.ondat.io_operatorconfigs.yaml @@ -0,0 +1,203 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: operatorconfigs.config.discoblocks.ondat.io +spec: + group: config.discoblocks.ondat.io + names: + kind: OperatorConfig + listKind: OperatorConfigList + plural: operatorconfigs + singular: operatorconfig + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: OperatorConfig is the Schema for the operatorconfigs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + cacheNamespace: + description: "CacheNamespace if specified restricts the manager's cache + to watch objects in the desired namespace Defaults to all namespaces + \n Note: If a namespace is specified, controllers can still Watch for + a cluster-scoped resource (e.g Node). For namespaced resources the + cache will only hold objects from the desired namespace." + type: string + controller: + description: Controller contains global configuration options for controllers + registered within this manager. + properties: + cacheSyncTimeout: + description: CacheSyncTimeout refers to the time limit set to wait + for syncing caches. Defaults to 2 minutes if not set. + format: int64 + type: integer + groupKindConcurrency: + additionalProperties: + type: integer + description: "GroupKindConcurrency is a map from a Kind to the number + of concurrent reconciliation allowed for that controller. \n When + a controller is registered within this manager using the builder + utilities, users have to specify the type the controller reconciles + in the For(...) call. If the object's kind passed matches one of + the keys in this map, the concurrency for that controller is set + to the number specified. \n The key is expected to be consistent + in form with GroupKind.String(), e.g. ReplicaSet in apps group (regardless + of version) would be `ReplicaSet.apps`." + type: object + type: object + gracefulShutDown: + description: GracefulShutdownTimeout is the duration given to runnable + to stop before the manager actually returns on stop. To disable graceful + shutdown, set to time.Duration(0) To use graceful shutdown without timeout, + set to a negative duration, e.G. time.Duration(-1) The graceful shutdown + is skipped for safety reasons in case the leader election lease is lost. + type: string + health: + description: Health contains the controller health configuration + properties: + healthProbeBindAddress: + description: HealthProbeBindAddress is the TCP address that the controller + should bind to for serving health probes + type: string + livenessEndpointName: + description: LivenessEndpointName, defaults to "healthz" + type: string + readinessEndpointName: + description: ReadinessEndpointName, defaults to "readyz" + type: string + type: object + jobContainerImage: + description: JobContainerImage is the container image for volume management + operations + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + leaderElection: + description: LeaderElection is the LeaderElection config to be used when + configuring the manager.Manager leader election + properties: + leaderElect: + description: leaderElect enables a leader election client to gain + leadership before executing the main loop. Enable this when running + replicated components for high availability. + type: boolean + leaseDuration: + description: leaseDuration is the duration that non-leader candidates + will wait after observing a leadership renewal until attempting + to acquire leadership of a led but unrenewed leader slot. This is + effectively the maximum duration that a leader can be stopped before + it is replaced by another candidate. This is only applicable if + leader election is enabled. + type: string + renewDeadline: + description: renewDeadline is the interval between attempts by the + acting master to renew a leadership slot before it stops leading. + This must be less than or equal to the lease duration. This is only + applicable if leader election is enabled. + type: string + resourceLock: + description: resourceLock indicates the resource object type that + will be used to lock during leader election cycles. + type: string + resourceName: + description: resourceName indicates the name of resource object that + will be used to lock during leader election cycles. + type: string + resourceNamespace: + description: resourceName indicates the namespace of resource object + that will be used to lock during leader election cycles. + type: string + retryPeriod: + description: retryPeriod is the duration the clients should wait between + attempting acquisition and renewal of a leadership. This is only + applicable if leader election is enabled. + type: string + required: + - leaderElect + - leaseDuration + - renewDeadline + - resourceLock + - resourceName + - resourceNamespace + - retryPeriod + type: object + metadata: + type: object + metrics: + description: Metrics contains thw controller metrics configuration + properties: + bindAddress: + description: BindAddress is the TCP address that the controller should + bind to for serving prometheus metrics. It can be set to "0" to + disable the metrics serving. + type: string + type: object + mutatorStrictMode: + description: MutatorStrictMode defines mutator's behavior on case of Discoblock + errors + type: boolean + proxyContainerImage: + description: JobContainerImage is the container image of volume metrics + sidecar + type: string + schedulerStrictMode: + description: SchedulerStrictMode defines scheduler's behavior on case + of Discoblock errors + type: boolean + supportedCsiDrivers: + description: SupportedCsiDrivers list of supported CSI driver IDs + items: + type: string + type: array + syncPeriod: + description: SyncPeriod determines the minimum frequency at which watched + resources are reconciled. A lower period will correct entropy more quickly, + but reduce responsiveness to change if there are many watched resources. + Change this value only if you know what you are doing. Defaults to 10 + hours if unset. there will a 10 percent jitter between the SyncPeriod + of all controllers so that all controllers will not send list requests + simultaneously. + type: string + webhook: + description: Webhook contains the controllers webhook configuration + properties: + certDir: + description: CertDir is the directory that contains the server key + and certificate. if not set, webhook server would look up the server + key and certificate in {TempDir}/k8s-webhook-server/serving-certs. + The server key and certificate must be named tls.key and tls.crt, + respectively. + type: string + host: + description: Host is the hostname that the webhook server binds to. + It is used to set webhook.Server.Host. + type: string + port: + description: Port is the port that the webhook server serves at. It + is used to set webhook.Server.Port. + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 3f63f10..f6b06e7 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,6 +3,7 @@ # It should be run by config/default resources: - bases/discoblocks.ondat.io_diskconfigs.yaml +# - bases/config.discoblocks.ondat.io_operatorconfigs.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml index 6c40015..bebbac4 100644 --- a/config/default/manager_config_patch.yaml +++ b/config/default/manager_config_patch.yaml @@ -9,7 +9,7 @@ spec: containers: - name: manager args: - - "--config=controller_manager_config.yaml" + - "--config=/controller_manager_config.yaml" volumeMounts: - name: manager-config mountPath: /controller_manager_config.yaml diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml index b3ae329..cb48e8c 100644 --- a/config/manager/controller_manager_config.yaml +++ b/config/manager/controller_manager_config.yaml @@ -1,5 +1,5 @@ -apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 -kind: ControllerManagerConfig +apiVersion: config.discoblocks.ondat.io/v1 +kind: OperatorConfig health: bindAddress: :8080 healthProbeBindAddress: :8081 @@ -7,7 +7,9 @@ metrics: bindAddress: 127.0.0.1:8080 webhook: port: 9443 -supportedCsiDrivers: ebs.csi.aws.com,csi.storageos.com +supportedCsiDrivers: +- ebs.csi.aws.com +- csi.storageos.com jobContainerImage: nixery.dev/shell/gawk/gnugrep/gnused/coreutils-full/cri-tools/docker-client/nerdctl/nvme-cli proxyContainerImage: nixery.dev/shell/frp schedulerStrictMode: true diff --git a/main.go b/main.go index 903ca78..3c045ed 100644 --- a/main.go +++ b/main.go @@ -39,7 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/webhook" - configdiscoblockv1 "github.com/ondat/discoblocks/api/config.discoblocks.io/v1" + configdiscoblockv1 "github.com/ondat/discoblocks/api/config.discoblocks.ondat.io/v1" discoblocksondatiov1 "github.com/ondat/discoblocks/api/v1" "github.com/ondat/discoblocks/controllers" "github.com/ondat/discoblocks/mutators" @@ -85,6 +85,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(discoblocksondatiov1.AddToScheme(scheme)) + utilruntime.Must(configdiscoblockv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -138,7 +139,8 @@ func main() { if configFile != "" { var err error - options, err = options.AndFrom(ctrl.ConfigFile().AtPath(configFile).OfKind(&operatorConfig)) + cfg := ctrl.ConfigFile() + options, err = options.AndFrom(cfg.AtPath(configFile).OfKind(&operatorConfig)) if err != nil { setupLog.Error(err, "unable to load the config file") os.Exit(1) diff --git a/tests/e2e/kuttl/kuttl-config-1.23.yaml b/tests/e2e/kuttl/kuttl-config-1.23.yaml index dbac729..a71787b 100644 --- a/tests/e2e/kuttl/kuttl-config-1.23.yaml +++ b/tests/e2e/kuttl/kuttl-config-1.23.yaml @@ -1,9 +1,12 @@ -apiVersion: kuttl.dev/v1beta1 +apiVersion: kuttl.dev/v1 kind: TestSuite testDirs: - ./tests/e2e/stable -kindConfig: tests/e2e/kind/kind-config-1.23.yaml startKIND: true +kindConfig: tests/e2e/kind/kind-config-1.23.yaml +kindNodeCache: true kindContainers: - local/discoblocks:e2e -timeout: 500 + - local/discoblocks:job-e2e + - local/discoblocks:proxy-e2e +timeout: 240 diff --git a/tests/e2e/stable/storageos/00-assert.yaml b/tests/e2e/stable/storageos/00-assert.yaml index 70fa5cc..833a4a3 100644 --- a/tests/e2e/stable/storageos/00-assert.yaml +++ b/tests/e2e/stable/storageos/00-assert.yaml @@ -5,33 +5,3 @@ metadata: namespace: cert-manager status: availableReplicas: 1 ---- -apiVersion: storageos.com/v1 -kind: StorageOSCluster -metadata: - name: storageoscluster - namespace: storageos -status: - conditions: - - message: Scheduler Ready - reason: Ready - status: "True" - type: SchedulerReady - - message: Node Ready - reason: Ready - status: "True" - type: NodeReady - - message: APIManager Ready - reason: Ready - status: "True" - type: APIManagerReady - - message: CSI Ready - reason: Ready - status: "True" - type: CSIReady - - message: Cluster Ready - reason: Ready - status: "True" - type: Ready - phase: Running - ready: 1/1 diff --git a/tests/e2e/stable/storageos/00-install-dependencies.yaml b/tests/e2e/stable/storageos/00-install-dependencies.yaml index a14228f..b9d9e48 100644 --- a/tests/e2e/stable/storageos/00-install-dependencies.yaml +++ b/tests/e2e/stable/storageos/00-install-dependencies.yaml @@ -1,8 +1,14 @@ -apiVersion: kuttl.dev/v1beta1 +apiVersion: kuttl.dev/v1 kind: TestStep commands: - command: make -C ../../../.. deploy-cert-manager - command: kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v6.0.1/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml - command: kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v6.0.1/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml - command: kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v6.0.1/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml - - command: kubectl storageos install --include-etcd --etcd-replicas 1 --stos-version=v2.9.0 + - command: kubectl storageos install --include-etcd --etcd-replicas 1 --stos-version=v2.10.0 --wait + timeout: 500 + # Workarund, otherwise cluster deletion times out! + - command: kubectl scale -n storageos --replicas=0 deploy/storageos-operator + - command: sleep 5 + - command: kubectl patch stos -n storageos storageoscluster -p '{"metadata":{"finalizers":null}}' --type=merge + - command: kubectl delete stos -n storageos storageoscluster \ No newline at end of file diff --git a/tests/e2e/stable/storageos/01-assert.yaml b/tests/e2e/stable/storageos/01-assert.yaml index fb245a8..3eec0b1 100644 --- a/tests/e2e/stable/storageos/01-assert.yaml +++ b/tests/e2e/stable/storageos/01-assert.yaml @@ -1,3 +1,27 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: discoblocks-manager-config + namespace: kube-system +data: + controller_manager_config.yaml: | + apiVersion: config.discoblocks.ondat.io/v1 + kind: OperatorConfig + health: + bindAddress: :8080 + healthProbeBindAddress: :8081 + metrics: + bindAddress: 127.0.0.1:8080 + webhook: + port: 9443 + supportedCsiDrivers: + - ebs.csi.aws.com + - csi.storageos.com + jobContainerImage: local/discoblocks:job-e2e + proxyContainerImage: local/discoblocks:proxy-e2e + schedulerStrictMode: true + mutatorStrictMode: true +--- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/tests/e2e/stable/storageos/01-install-discoblocks.yaml b/tests/e2e/stable/storageos/01-install-discoblocks.yaml index f69e9ff..3315c1f 100644 --- a/tests/e2e/stable/storageos/01-install-discoblocks.yaml +++ b/tests/e2e/stable/storageos/01-install-discoblocks.yaml @@ -1,4 +1,6 @@ -apiVersion: kuttl.dev/v1beta1 +apiVersion: kuttl.dev/v1 kind: TestStep commands: - - command: sh -c 'IMG=local/discoblocks:e2e make -C ../../../.. deploy' + - script: sed -i "s|$(grep jobContainerImage ../../../../config/manager/controller_manager_config.yaml | awk '{print $2}')|local/discoblocks:job-e2e|" ../../../../config/manager/controller_manager_config.yaml + - script: sed -i "s|$(grep proxyContainerImage ../../../../config/manager/controller_manager_config.yaml | awk '{print $2}')|local/discoblocks:proxy-e2e|" ../../../../config/manager/controller_manager_config.yaml + - script: REGISTRY=local IMAGE_NAME=discoblocks IMAGE_TAG=e2e make -C ../../../.. _deploy diff --git a/tests/e2e/stable/storageos/02-install-workload.yaml b/tests/e2e/stable/storageos/02-install-workload.yaml index 6aa1baa..12fc14b 100644 --- a/tests/e2e/stable/storageos/02-install-workload.yaml +++ b/tests/e2e/stable/storageos/02-install-workload.yaml @@ -1,4 +1,4 @@ -apiVersion: kuttl.dev/v1beta1 +apiVersion: kuttl.dev/v1 kind: TestStep commands: - command: kubectl apply -f diskconfig-readwriteonce.yaml diff --git a/tests/e2e/stable/storageos/03-validate.yaml b/tests/e2e/stable/storageos/03-validate.yaml index ecdd684..6585a5b 100644 --- a/tests/e2e/stable/storageos/03-validate.yaml +++ b/tests/e2e/stable/storageos/03-validate.yaml @@ -1,8 +1,8 @@ -apiVersion: kuttl.dev/v1beta1 +apiVersion: kuttl.dev/v1 kind: TestStep commands: - - command: | - sh -c 'cat 03-assert.tpl.yaml | sed \ + - script: | + cat 03-assert.tpl.yaml | sed \ -e "s/#PVC_ONCE_NAME#/$(kubectl get pvc -l discoblocks=diskconfig-sample-storageos-once --no-headers -o custom-columns=":metadata.name")/" \ -e "s/#PV_ONCE_NAME#/$(kubectl get pvc -l discoblocks=diskconfig-sample-storageos-once --no-headers -o custom-columns=":spec.volumeName")/" \ -e "s/#PVC_SAME_NAME#/$(kubectl get pvc -l discoblocks=diskconfig-sample-storageos-same --no-headers -o custom-columns=":metadata.name")/" \ @@ -12,5 +12,5 @@ commands: -e "s/#PVC_HOSTPID_NAME#/$(kubectl get pvc -n pod-with-host-pid -l discoblocks=diskconfig-sample-storageos-once --no-headers -o custom-columns=":metadata.name")/" \ -e "s/#PV_HOSTPID_NAME#/$(kubectl get pvc -n pod-with-host-pid -l discoblocks=diskconfig-sample-storageos-once --no-headers -o custom-columns=":spec.volumeName")/" \ -e "s/#POD_NAME#/$(kubectl get po -l app=nginx --no-headers -o custom-columns=":metadata.name")/" \ - > workload/00-assert.yaml' + > workload/00-assert.yaml - command: kubectl-kuttl assert workload \ No newline at end of file diff --git a/tests/e2e/stable/storageos/04-fill-volume.yaml b/tests/e2e/stable/storageos/04-fill-volume.yaml index 114efe6..4cc11e2 100644 --- a/tests/e2e/stable/storageos/04-fill-volume.yaml +++ b/tests/e2e/stable/storageos/04-fill-volume.yaml @@ -1,15 +1,15 @@ -apiVersion: kuttl.dev/v1beta1 +apiVersion: kuttl.dev/v1 kind: TestStep commands: - - command: sh -c "kubectl exec $(kubectl get po --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/same-0/data0 count=1M" - - command: sh -c "kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/once-0/data0 count=1M" - - command: sh -c "kubectl exec -n kube-system $(kubectl get po -n kube-system -l name=fluentd-elasticsearch --no-headers | tail -1 | awk '{print $1}') -- touch /media/discoblocks/daemon-0/data0" + - script: kubectl exec $(kubectl get po --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/same-0/data0 count=1M + - script: kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/once-0/data0 count=1M + - script: kubectl exec -n kube-system $(kubectl get po -n kube-system -l name=fluentd-elasticsearch --no-headers | tail -1 | awk '{print $1}') -- touch /media/discoblocks/daemon-0/data0 - command: sleep 40 - - command: sh -c "kubectl exec $(kubectl get po --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/same-0/data0 count=3M" - - command: sh -c "kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/once-0/data0 count=3M" + - script: kubectl exec $(kubectl get po --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/same-0/data0 count=3M + - script: kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/once-0/data0 count=3M - command: sleep 60 - - command: sh -c "kubectl exec $(kubectl get po --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/same-1/data1 count=1M" - - command: sh -c "kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/once-1/data1 count=1M" + - script: kubectl exec $(kubectl get po --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/same-1/data1 count=1M + - script: kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/once-1/data1 count=1M - command: sleep 40 - - command: sh -c "kubectl exec $(kubectl get po --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/same-1/data1 count=3M" - - command: sh -c "kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/once-1/data1 count=3M" \ No newline at end of file + - script: kubectl exec $(kubectl get po --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/same-1/data1 count=3M + - script: kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid --no-headers | tail -1 | awk '{print $1}') -- dd if=/dev/zero of=/media/discoblocks/once-1/data1 count=3M \ No newline at end of file diff --git a/tests/e2e/stable/storageos/05-restart-workload.yaml b/tests/e2e/stable/storageos/05-restart-workload.yaml index 62c5eeb..37892d6 100644 --- a/tests/e2e/stable/storageos/05-restart-workload.yaml +++ b/tests/e2e/stable/storageos/05-restart-workload.yaml @@ -1,5 +1,5 @@ -apiVersion: kuttl.dev/v1beta1 +apiVersion: kuttl.dev/v1 kind: TestStep commands: - - command: sh -c "kubectl delete po $(kubectl get po -l app=nginx --no-headers -o custom-columns=":metadata.name")" - - command: sh -c "kubectl delete po -n kube-system $(kubectl get po -n kube-system -l name=fluentd-elasticsearch --no-headers -o custom-columns=":metadata.name")" \ No newline at end of file + - script: kubectl delete po $(kubectl get po -l app=nginx --no-headers -o custom-columns=":metadata.name") + - script: kubectl delete po -n kube-system $(kubectl get po -n kube-system -l name=fluentd-elasticsearch --no-headers -o custom-columns=":metadata.name") \ No newline at end of file diff --git a/tests/e2e/stable/storageos/06-assert.yaml b/tests/e2e/stable/storageos/06-assert.yaml new file mode 100644 index 0000000..0dd8007 --- /dev/null +++ b/tests/e2e/stable/storageos/06-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: kuttl.dev/v1 +kind: TestAssert +timeout: 15 +commands: + - script: kubectl exec $(kubectl get po | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/once-0 + - script: kubectl exec $(kubectl get po | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/same-0/data0 + - script: kubectl exec $(kubectl get po | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/same-1/data1 + - script: kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/once-0/data0 + - script: kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/once-1/data1 + - script: kubectl exec -n kube-system $(kubectl get po -n kube-system -l name=fluentd-elasticsearch --no-headers | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/daemon-0/data0 \ No newline at end of file diff --git a/tests/e2e/stable/storageos/06-validate-volumes.yaml b/tests/e2e/stable/storageos/06-validate-volumes.yaml deleted file mode 100644 index fa46d42..0000000 --- a/tests/e2e/stable/storageos/06-validate-volumes.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - command: sh -c "kubectl exec $(kubectl get po | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/once-0" - - command: sh -c "kubectl exec $(kubectl get po | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/same-0/data0" - - command: sh -c "kubectl exec $(kubectl get po | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/same-1/data1" - - command: sh -c "kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/once-0/data0" - - command: sh -c "kubectl exec -n pod-with-host-pid $(kubectl get po -n pod-with-host-pid | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/once-1/data1" - - command: sh -c "kubectl exec -n kube-system $(kubectl get po -n kube-system -l name=fluentd-elasticsearch --no-headers | tail -1 | awk '{print $1}') -- ls -l /media/discoblocks/daemon-0/data0" \ No newline at end of file