Skip to content

Fix the error of vineyardctl and add the related test. #1374

Fix the error of vineyardctl and add the related test.

Fix the error of vineyardctl and add the related test. #1374

# Copyright 2020-2023 Alibaba Group Holding Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Vineyard Operator
on:
push:
branches:
- main
tags:
- 'v*'
paths:
- '.github/workflows/vineyard-operator.yaml'
- 'charts/vineyard-operator/**'
- 'docker/Dockerfile.vineyardd*'
- 'k8s/**'
- 'src/server/util/kubectl*'
pull_request:
branches:
- main
- dev/kubernetes
paths:
- '.github/workflows/vineyard-operator.yaml'
- 'charts/vineyard-operator/**'
- 'docker/Dockerfile.vineyardd*'
- 'k8s/**'
- 'src/server/util/kubectl*'
workflow_dispatch:
concurrency:
group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true
env:
REGISTRY: ghcr.io/v6d-io/v6d
IMG: vineyardcloudnative/vineyard-operator:latest
NIGHTLY_IMG: ghcr.io/v6d-io/v6d/vineyard-operator:nightly
LATEST_IMG: ghcr.io/v6d-io/v6d/vineyard-operator:latest
KUBECONFIG: /tmp/e2e-k8s.config
jobs:
ci:
runs-on: ubuntu-20.04
if: ${{ github.repository == 'v6d-io/v6d' }}
strategy:
fail-fast: false
matrix:
job:
- release
- unit-tests
- e2e-tests-airflow-integration
- e2e-tests-assembly-local
- e2e-tests-assembly-distributed
- e2e-tests-autogenerated-helm-chart
- e2e-tests-failover
- e2e-tests-repartition-dask
- e2e-tests-schedule-workflow
- e2e-tests-schedule-workflow-without-crd
- e2e-tests-schedule-workload
- e2e-tests-serialize
- e2e-tests-sidecar
- e2e-tests-spill
- e2e-tests-workflow
- e2e-tests-mars-examples
- e2e-tests-vineyardctl
- e2e-tests-vineyardctl-python-api
- e2e-tests-deploy-raw-backup-and-recover
- e2e-tests-three-etcd-nodes-failover
- e2e-tests-five-etcd-nodes-failover
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Free disk space
run: |
bash .github/free-disk-space.sh
- uses: actions/setup-go@v3
with:
go-version: 1.19
- name: Cache for golang
uses: actions/cache@v3
with:
path: ~/go
key: vineyard-kubernetes-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
vineyard-kubernetes-go-
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.50.0
working-directory: k8s
skip-pkg-cache: true
- name: Golang format
run: |
function prepend() { while read line; do echo "${1}${line}"; done; }
make -C k8s fmt
GIT_DIFF=$(git diff --ignore-submodules)
if [[ -n $GIT_DIFF ]]; then
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo "| gofmt failures found!"
echo "|"
echo "$GIT_DIFF" | prepend "| "
echo "|"
echo "| Run: "
echo "|"
echo "| make -C k8s fmt"
echo "|"
echo "| to fix this error."
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
exit -1
fi
- name: Log in to GitHub Docker Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build docker image
run: |
make -C k8s docker-build
- name: Extract tag name
id: tag
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
- name: Set up qemu
if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }}
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }}
uses: docker/setup-buildx-action@v2
- name: Upload latest docker image of vineyard-operator
if: ${{ matrix.job == 'release' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }}
run: |
# pre-publish to ghcr.io to avoid the "no basic auth credentials" error
export IMG=ghcr.io/v6d-io/v6d/vineyard-operator:${{ steps.tag.outputs.TAG }}
make -C k8s docker-build-push-multi-arch REGISTRY=${{ env.REGISTRY }} VERSION=${{ steps.tag.outputs.TAG }}
export IMG=ghcr.io/v6d-io/v6d/vineyard-operator:latest
make -C k8s docker-build-push-multi-arch REGISTRY=${{ env.REGISTRY }} VERSION=latest
- name: Generate the python and vineyardd image for tests
run: |
go build -a -o python/vineyard/bdist/vineyardctl k8s/cmd/main.go && \
make -C k8s/test/e2e build-base-images
- name: unit-tests
if: ${{ matrix.job == 'unit-tests' }}
run: |
make -C k8s/test/e2e unittest
- name: Prepare directories for logs
run: |
# create log directories for all tasks to prevent the "Path to upload does not exist" warnings
for job in e2e-tests-airflow-integration \
e2e-tests-assembly-local \
e2e-tests-assembly-distributed \
e2e-tests-autogenerated-helm-chart \
e2e-tests-failover \
e2e-tests-repartition-dask \
e2e-tests-schedule-workflow \
e2e-tests-schedule-workflow-without-crd \
e2e-tests-schedule-workload \
e2e-tests-serialize \
e2e-tests-sidecar \
e2e-tests-spill \
e2e-tests-workflow \
e2e-tests-mars-examples \
e2e-tests-vineyardctl \
e2e-tests-deploy-raw-backup-and-recover \
e2e-tests-three-etcd-nodes-failover \
e2e-tests-five-etcd-nodes-failover \
unit-tests; do
mkdir -p $job-logs
chmod a+rwx $job-logs
done
- name: Start to export kubernetes logs
uses: dashanji/kubernetes-log-export-action@v6
env:
SHOW_TIMESTAMPS: 'true'
OUTPUT_DIR: ${{ github.workspace }}/${{ matrix.job }}-logs
NAMESPACES: vineyard*
RESOURCES: configmap
MODE: start
- name: e2e-tests-airflow-integration
if: ${{ matrix.job == 'e2e-tests-airflow-integration' }}
run: |
make -C k8s/test/e2e e2e-tests-airflow-integration
- name: e2e-tests-assembly-local
if: ${{ matrix.job == 'e2e-tests-assembly-local' }}
run: |
make -C k8s/test/e2e e2e-tests-assembly-local
- name: e2e-tests-assembly-distributed
if: ${{ matrix.job == 'e2e-tests-assembly-distributed' }}
run: |
make -C k8s/test/e2e e2e-tests-assembly-distributed
- name: e2e-tests-autogenerated-helm-chart
if: ${{ matrix.job == 'e2e-tests-autogenerated-helm-chart' }}
run: |
make -C k8s/test/e2e e2e-tests-autogenerated-helm-chart
- name: e2e-tests-failover
if: ${{ matrix.job == 'e2e-tests-failover' }}
run: |
make -C k8s/test/e2e e2e-tests-failover
- name: e2e-tests-repartition-dask
if: ${{ matrix.job == 'e2e-tests-repartition-dask' }}
run: |
make -C k8s/test/e2e e2e-tests-repartition-dask
- name: e2e-tests-schedule-workflow
if: ${{ matrix.job == 'e2e-tests-schedule-workflow' }}
run: |
make -C k8s/test/e2e e2e-tests-schedule-workflow
- name: e2e-tests-schedule-workflow-without-crd
if: ${{ matrix.job == 'e2e-tests-schedule-workflow-without-crd' }}
run: |
make -C k8s/test/e2e e2e-tests-schedule-workflow-without-crd
- name: e2e-tests-schedule-workload
if: ${{ matrix.job == 'e2e-tests-schedule-workload' }}
run: |
make -C k8s/test/e2e e2e-tests-schedule-workload
- name: e2e-tests-serialize
if: ${{ matrix.job == 'e2e-tests-serialize' }}
run: |
make -C k8s/test/e2e e2e-tests-serialize
- name: e2e-tests-sidecar
if: ${{ matrix.job == 'e2e-tests-sidecar' }}
run: |
make -C k8s/test/e2e e2e-tests-sidecar
- name: e2e-tests-spill
if: ${{ matrix.job == 'e2e-tests-spill' }}
run: |
make -C k8s/test/e2e e2e-tests-spill
- name: e2e-tests-workflow
if: ${{ matrix.job == 'e2e-tests-workflow' }}
run: |
make -C k8s/test/e2e e2e-tests-workflow
- name: e2e-tests-mars-examples
if: false # temporarily disabled, we need some changes in mars
# if: ${{ matrix.job == 'e2e-tests-mars-examples' }}
run: |
make -C k8s/test/e2e e2e-tests-mars-examples
- name: e2e-tests-vineyardctl
if: ${{ matrix.job == 'e2e-tests-vineyardctl' }}
run: |
make -C k8s/test/e2e e2e-tests-vineyardctl
- name: e2e-tests-vineyardctl-python-api
if: ${{ matrix.job == 'e2e-tests-vineyardctl-python-api' }}
run: |
make -C k8s/test/e2e e2e-tests-vineyardctl-python-api
- name: e2e-tests-deploy-raw-backup-and-recover
if: ${{ matrix.job == 'e2e-tests-deploy-raw-backup-and-recover' }}
run: |
make -C k8s/test/e2e e2e-tests-deploy-raw-backup-and-recover
- name: e2e-tests-three-etcd-nodes-failover
if: ${{ matrix.job == 'e2e-tests-three-etcd-nodes-failover' }}
run: |
make -C k8s/test/e2e e2e-tests-three-etcd-nodes-failover
- name: e2e-tests-five-etcd-nodes-failover
if: ${{ matrix.job == 'e2e-tests-five-etcd-nodes-failover' }}
run: |
make -C k8s/test/e2e e2e-tests-five-etcd-nodes-failover
- name: Stop to export kubernetes logs
uses: dashanji/kubernetes-log-export-action@v6
if: ${{ failure() }}
env:
MODE: stop
OUTPUT_DIR: ${{ github.workspace }}/${{ matrix.job }}-logs
WITH_STOAT: true
- name: Setup tmate session
if: false
uses: mxschmitt/action-tmate@v3
- uses: stoat-dev/stoat-action@v0
if: ${{ failure() }}
- name: Leave a message for logs URL
if: ${{ failure() }}
run: |
sha=${{ github.event.pull_request.head.sha || github.sha }}
short_sha=$(git rev-parse --short $sha)
case ${{ matrix.job }} in
e2e-tests-spill)
signed_url_key=0707
;;
e2e-tests-sidecar)
signed_url_key=eb62
;;
e2e-tests-failover)
signed_url_key=4302
;;
e2e-tests-workflow)
signed_url_key=2355
;;
e2e-tests-serialize)
signed_url_key=ab53
;;
e2e-tests-vineyardctl)
signed_url_key=cb83
;;
e2e-tests-assembly-local)
signed_url_key=2cdf
;;
e2e-tests-repartition-dask)
signed_url_key=fb84
;;
e2e-tests-assembly-distributed)
signed_url_key=2573
;;
e2e-tests-autogenerated-helm-chart)
signed_url_key=7a77
;;
e2e-tests-schedule-workload)
signed_url_key=3d07
;;
e2e-tests-schedule-workflow)
signed_url_key=44c1
;;
e2e-tests-schedule-workflow-without-crd)
signed_url_key=9a01
;;
e2e-tests-deploy-raw-backup-and-recover)
signed_url_key=6552
;;
e2e-tests-airflow-integration)
signed_url_key=2e12
;;
e2e-tests-three-etcd-nodes-failover)
signed_url_key=5c91
;;
e2e-tests-five-etcd-nodes-failover)
signed_url_key=7793
;;
esac
echo "::notice:: ${{ matrix.job }}: https://www.stoat.dev/file-viewer?root=https://v6d-io--v6d--$short_sha--e2e-tests-$signed_url_key.stoat.page"