Robot Upgrade: gateway chart upgrade from 1.17.1 to 1.24.1 #2978
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR E2E C | |
| env: | |
| TRIVY_SCAN_SKIP_CHART_LIST: "nmstate cert-manager contour ingress-nginx" | |
| IMAGE_HINT_FILE_NAME: ".relok8s-images.yaml" | |
| SCHEMA_FILE_NAME: "values.schema.json" | |
| on: | |
| pull_request_target: | |
| branches: | |
| - main | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: check chart config | |
| run : | | |
| TMP=`ls charts` | |
| for ITEM in $TMP ; do | |
| [ ! -d "charts/${ITEM}" ] && continue | |
| echo "check config for chart $ITEM " | |
| [ ! -f "charts/${ITEM}/config" ] && echo "error, miss charts/${ITEM}/config " && exit 1 | |
| for VAR in DAOCLOUD_REPO_PROJECT UPGRADE_METHOD ; do | |
| VAL=` source charts/${ITEM}/config && echo ${!VAR} ` | |
| [ -n "$VAL" ] || { echo "error, miss $VAR in charts/${ITEM}/config"; exit 1 ; } | |
| done | |
| UPGRADE_METHOD=` source charts/${ITEM}/config && echo $UPGRADE_METHOD ` | |
| if [ "$UPGRADE_METHOD" == "pr" ] || [ "$UPGRADE_METHOD" == "issue" ] ; then | |
| for VAR in REPO_URL REPO_NAME CHART_NAME VERSION UPGRADE_REVIWER ; do | |
| VAL=` source charts/${ITEM}/config && echo ${!VAR} ` | |
| [ -n "$VAL" ] || { echo "error, miss $VAR in charts/${ITEM}/config"; exit 1 ; } | |
| done | |
| elif [ "$UPGRADE_METHOD" == "none" ] ; then | |
| echo "" | |
| else | |
| echo "error, UPGRADE_METHOD=$UPGRADE_METHOD must be 'pr'/'issue'/'none' in charts/${ITEM}/config" | |
| exit 1 | |
| fi | |
| done | |
| exit 0 | |
| check_change: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| project: ${{ steps.check_change.outputs.test_project }} | |
| pro_version_changed: ${{ env.PRO_CHANGED_VERSION }} | |
| steps: | |
| - name: "Setup go" | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: '1.19' | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: check change | |
| id: check_change | |
| run: | | |
| set -x | |
| # Using the Github API to detect the files changed as git merge-base stops working when the branch is behind | |
| # and jitterbit/get-changed-files does not support pull_request_target | |
| URL="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" | |
| files_changed_data=$(curl -s --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -X GET -G "$URL") | |
| echo "files_changed_data: $files_changed_data" | |
| files_changed="$(echo $files_changed_data | jq -r '.[] | .filename')" | |
| echo "files_changed: $files_changed" | |
| #-------------------- | |
| # Adding || true to avoid "Process exited with code 1" errors | |
| charts_changed="$(echo "$files_changed" | xargs dirname | grep -o "charts/[^/]*" | sort | uniq | awk -F'/' '{print $2}' | tr '\n' ' ' || true)" | |
| echo "charts_changed: $charts_changed" | |
| #-------------------- | |
| # issue: just release one project | |
| PRO_CHANGED_VERSION="" | |
| for PRO in ${charts_changed} ; do | |
| if [ -d "charts/${PRO}" ] ; then | |
| if grep "charts/${PRO}/${PRO}/Chart.yaml" <<< "${files_changed}" &>/dev/null ; then | |
| PRO_CHANGED_VERSION="${PRO}" | |
| break | |
| fi | |
| fi | |
| done | |
| echo "PRO_CHANGED_VERSION=${PRO_CHANGED_VERSION}" | |
| echo "PRO_CHANGED_VERSION=${PRO_CHANGED_VERSION}" >> $GITHUB_ENV | |
| #------------------- | |
| # for project test script | |
| additionall_test_project="" | |
| project_test_dir_changed="$(echo "$files_changed" | xargs dirname | grep -o "test/[^/]*" | sort | uniq | awk -F'/' '{print $2}' || true)" | |
| for PRO in ${project_test_dir_changed} ; do | |
| if ! grep " ${PRO} " <<< " ${charts_changed} " &>/dev/null ; then | |
| additionall_test_project+=" $PRO " | |
| echo "test script changes for ${PRO} " | |
| fi | |
| done | |
| #-------------------- | |
| # when other files changed | |
| test_dir_changed="$(echo "$files_changed" | grep -o "test/[^/]*" | sort | uniq || true)" | |
| if [ -n "$test_dir_changed" ] && [ -z "${additionall_test_project}" ] && [ -z "$charts_changed" ] ; then | |
| # add spiderppool for testing test changes | |
| charts_changed=` echo "$charts_changed spiderpool" | tr -s ' ' | tr ' ' '\n' | sort | uniq | tr '\n' ' ' || true` | |
| echo "detect test changes, add spiderpool, charts_changed: $charts_changed" | |
| fi | |
| charts_changed=` echo $charts_changed ` | |
| RES="" | |
| for PRO in ${charts_changed} ; do | |
| if [ -d "charts/${PRO}" ] ; then | |
| RES+="${PRO} " | |
| fi | |
| done | |
| echo ::set-output name=changed_project::${RES} | |
| ALL_PRO=" ${RES} ${additionall_test_project} " | |
| echo ::set-output name=test_project::${ALL_PRO} | |
| - name: check chart override | |
| run: | | |
| BASE=$PWD | |
| cd cmd/chart-override-check | |
| for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do | |
| PROJECT_PATH=${BASE}/charts/${PROJECT} | |
| go run main.go -path ${PROJECT_PATH}/${PROJECT} -skipCPath ${PROJECT_PATH}/skip-check.yaml | |
| done | |
| - name: check image hint file | |
| run: | | |
| BASE=$PWD | |
| for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do | |
| source ${BASE}/charts/${PROJECT}/config && [ "${NO_IMAGE}" == "true" ] && continue | |
| CHART_PATH=${BASE}/charts/${PROJECT}/${PROJECT} | |
| [ -f "${CHART_PATH}/${{ env.IMAGE_HINT_FILE_NAME }}" ] \ | |
| || { echo "error, miss ${{ env.IMAGE_HINT_FILE_NAME }} file in chart ${PROJECT} " ; exit 1 ; } | |
| ! grep -E '\-[[:space:]]+[^"].*[^"]' ${CHART_PATH}/${{ env.IMAGE_HINT_FILE_NAME }} \ | |
| || { echo "error, must use Double quotation in ${PROJECT}/.relok8s-images.yaml " ; exit 1 ; } | |
| done | |
| - name: check Chart yaml | |
| run: | | |
| BASE=$PWD | |
| for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do | |
| CHART_PATH=${BASE}/charts/${PROJECT}/${PROJECT} | |
| if ! grep "keywords:" ${CHART_PATH}/Chart.yaml &>/dev/null ;then | |
| echo "error, please input 'keywords' into the Chart.yaml of ${PROJECT} " | |
| exit 1 | |
| fi | |
| NAME=` yq '.name' ${CHART_PATH}/Chart.yaml ` | |
| echo "chart NAME '${NAME}' in project '${PROJECT}' " | |
| [ "${NAME}" == "${PROJECT}" ] || { echo "error, chart name is not ${PROJECT}" ; exit 1 ; } | |
| done | |
| - name: check image repo | |
| run: | | |
| BASE=$PWD | |
| for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do | |
| source ${BASE}/charts/${PROJECT}/config && [ "${NO_IMAGE}" == "true" ] && continue | |
| CHART_PATH=${BASE}/charts/${PROJECT}/${PROJECT} | |
| if helm template test ${CHART_PATH} | grep ' image: ' | cut -d ":" -f2,3 | grep -v "daocloud" &>/dev/null ; then | |
| echo "error, ${PROJECT} image is not in daocloud repo" | |
| helm template test ${CHART_PATH} | grep ' image: ' | cut -d ":" -f2,3 | grep -v "daocloud" | |
| exit 1 | |
| fi | |
| done | |
| - name: check resource limit | |
| run: | | |
| BASE=$PWD | |
| for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do | |
| CHART_PATH=${BASE}/charts/${PROJECT}/${PROJECT} | |
| if helm template test ${CHART_PATH} | grep -E "kind: DaemonSet|kind: Deployment|kind: Job|kind: CronJob" &>/dev/null ; then | |
| echo "check resource limit and request" | |
| if ! ( helm template test ${CHART_PATH} | grep ' limits:' &>/dev/null ) ; then | |
| echo "error, did not find resource limits settings in ${PROJECT}" | |
| exit 1 | |
| fi | |
| if ! ( helm template test ${CHART_PATH} | grep ' requests:' &>/dev/null ) ; then | |
| echo "error, did not find resource requests settings in ${PROJECT}" | |
| exit 1 | |
| fi | |
| fi | |
| done | |
| - name: check schema file | |
| run: | | |
| BASE=$PWD | |
| for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do | |
| CHART_PATH=${BASE}/charts/${PROJECT}/${PROJECT} | |
| # skip schema check | |
| source ${BASE}/charts/${PROJECT}/config && [ "${SKIP_SCHEMA}" == "true" ] && echo "ignore schema check" && continue | |
| [ -f "${CHART_PATH}/${{ env.SCHEMA_FILE_NAME }}" ] \ | |
| || { echo "error, miss ${{ env.SCHEMA_FILE_NAME }} file in chart ${PROJECT} " ; exit 1 ; } | |
| done | |
| - name: check chart trivy | |
| run: | | |
| BASE=$PWD | |
| for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do | |
| # skip chart trivy | |
| source ${BASE}/charts/${PROJECT}/config && [ "${NO_TRIVY}" == "true" ] && echo "ignore trivy" && continue | |
| docker run --rm \ | |
| -v /tmp/trivy:/root/trivy.cache/ \ | |
| -v ${BASE}:/tmp/src \ | |
| aquasec/trivy:latest config --exit-code 1 --severity CRITICAL /tmp/src/charts/${PROJECT}/${PROJECT} | |
| (( $? == 0 )) || { echo "error, failed to check chart trivy" && exit 1 ; } | |
| echo "chart trivy check: pass charts/${PROJECT}/${PROJECT} " | |
| done | |
| - name: check upgrade | |
| run: | | |
| for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do | |
| make build_chart -e PROJECT=${PROJECT} \ | |
| || { echo "error, failed to call 'make build_chart' to update ${PROJECT} " ; exit 1 ; } | |
| if ! git diff --exit-code --quiet ; then | |
| git status | |
| git diff | |
| echo "error, after 'make build_chart', chart is not updated" | |
| exit 1 | |
| fi | |
| done | |
| - name: run validate schema test | |
| run: | | |
| BASE=$PWD | |
| python -m unittest ${BASE}/scripts/validate_schema_keys/test_validate.py -v || { echo "error, failed to run test script" ; exit 1 ; } | |
| - name: validate schema for value | |
| run: | | |
| BASE=$PWD | |
| for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do | |
| CHART_PATH=charts/${PROJECT}/${PROJECT} | |
| SCHEMA_FILE=${CHART_PATH}/${{ env.SCHEMA_FILE_NAME }} | |
| PARENT_SCHEMA_FILE=charts/${PROJECT}/parent/${{ env.SCHEMA_FILE_NAME }} | |
| [ -f ${PARENT_SCHEMA_FILE} ] && SCHEMA_FILE=${PARENT_SCHEMA_FILE} | |
| python ${BASE}/scripts/validate_schema_keys/validate.py ${SCHEMA_FILE} ${CHART_PATH}/values.yaml | |
| [ $? -eq 0 ] || { echo "error, failed to validate ${CHART_PATH}/values.yaml with ${SCHEMA_FILE}" ; exit 1 ; } | |
| PROJECT_SRC_CONFIG_PATH=charts/${PROJECT}/config | |
| source $PROJECT_SRC_CONFIG_PATH | |
| if [ -n "${APPEND_VALUES_FILE}" ] && [ -s ${PROJECT_SRC_DIR}/${APPEND_VALUES_FILE} ] ; then | |
| python ${BASE}/scripts/validate_schema_keys/validate.py ${SCHEMA_FILE} ${PROJECT_SRC_DIR}/${APPEND_VALUES_FILE} | |
| [ $? -eq 0 ] || { echo "error, failed to validate ${PROJECT_SRC_DIR}/${APPEND_VALUES_FILE} with ${SCHEMA_FILE}" ; exit 1 ; } | |
| fi | |
| done | |
| call_e2e: | |
| uses: ./.github/workflows/call-e2e.yml | |
| needs: [check_change] | |
| if: ${{ needs.check_change.outputs.project != '' }} | |
| with: | |
| project: ${{ needs.check_change.outputs.project }} | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| secrets: inherit | |
| # nofity_version_changed: | |
| # runs-on: ubuntu-latest | |
| # needs: [ check_change ] | |
| # if: ${{ needs.check_change.outputs.pro_version_changed != '' }} | |
| # steps: | |
| # - name: create an issue | |
| # uses: dacbd/create-issue-action@main | |
| # with: | |
| # token: ${{ secrets.GITHUB_TOKEN }} | |
| # repo: DaoCloud/DaoCloud-docs | |
| # title: "project ${{ needs.check_change.outputs.pro_version_changed }} update to version " | |
| # body: my new issue | |
| # | |
| # - name: Print outputs | |
| # run: | | |
| # echo ${{ steps.create-issue.outputs.id }} | |
| # echo ${{ steps.create-issue.outputs.number }} |