Skip to content

Commit

Permalink
Merge pull request #446 from wallarm/NODE-5583
Browse files Browse the repository at this point in the history
Node-5583 Required variables for smoke-tests
  • Loading branch information
braek-neck authored Aug 9, 2024
2 parents 5d9b00c + 02dc807 commit 782165f
Show file tree
Hide file tree
Showing 8 changed files with 282 additions and 140 deletions.
2 changes: 2 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ SMOKE_REGISTRY_SECRET=...
# Just keep it here since it is used in main Makefile if we don't have GO installed locally
ARCH=amd64

PYTEST_WORKERS=10

# Local env features

# To download from private registry set the following dkr.wallarm.com/wallarm-node/ingress-images-factory
Expand Down
68 changes: 67 additions & 1 deletion .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,26 @@ jobs:
- name: Checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- name: Import secrets
uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e # v2.5.0
id: secrets
with:
exportEnv: true
url: ${{ secrets.VAULT_URL }}
role: ${{ secrets.VAULT_ROLE }}
method: kubernetes
path: kubernetes-ci
secrets: |
kv-gitlab-ci/data/github/shared/dockerhub-creds user | DOCKERHUB_USER ;
kv-gitlab-ci/data/github/shared/dockerhub-creds password | DOCKERHUB_PASSWORD ;
- name: Build controller images
env:
REGISTRY: wallarm
TAG: 1.0.0-dev
USER: runner
run: |
echo ${DOCKERHUB_PASSWORD} | docker login -u ${DOCKERHUB_USER} --password-stdin
echo "Building controller images ..."
make clean-image build image image-chroot
echo "Building E2E image ..."
Expand Down Expand Up @@ -134,6 +148,8 @@ jobs:
kv-gitlab-ci/data/github/shared/allure allure_token ;
kv-gitlab-ci/data/github/shared/smoke-tests-registry-creds token_name ;
kv-gitlab-ci/data/github/shared/smoke-tests-registry-creds token_secret ;
kv-gitlab-ci/data/github/shared/dockerhub-creds user ;
kv-gitlab-ci/data/github/shared/dockerhub-creds password ;
- name: Checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
Expand All @@ -146,13 +162,18 @@ jobs:
- name: Load images
run: docker load -i controller-${{ env.ARCH }}.tar

- name: Login to DockerHub
run: echo ${{ steps.secrets.outputs.password }} | docker login -u ${{ steps.secrets.outputs.user }} --password-stdin

- name: Create cluster
run: kind create cluster --image=kindest/node:${{ matrix.k8s }}

- name: Run smoke tests
env:
SKIP_CLUSTER_CREATION: true
SKIP_IMAGE_CREATION: true
DOCKERHUB_USER: ${{ steps.secrets.outputs.user }}
DOCKERHUB_PASSWORD: ${{ steps.secrets.outputs.password }}
WALLARM_API_TOKEN: ${{ steps.secrets.outputs.api_token }}
WALLARM_API_HOST: ${{ steps.secrets.outputs.api_host }}
WALLARM_API_PRESET: ${{ steps.secrets.outputs.api_preset }}
Expand All @@ -161,8 +182,10 @@ jobs:
SMOKE_REGISTRY_SECRET: ${{ steps.secrets.outputs.token_secret }}
WEBHOOK_API_KEY: ${{ steps.secrets.outputs.webhook_api_key }}
WEBHOOK_UUID: ${{ steps.secrets.outputs.webhook_uuid }}
CLIENT_ID: 5
ALLURE_UPLOAD_REPORT: true
ALLURE_GENERATE_REPORT: true
ALLURE_PROJECT_ID: 10
ALLURE_TOKEN: ${{ steps.secrets.outputs.allure_token }}
ALLURE_ENVIRONMENT_K8S: ${{ matrix.k8s }}
ALLURE_ENVIRONMENT_ARCH: ${{ matrix.ARCH }}
Expand Down Expand Up @@ -192,7 +215,10 @@ jobs:
role: ${{ secrets.VAULT_ROLE }}
method: kubernetes
path: kubernetes-ci
secrets: kv-gitlab-ci/data/github/ingress api_token
secrets: |
kv-gitlab-ci/data/github/ingress api_token ;
kv-gitlab-ci/data/github/shared/dockerhub-creds user ;
kv-gitlab-ci/data/github/shared/dockerhub-creds password ;
- name: Checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
Expand All @@ -207,6 +233,9 @@ jobs:
- name: Load images
run: docker load -i controller-${{ env.ARCH }}.tar

- name: Login to DockerHub
run: echo ${{ steps.secrets.outputs.password }} | docker login -u ${{ steps.secrets.outputs.user }} --password-stdin

- name: Create cluster
run: kind create cluster --image=kindest/node:v1.25.8 --config test/e2e/kind.yaml

Expand Down Expand Up @@ -243,6 +272,22 @@ jobs:
- name: Load controller images
run: docker load -i controller-${{ env.ARCH }}.tar

- name: Import secrets
uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
id: secrets
with:
exportEnv: false
url: ${{ secrets.VAULT_URL }}
role: ${{ secrets.VAULT_ROLE }}
method: kubernetes
path: kubernetes-ci
secrets: |
kv-gitlab-ci/data/github/shared/dockerhub-creds user ;
kv-gitlab-ci/data/github/shared/dockerhub-creds password ;
- name: Login to DockerHub
run: echo ${{ steps.secrets.outputs.password }} | docker login -u ${{ steps.secrets.outputs.user }} --password-stdin

- name: Create cluster
run: kind create cluster --image=kindest/node:v1.25.8 --config test/e2e/kind.yaml

Expand Down Expand Up @@ -285,6 +330,8 @@ jobs:
secrets: |
kv-gitlab-ci/data/github/ingress api_token ;
kv-gitlab-ci/data/github/ingress api_host ;
kv-gitlab-ci/data/github/shared/dockerhub-creds user ;
kv-gitlab-ci/data/github/shared/dockerhub-creds password ;
- name: Checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
Expand All @@ -297,6 +344,9 @@ jobs:
- name: Load controller images
run: docker load -i controller-${{ env.ARCH }}.tar

- name: Login to DockerHub
run: echo ${{ steps.secrets.outputs.password }} | docker login -u ${{ steps.secrets.outputs.user }} --password-stdin

- name: Create cluster ${{ matrix.k8s }}
run: kind create cluster --image=kindest/node:${{ matrix.k8s }} --config test/e2e/kind.yaml

Expand Down Expand Up @@ -333,6 +383,22 @@ jobs:
- name: Load controller images
run: docker load -i controller-${{ env.ARCH }}.tar

- name: Import secrets
uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0
id: secrets
with:
exportEnv: false
url: ${{ secrets.VAULT_URL }}
role: ${{ secrets.VAULT_ROLE }}
method: kubernetes
path: kubernetes-ci
secrets: |
kv-gitlab-ci/data/github/shared/dockerhub-creds user ;
kv-gitlab-ci/data/github/shared/dockerhub-creds password ;
- name: Login to DockerHub
run: echo ${{ steps.secrets.outputs.password }} | docker login -u ${{ steps.secrets.outputs.user }} --password-stdin

- name: Scan controller image
uses: anchore/scan-action@3343887d815d7b07465f6fdcd395bd66508d486a
with:
Expand Down
4 changes: 2 additions & 2 deletions charts/ingress-nginx/templates/tarantool-daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- if .Values.controller.wallarm.imagePullSecrets }}
imagePullSecrets: {{ toYaml .Values.controller.wallarm.imagePullSecrets | nindent 8 }}
{{- if .Values.imagePullSecrets }}
imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.controller.wallarm.tarantool.terminationGracePeriodSeconds }}
initContainers:
Expand Down
6 changes: 3 additions & 3 deletions charts/ingress-nginx/templates/tarantool-deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- if .Values.controller.wallarm.imagePullSecrets }}
imagePullSecrets: {{ toYaml .Values.controller.wallarm.imagePullSecrets | nindent 8 }}
{{- if .Values.imagePullSecrets }}
imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.controller.wallarm.tarantool.terminationGracePeriodSeconds }}
initContainers:
Expand Down Expand Up @@ -122,4 +122,4 @@ spec:
emptyDir: {}
{{- include "ingress-nginx.wallarmTokenVolume" . | nindent 8 }}
{{- end }}
{{- end }}
{{- end }}
161 changes: 161 additions & 0 deletions test/smoke/functions.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
# This file used for import in other files

RED='\033[0;31m'
NC='\033[0m'

function check_mandatory_vars() {

declare -a mandatory
declare -a allure_mandatory

mandatory=(
WALLARM_API_TOKEN
WALLARM_API_HOST
WALLARM_API_PRESET
CLIENT_ID
USER_TOKEN
WEBHOOK_API_KEY
WEBHOOK_UUID
SMOKE_REGISTRY_TOKEN
SMOKE_REGISTRY_SECRET
)

env_list=""

for var in "${mandatory[@]}"; do
if [[ -z "${!var:-}" ]]; then
env_list+=" $var"
fi
done

if [[ "${ALLURE_UPLOAD_REPORT:-false}" == "true" ]]; then

allure_mandatory=(
ALLURE_TOKEN
ALLURE_ENVIRONMENT_ARCH
ALLURE_PROJECT_ID
ALLURE_GENERATE_REPORT
ALLURE_ENVIRONMENT_K8S
)

for var in "${allure_mandatory[@]}"; do
if [[ -z "${!var:-}" ]]; then
env_list+=" $var"
fi
done
fi

if [[ -n "$env_list" ]]; then
for var in ${env_list}; do
echo -e "${RED}Environment variable $var must be set${NC}"
done
exit 1
fi

}

function cleanup() {
if [[ "${KUBETEST_IN_DOCKER:-}" == "true" ]]; then
kind "export" logs --name ${KIND_CLUSTER_NAME} "${ARTIFACTS}/logs" || true
fi
if [[ "${CI:-}" == "true" ]]; then
kind delete cluster \
--verbosity=${KIND_LOG_LEVEL} \
--name ${KIND_CLUSTER_NAME}
fi
}

function describe_pods_on_exit() {
controller_label="app.kubernetes.io/component=controller"
tarantool_label="app.kubernetes.io/component=controller-wallarm-tarantool"
workload_label="app=workload"

echo "#################### Describe controller POD ####################"
kubectl describe pod -l $controller_label
echo "#################### Describe Tarantool POD ####################"
kubectl describe pod -l $tarantool_label
echo "#################### Describe workload POD ####################"
kubectl describe pod -l $workload_label
}

function clean_allure_report() {
[[ "$ALLURE_GENERATE_REPORT" == false && -d "allure_report" ]] && rm -rf allure_report/* 2>/dev/null || true
}

function get_logs_and_fail() {
get_logs
extra_debug_logs
clean_allure_report
exit 1
}

function get_logs() {
echo "#################################"
echo "###### Init container logs ######"
echo "#################################"
kubectl logs -l "app.kubernetes.io/component=controller" -c addnode --tail=-1
echo -e "#################################\n"

echo "#######################################"
echo "###### Controller container logs ######"
echo "#######################################"
kubectl logs -l "app.kubernetes.io/component=controller" -c controller --tail=-1
echo -e "#######################################\n"

echo "#################################"
echo "###### Cron container logs ######"
echo "#################################"
kubectl logs -l "app.kubernetes.io/component=controller" -c cron --tail=-1
echo -e "#################################\n"

echo "###################################"
echo "###### API-WF container logs ######"
echo "###################################"
kubectl logs -l "app.kubernetes.io/component=controller" -c api-firewall --tail=-1 || true
echo -e "####################################\n"

echo "####################################################"
echo "###### List directory /opt/wallarm/etc/wallarm #####"
echo "####################################################"
kubectl exec "${POD}" -c controller -- sh -c "ls -laht /opt/wallarm/etc/wallarm && cat /opt/wallarm/etc/wallarm/node.yaml" || true
echo -e "#####################################################\n"

echo "############################################"
echo "###### List directory /var/lib/nginx/wallarm"
echo "############################################"
kubectl exec "${POD}" -c controller -- sh -c "ls -laht /opt/wallarm/var/lib/nginx/wallarm && ls -laht /opt/wallarm/var/lib/nginx/wallarm/shm" || true
echo -e "############################################\n"

echo "############################################################"
echo "###### List directory /opt/wallarm/var/lib/wallarm-acl #####"
echo "############################################################"
kubectl exec "${POD}" -c controller -- sh -c "ls -laht /opt/wallarm/var/lib/wallarm-acl" || true
echo -e "############################################################\n"

echo "##################################################"
echo "###### TARANTOOL Pod - Cron container logs ######"
echo "##################################################"
kubectl logs -l "app.kubernetes.io/component=controller-wallarm-tarantool" -c cron --tail=-1
echo -e "##################################################\n"

echo "######################################################"
echo "###### TARANTOOL Pod - Tarantool container logs ######"
echo "######################################################"
kubectl logs -l "app.kubernetes.io/component=controller-wallarm-tarantool" -c tarantool --tail=-1
echo -e "######################################################\n"
}

function extra_debug_logs {
echo "############################################"
echo "###### Extra cluster debug info ############"
echo "############################################"

echo "Grepping cluster OOMKilled events..."
kubectl get events -A | grep -i OOMKill || true

echo "Displaying pods state in default namespace..."
kubectl get pods

}


Loading

0 comments on commit 782165f

Please sign in to comment.