From 0453daa07e8a8f706fc941b6040b579be09bcfd4 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <141920865+universal-itengineer@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:15:52 +0300 Subject: [PATCH] test(performance): status-access-vms (#44) This is part of the performance tests. Check via ansible ssh access to vm, also added yandex-tank for emulate workload for apis. --------- Signed-off-by: Nikita Korolev Co-authored-by: Tishkov Pavel --- .gitignore | 12 ++ tests/performance/.helmignore | 7 + tests/performance/Taskfile.yaml | 8 + tests/performance/ssh/id_ed | 2 +- .../ansible/Taskfile.ansible.yaml | 21 +++ .../status-access-vms/ansible/ansible.cfg | 29 ++++ .../status-access-vms/ansible/playbook.yaml | 7 + .../status-access-vms/ansible/run.sh | 152 ++++++++++++++++++ .../ansible/vmops/vmops_restart.sh | 61 +++++++ .../status-access-vms/tank/Taskfile.tank.yaml | 24 +++ .../status-access-vms/tank/load.yaml | 17 ++ .../status-access-vms/tank/run_tank.sh | 50 ++++++ 12 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 tests/performance/.helmignore create mode 100644 tests/performance/status-access-vms/ansible/Taskfile.ansible.yaml create mode 100644 tests/performance/status-access-vms/ansible/ansible.cfg create mode 100644 tests/performance/status-access-vms/ansible/playbook.yaml create mode 100755 tests/performance/status-access-vms/ansible/run.sh create mode 100755 tests/performance/status-access-vms/ansible/vmops/vmops_restart.sh create mode 100644 tests/performance/status-access-vms/tank/Taskfile.tank.yaml create mode 100644 tests/performance/status-access-vms/tank/load.yaml create mode 100755 tests/performance/status-access-vms/tank/run_tank.sh diff --git a/.gitignore b/.gitignore index 8a957d706..b4b304a9e 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ __pycache__/ *$py.class .pytest_cache/ +tmp/ tmp/* # Kubernetes @@ -41,3 +42,14 @@ tmp/* # direnv .envrc + +# logs +log/ +logs/ +*.log + +# ansible inventory +inventory/ + +# generated directories +retry/ \ No newline at end of file diff --git a/tests/performance/.helmignore b/tests/performance/.helmignore new file mode 100644 index 000000000..f45ef12c0 --- /dev/null +++ b/tests/performance/.helmignore @@ -0,0 +1,7 @@ +# Match any file or path named .helmignore +.helmignore + +# Match any file or path named .git +.git + +status-access-vms/ \ No newline at end of file diff --git a/tests/performance/Taskfile.yaml b/tests/performance/Taskfile.yaml index 255668034..51f01de06 100644 --- a/tests/performance/Taskfile.yaml +++ b/tests/performance/Taskfile.yaml @@ -8,6 +8,14 @@ includes: shatal: taskfile: ./shatal dir: ./shatal + tank: + taskfile: status-access-vms/tank/Taskfile.tank.yaml + dir: status-access-vms/tank + optional: true + ansible: + taskfile: status-access-vms/ansible/Taskfile.ansible.yaml + dir: status-access-vms/ansible + optional: true vars: COUNT: '{{ .COUNT | default "1" }}' diff --git a/tests/performance/ssh/id_ed b/tests/performance/ssh/id_ed index 52b64f98c..ad9652cae 100644 --- a/tests/performance/ssh/id_ed +++ b/tests/performance/ssh/id_ed @@ -4,4 +4,4 @@ QyNTUxOQAAACBcXFx5sGhpyfLHCWhDeUc5JQT2aVUonOBnWgLCo0KHgAAAAKDCANDUwgDQ 1AAAAAtzc2gtZWQyNTUxOQAAACBcXFx5sGhpyfLHCWhDeUc5JQT2aVUonOBnWgLCo0KHgA AAAED/iI2D9QTc70eISkYFC/TrXG3JpHYLu5FqQhGCTxveElxcXHmwaGnJ8scJaEN5Rzkl BPZpVSic4GdaAsKjQoeAAAAAFnlvdXJfZW1haWxAZXhhbXBsZS5jb20BAgMEBQYH ------END OPENSSH PRIVATE KEY----- \ No newline at end of file +-----END OPENSSH PRIVATE KEY----- diff --git a/tests/performance/status-access-vms/ansible/Taskfile.ansible.yaml b/tests/performance/status-access-vms/ansible/Taskfile.ansible.yaml new file mode 100644 index 000000000..efb98aa37 --- /dev/null +++ b/tests/performance/status-access-vms/ansible/Taskfile.ansible.yaml @@ -0,0 +1,21 @@ +# https://taskfile.dev + +version: "3" + +silent: true + +vars: + SSK_KEY: '{{.SSK_KEY | default "../../ssh/id_ed"}}' + ANSIBLE_CFG: '{{.ANSIBLE_CFG | default ".ansible.cfg"}}' + INVENTORY_FILE: '{{.INVENTORY_FILE | default "inventory/hosts.yml"}}' + +tasks: + help: + desc: "Help about run_tank" + cmds: + - ./run.sh -h + + run: + desc: "Start endless fact-gathering and checks vms availability (need pass namespace like 'run -- -n testnamespace')" + cmds: + - ./run.sh {{.CLI_ARGS}} diff --git a/tests/performance/status-access-vms/ansible/ansible.cfg b/tests/performance/status-access-vms/ansible/ansible.cfg new file mode 100644 index 000000000..28580655e --- /dev/null +++ b/tests/performance/status-access-vms/ansible/ansible.cfg @@ -0,0 +1,29 @@ +[defaults] +ansible_managed=Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S +inventory=$PWD/inventory +interpreter_python=python3 + +forks=25 +transport=smart +host_key_checking=false +# fact_caching=jsonfile +# fact_caching_connection=$PWD/tmp +# fact_caching_timeout=3600 + +# gathering=smart + +bin_ansible_callbacks=true +deprecation_warnings=false + +# retry_files_enabled=true +# retry_files_save_path=./retry + +remote_user=cloud +ansible_ssh_user=cloud +private_key_file=../../ssh/id_ed + + +[ssh_connection] +pipelining=true +ssh_args = -o ProxyCommand='d8 v port-forward --stdio=true %h %p' +ansible_ssh_args = -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=2m -o ConnectionAttempts=50 -o UserKnownHostsFile=/dev/null diff --git a/tests/performance/status-access-vms/ansible/playbook.yaml b/tests/performance/status-access-vms/ansible/playbook.yaml new file mode 100644 index 000000000..f202f0123 --- /dev/null +++ b/tests/performance/status-access-vms/ansible/playbook.yaml @@ -0,0 +1,7 @@ +- name: Run + hosts: all + + tasks: + - name: Print hostname and ip address + ansible.builtin.debug: + msg: "{{ ansible_host }} | {{ ansible_hostname }} - {{ ansible_default_ipv4.address }}" diff --git a/tests/performance/status-access-vms/ansible/run.sh b/tests/performance/status-access-vms/ansible/run.sh new file mode 100755 index 000000000..9fe87ab3d --- /dev/null +++ b/tests/performance/status-access-vms/ansible/run.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +SSK_KEY="../../ssh/id_ed" +ANSIBLE_CFG="./ansible.cfg" +INVENTORY_FILE="inventory/hosts.yml" + +function Help() { +# Display Help + cat <&2 + Help + exit 1 ;; + esac +done + +function enable_unreachable_host { + local ENABLE=$1 + + if $ENABLE; then + echo "Enable write to file unreachable host" + sed -i -E "s|^# retry_files_enabled=true|retry_files_enabled=true|" $ANSIBLE_CFG + sed -i -E "s|^# retry_files_save_path=./retry|retry_files_save_path=./retry|" $ANSIBLE_CFG + else + echo "Disable write to file unreachable host" + sed -i -E "s|^retry_files_enabled=true|# retry_files_enabled=true|" $ANSIBLE_CFG + sed -i -E "s|^retry_files_save_path=./retry|# retry_files_save_path=./retry|" $ANSIBLE_CFG + fi +} + +function darwin_sed { + if $ENABLE; then + echo "Enable write to file unreachable host" + sed -i '' -E "s|^# retry_files_enabled=true|retry_files_enabled=true|" $ANSIBLE_CFG + sed -i '' -E "s|^# retry_files_save_path=./retry|retry_files_save_path=./retry|" $ANSIBLE_CFG + else + echo "Disable write to file unreachable host" + sed -i '' -E "s|^retry_files_enabled=true|# retry_files_enabled=true|" $ANSIBLE_CFG + sed -i '' -E "s|^retry_files_save_path=./retry|# retry_files_save_path=./retry|" $ANSIBLE_CFG + fi +} + +function enable_unreachable_host_file { + local ENABLE=$1 + + if [ "$(uname)" = "Darwin" ]; then + darwin_sed $ENABLE + elif [ "$(uname)" = "Linux" ]; then + enable_unreachable_host $ENABLE + else + echo "unknown OS" + echo "try linux" + enable_unreachable_host $ENABLE + fi + +} + +function prepare_ssh_key { + chmod 600 $SSK_KEY + if [ "$(uname)" = "Darwin" ]; then + sed -i '' -E "s|private_key_file=.+|private_key_file=${SSK_KEY}|" $ANSIBLE_CFG + elif [ "$(uname)" = "Linux" ]; then + sed -i -E "s|private_key_file=.+|private_key_file=${SSK_KEY}|" $ANSIBLE_CFG + else + echo "unknown OS" + echo "try linux" + sed -i -E "s|private_key_file=.+|private_key_file=${SSK_KEY}|" $ANSIBLE_CFG + fi + +} + +exit_handler() { + echo "Exit" + exit 0 +} +trap 'exit_handler' EXIT + +if [ -z $NAMESPACE ]; then echo "Namespace must be defined"; exit 1;fi + +if [[ $UNREACHABLE_HOST_FILE == true ]] ; then + enable_unreachable_host_file true +else + enable_unreachable_host_file false +fi + +function generate_inventory { + VMS=$(kubectl -n $NAMESPACE get vm -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + mkdir -p inventory + echo "--- +all: + hosts:" > $INVENTORY_FILE + + while IFS= read -r VM_NAME; do + echo " ${VM_NAME}.${NAMESPACE}:" >> $INVENTORY_FILE + done <<< "$VMS" +} + +function main { + prepare_ssh_key + ANSIBLE_REPORT_FILE=play_report.log + + while true + do + echo "Generate inventory" + generate_inventory + + echo "Try to access all hosts from inventory $(date); Namespace: $NAMESPACE" + ansible-playbook playbook.yaml | sed -n '/PLAY RECAP/,$p' > $ANSIBLE_REPORT_FILE + while [ ! -f $ANSIBLE_REPORT_FILE ]; do sleep 2; done + + echo $(wc -l $ANSIBLE_REPORT_FILE) + + if [ "$(uname)" = "Darwin" ]; then + HOSTS_TOTAL=$(( $(wc -l $ANSIBLE_REPORT_FILE | grep -Eo '\d{1,7}') - 2 )) + elif [ "$(uname)" = "Linux" ]; then + HOSTS_TOTAL=$(( $(wc -l $ANSIBLE_REPORT_FILE | cut -d ' ' -f1) - 2 )) + fi + + + HOSTS_OK=$(( $(grep -E 'ok=[1-9]+' $ANSIBLE_REPORT_FILE | wc -l) )) + HOSTS_UNREACHABLE=$(( $(grep -E 'unreachable=[1-9]+' $ANSIBLE_REPORT_FILE | wc -l) )) + OK_PCT=$(bc -l <<< "scale=2; $HOSTS_OK/$HOSTS_TOTAL*100") + + if [[ $HOSTS_UNREACHABLE -ne 0 ]]; then + grep -E 'unreachable=[1-9]+' $ANSIBLE_REPORT_FILE + fi + + echo "OK hosts count:$HOSTS_OK pct.:$OK_PCT% | Unreachable hosts $HOSTS_UNREACHABLE | Total hosts $HOSTS_TOTAL" + echo "Wait 2 sec" + echo -e "---\n" + sleep 2 + done +} + +main \ No newline at end of file diff --git a/tests/performance/status-access-vms/ansible/vmops/vmops_restart.sh b/tests/performance/status-access-vms/ansible/vmops/vmops_restart.sh new file mode 100755 index 000000000..be6997ffc --- /dev/null +++ b/tests/performance/status-access-vms/ansible/vmops/vmops_restart.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +VMS_UNREACHBLE_FILE="../ansible/retry/playbook.retry" + +function Help() { +# Display Help + cat <&2 + Help + exit 1 ;; + esac +done + +exit_handler() { + echo "Exit" + exit 0 +} +trap 'exit_handler' EXIT + +if [ ! -f $VMS_UNREACHBLE_FILE ]; then echo "File does not exist"; exit 1;fi +if [ -z $NAMESPACE ]; then echo "Namespace must be defined"; exit 1;fi + +VMS_UNREACHBLE=( $(cut -d '.' -f1 $VMS_UNREACHBLE_FILE) ) + +echo "Generate VirtualMachineOperation for vm in namespace $NAMESPACE" +for vm in "${VMS_UNREACHBLE[@]}"; do + + kubectl apply -f -<&2 + Help + exit 1 ;; + esac +done + +if [ -z $TARGET_ADDRESS ]; then echo "Target addres must be defined"; exit 1;fi +if [ ! -f $TANK_CONFIG_PATH ]; then echo "Config file must be defined"; exit 1;fi + +function prepare_tank_config { + sed -i '' -E "s/address: .+/address: $TARGET_ADDRESS/" $TANK_CONFIG_PATH + sed -i '' -E "s|\[Host: .+\]|\[Host: $TARGET_ADDRESS\]|" $TANK_CONFIG_PATH +} + +function main { + prepare_tank_config + + docker run \ + --rm \ + -v $(pwd)/$TANK_CONFIG_PATH:/var/loadtest/$TANK_CONFIG_PATH \ + -v $SSH_AUTH_SOCK:/ssh-agent -e SSH_AUTH_SOCK=/ssh-agent \ + --net host \ + -it yandex/yandex-tank -c $TANK_CONFIG_PATH +} + +main \ No newline at end of file