From dcf689272f0542e2c12741ece9a8e2ae376e4bc6 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Mon, 12 Feb 2024 23:25:47 -0800 Subject: [PATCH 01/19] [auth] Do not run Gantry if log into registry failed. --- src/lib-gantry.sh | 50 ++++++++++++++++++++--------- tests/gantry_login_spec.sh | 66 +++----------------------------------- 2 files changed, 38 insertions(+), 78 deletions(-) diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index 465d4c4..db6984e 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -37,9 +37,10 @@ _login_registry() { # shellcheck disable=SC2086 if ! LOGIN_MSG=$(echo "${PASSWORD}" | docker ${DOCKER_CONFIG} login --username="${USER}" --password-stdin "${HOST}" 2>&1); then log ERROR "Failed to login to registry${CONFIG_MESSAGE}. ${LOGIN_MSG}" - else - log INFO "Logged into registry${CONFIG_MESSAGE}. ${LOGIN_MSG}" + return 1 fi + log INFO "Logged into registry${CONFIG_MESSAGE}. ${LOGIN_MSG}" + return 0 } gantry_read_registry_username() { @@ -69,10 +70,15 @@ _authenticate_to_registries() { if ! USER=$(gantry_read_registry_username 2>&1); then log ERROR "Failed to set USER: ${USER}" && return 1; fi + local ACCUMULATED_ERRORS=0 if [ -n "${USER}" ]; then _login_registry "${USER}" "${PASSWORD}" "${HOST}" "${CONFIG}" + ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?)) + fi + if [ -z "${CONFIGS_FILE}" ]; then + [ ${ACCUMULATED_ERRORS} -gt 0 ] && return 1 + return 0 fi - [ -z "${CONFIGS_FILE}" ] && return 0 [ ! -r "${CONFIGS_FILE}" ] && log ERROR "Failed to read ${CONFIGS_FILE}." && return 1 local LINE= while read -r LINE; do @@ -86,14 +92,24 @@ _authenticate_to_registries() { USER=$(echo "${LINE}" | cut -d ' ' -f 3) PASSWORD=$(echo "${LINE}" | cut -d ' ' -f 4) OTHERS=$(echo "${LINE}" | cut -d ' ' -f 5-) - if [ -n "${OTHERS}" ] || [ -z "${CONFIG}" ] || \ - [ -z "${HOST}" ] || [ -z "${USER}" ] || [ -z "${PASSWORD}" ]; then - log ERROR "CONFIGS_FILE ${CONFIGS_FILE} format error. A line should contains only \" \"." + local ERROR_MSG= + if [ -n "${OTHERS}" ]; then + ERROR_MSG="Found extra item(s)." + fi + if [ -z "${CONFIG}" ] || [ -z "${HOST}" ] || [ -z "${USER}" ] || [ -z "${PASSWORD}" ]; then + ERROR_MSG="Missing item(s)." + fi + if [ -n "${ERROR_MSG}" ]; then + log ERROR "CONFIGS_FILE ${CONFIGS_FILE} format error. ${ERROR_MSG} A line should contains exactly \" \"." log DEBUG "CONFIGS_FILE ${CONFIGS_FILE} format error. Got \"${LINE}\"." - return 1 + ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1)) + continue fi _login_registry "${USER}" "${PASSWORD}" "${HOST}" "${CONFIG}" + ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?)) done < <(cat "${CONFIGS_FILE}"; echo;) + [ ${ACCUMULATED_ERRORS} -gt 0 ] && return 1 + return 0 } _send_notification() { @@ -134,7 +150,7 @@ _remove_container() { log ERROR "Failed to list ${STATUS} containers with image ${IMAGE}."; echo "${CIDS}" | log_lines ERROR return 1; - fi; + fi local CID CNAME CRM_MSG for CID in ${CIDS}; do CNAME=$(docker container inspect --format '{{.Name}}' "${CID}"); @@ -144,8 +160,8 @@ _remove_container() { continue; fi log INFO "Removed ${STATUS} container ${CNAME}. It was using image ${IMAGE}."; - done; -}; + done +} gantry_remove_images() { local IMAGES_TO_REMOVE="${1}" @@ -154,16 +170,16 @@ gantry_remove_images() { if ! docker image inspect "${IMAGE}" 1>/dev/null 2>&1 ; then log DEBUG "There is no image ${IMAGE} on the node."; continue; - fi; + fi _remove_container "${IMAGE}" exited; _remove_container "${IMAGE}" dead; if ! RMI_MSG=$(docker rmi "${IMAGE}" 2>&1); then log ERROR "Failed to remove image ${IMAGE}."; echo "${RMI_MSG}" | log_lines ERROR continue; - fi; + fi log INFO "Removed image ${IMAGE}."; - done; + done log INFO "Done removing images."; } @@ -328,9 +344,9 @@ _current_container_name() { _static_variable_add_unique_to_list STATIC_VAR_CURRENT_CONTAINER_NAME "${NAME}" echo "${NAME}"; return 0; - done; - done; - done; + done + done + done return 0; } @@ -631,6 +647,8 @@ _rollback_service() { log INFO "Rolled back ${SERVICE_NAME}." } +# return 0 when there is no error or failure. +# return 1 when there are error(s) or failure(s). _update_single_service() { local UPDATE_TIMEOUT_SECONDS="${GANTRY_UPDATE_TIMEOUT_SECONDS:-300}" local UPDATE_OPTIONS="${GANTRY_UPDATE_OPTIONS:-""}" diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh index 097206e..85823bc 100644 --- a/tests/gantry_login_spec.sh +++ b/tests/gantry_login_spec.sh @@ -150,7 +150,7 @@ Describe 'Login' The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End End - Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format_extra" "container_test:false" + Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:false" TEST_NAME="test_login_REGISTRY_CONFIGS_FILE_bad_format_extra" IMAGE_WITH_TAG=$(get_image_with_tag) SERVICE_NAME="gantry-test-$(unique_id)" @@ -170,65 +170,6 @@ Describe 'Login' docker service update --quiet --label-add "${LABEL}=${CONFIG}" "${SERVICE_NAME}" # Add an extra item to the line. echo "${CONFIG} ${REGISTRY} ${USERNAME} ${PASSWORD} Extra" >> "${CONFIGS_FILE}" - export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}" - local RETURN_VALUE= - run_gantry "${TEST_NAME}" - RETURN_VALUE="${?}" - rm "${CONFIGS_FILE}" - [ -d "${CONFIG}" ] && rm -r "${CONFIG}" - return "${RETURN_VALUE}" - } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_login_REGISTRY_CONFIGS_FILE_bad_format_extra "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" - The status should be failure - The stdout should satisfy display_output - The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "format error.*A line should contains only \" \".*" - The stderr should satisfy spec_expect_no_message "Logged into registry *${TEST_REGISTRY} for config ${CONFIG}" - The stderr should satisfy spec_expect_message "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}" - The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*" - The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" - The stderr should satisfy spec_expect_message "${NO_IMAGES_TO_REMOVE}" - The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" - The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" - The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" - End - End - Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format_missing" "container_test:false" - TEST_NAME="test_login_REGISTRY_CONFIGS_FILE_bad_format_missing" - IMAGE_WITH_TAG=$(get_image_with_tag) - SERVICE_NAME="gantry-test-$(unique_id)" - test_login_REGISTRY_CONFIGS_FILE_bad_format_missing() { - local TEST_NAME=${1} - local SERVICE_NAME=${2} - local REGISTRY=${3} - local USERNAME=${4} - local PASSWORD=${5} - if [ -z "${REGISTRY}" ] || [ -z "${USERNAME}" ] || [ -z "${PASSWORD}" ]; then - echo "No REGISTRY, USERNAME or PASSWORD provided." >&2 - return 1 - fi - local LABEL="gantry.auth.config" - CONFIG="C$(unique_id)" - CONFIGS_FILE=$(mktemp) - docker service update --quiet --label-add "${LABEL}=${CONFIG}" "${SERVICE_NAME}" # Missing an item from the line. echo "${REGISTRY} ${USERNAME} ${PASSWORD}" >> "${CONFIGS_FILE}" export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}" @@ -242,11 +183,12 @@ Describe 'Login' Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_gantry' - When call test_login_REGISTRY_CONFIGS_FILE_bad_format_missing "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When call test_login_REGISTRY_CONFIGS_FILE_bad_format_extra "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "format error.*A line should contains only \" \".*" + The stderr should satisfy spec_expect_message "format error.*Found extra item\(s\)" + The stderr should satisfy spec_expect_message "format error.*Missing item\(s\)" The stderr should satisfy spec_expect_no_message "Logged into registry *${TEST_REGISTRY} for config ${CONFIG}" The stderr should satisfy spec_expect_message "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" From 23d672359718cfbfbf1a1062663a2fc31e86d4fb Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Wed, 14 Feb 2024 17:53:19 -0800 Subject: [PATCH 02/19] [gantry] When calling wait_service_state, do not add --complete We do not add --detach to docker_global_job should, thus that function should check converge. --- src/lib-common.sh | 42 +++++++++++++++++++----------------------- src/lib-gantry.sh | 26 ++++++++++++++++++-------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/src/lib-common.sh b/src/lib-common.sh index a4dc995..0ef454b 100755 --- a/src/lib-common.sh +++ b/src/lib-common.sh @@ -83,7 +83,7 @@ log() { if _log_level "${1}" >/dev/null; then LEVEL="${1}"; shift; - fi; + fi _log_formatter "${LEVEL}" "$(date -Iseconds)" "${NODE_NAME}" "${LOG_SCOPE}" "${@}"; } @@ -329,50 +329,45 @@ wait_service_state() { local WAIT_RUNNING WAIT_COMPLETE; WAIT_RUNNING=$(echo "${@}" | grep -q -- "--running" && echo "true" || echo "false") WAIT_COMPLETE=$(echo "${@}" | grep -q -- "--complete" && echo "true" || echo "false") - local RETURN_VALUE=0 local SLEEP_SECONDS=1 local STATES= - STATES=$(_docker_service_task_states "${SERVICE_NAME}" 2>&1) - while is_true "${WAIT_RUNNING}" || is_true "${WAIT_COMPLETE}" ; do + while STATES=$(_docker_service_task_states "${SERVICE_NAME}" 2>&1); do local NUM_LINES=0 local NUM_RUNS=0 local NUM_DONES=0 local NUM_FAILS=0 while read -r LINE; do [ -z "${LINE}" ] && continue; + log INFO "Service ${SERVICE_NAME}: ${LINE}." NUM_LINES=$((NUM_LINES+1)); echo "${LINE}" | grep -q "Running" && NUM_RUNS=$((NUM_RUNS+1)); echo "${LINE}" | grep -q "Complete" && NUM_DONES=$((NUM_DONES+1)); echo "${LINE}" | grep -q "Failed" && NUM_FAILS=$((NUM_FAILS+1)); done < <(echo "${STATES}") - if [ ${NUM_LINES} -gt 0 ]; then - if ${WAIT_RUNNING} && [ ${NUM_RUNS} -eq ${NUM_LINES} ]; then - break + if [ "${NUM_LINES}" -gt 0 ]; then + if "${WAIT_RUNNING}" && [ "${NUM_RUNS}" -eq "${NUM_LINES}" ]; then + return 0; fi - if ${WAIT_COMPLETE} && [ ${NUM_DONES} -eq ${NUM_LINES} ]; then - break + if "${WAIT_COMPLETE}" && [ "${NUM_DONES}" -eq "${NUM_LINES}" ]; then + return 0; fi - if ${WAIT_COMPLETE} && [ ${NUM_FAILS} -gt 0 ]; then + if "${WAIT_COMPLETE}" && [ "${NUM_FAILS}" -gt 0 ]; then # Get return value of the task from the string "task: non-zero exit (1)". - local TASK_STATE= local TASK_RETURN_VALUE= - TASK_STATE=$(echo "${STATES}" | grep "Failed") - TASK_RETURN_VALUE=$(echo "${TASK_STATE}" | sed -n 's/.*task: non-zero exit (\([0-9]\+\)).*/\1/p') + TASK_RETURN_VALUE=$(echo "${STATES}" | grep "Failed" | sed -n 's/.*task: non-zero exit (\([0-9]\+\)).*/\1/p') # Get the first error code. + local RETURN_VALUE=0 RETURN_VALUE=$(echo "${TASK_RETURN_VALUE:-1}" | cut -d ' ' -f 1) - break + return "${RETURN_VALUE}" fi fi - sleep "${SLEEP_SECONDS}" - if ! STATES=$(_docker_service_task_states "${SERVICE_NAME}" 2>&1); then - log ERROR "Failed to obtain task states of service ${SERVICE_NAME}: ${STATES}" - return 1 + if ! ("${WAIT_RUNNING}" || "${WAIT_COMPLETE}"); then + return 0; fi + sleep "${SLEEP_SECONDS}" done - echo "${STATES}" | while read -r LINE; do - log INFO "Service ${SERVICE_NAME}: ${LINE}." - done - return "${RETURN_VALUE}" + log ERROR "Failed to obtain task states of service ${SERVICE_NAME}: ${STATES}" + return 1 } docker_service_remove() { @@ -389,7 +384,8 @@ docker_service_remove() { # We do not expect failures when using docker_global_job. # Docker will try to restart the failed tasks. -# We do not check the converge of the service. It must be used togther with wait_service_state. +# We do not check the converge of the service, thus some jobs may failed on some nodes. +# It is better to be used togther with wait_service_state. docker_global_job() { local SERVICE_NAME= SERVICE_NAME=$(_get_docker_command_name_arg "${@}") diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index db6984e..55f6a3d 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -20,6 +20,9 @@ _login_registry() { local PASSWORD="${2}" local HOST="${3}" local CONFIG="${4}" + if [ -z "${USER}" ] && [ -z "${PASSWORD}" ] && [ -z "${HOST}" ] && [ -z "${CONFIG}" ]; then + return 0 + fi [ -z "${USER}" ] && log ERROR "USER is empty." && return 1 [ -z "${PASSWORD}" ] && log ERROR "PASSWORD is empty." && return 1 local DOCKER_CONFIG= @@ -57,26 +60,32 @@ gantry_read_registry_host() { _authenticate_to_registries() { local CONFIGS_FILE="${GANTRY_REGISTRY_CONFIGS_FILE:-""}" + local ACCUMULATED_ERRORS=0 local CONFIG HOST PASSWORD USER if ! CONFIG=$(read_config GANTRY_REGISTRY_CONFIG 2>&1); then - log ERROR "Failed to set CONFIG: ${CONFIG}" && return 1; + log ERROR "Failed to read CONFIG: ${CONFIG}" + ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1)) fi if ! HOST=$(gantry_read_registry_host 2>&1); then - log ERROR "Failed to set HOST: ${HOST}" && return 1; + log ERROR "Failed to read HOST: ${HOST}" + ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1)) fi if ! PASSWORD=$(gantry_read_registry_password 2>&1); then - log ERROR "Failed to set PASSWORD: ${PASSWORD}" && return 1; + log ERROR "Failed to read PASSWORD: ${PASSWORD}" + ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1)) fi if ! USER=$(gantry_read_registry_username 2>&1); then - log ERROR "Failed to set USER: ${USER}" && return 1; + log ERROR "Failed to read USER: ${USER}" + ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1)) fi - local ACCUMULATED_ERRORS=0 - if [ -n "${USER}" ]; then + if [ "${ACCUMULATED_ERRORS}" -gt 0 ]; then + log ERROR "Skip logging in due to previous errors." + else _login_registry "${USER}" "${PASSWORD}" "${HOST}" "${CONFIG}" ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?)) fi if [ -z "${CONFIGS_FILE}" ]; then - [ ${ACCUMULATED_ERRORS} -gt 0 ] && return 1 + [ "${ACCUMULATED_ERRORS}" -gt 0 ] && return 1 return 0 fi [ ! -r "${CONFIGS_FILE}" ] && log ERROR "Failed to read ${CONFIGS_FILE}." && return 1 @@ -193,6 +202,7 @@ _remove_images() { return 0 fi local SERVICE_NAME="${1:-"gantry-image-remover"}" + SERVICE_NAME=$(echo "${SERVICE_NAME}" | tr ' ' '-') docker_service_remove "${SERVICE_NAME}" local IMAGES_TO_REMOVE= IMAGES_TO_REMOVE=$(_static_variable_read_list STATIC_VAR_IMAGES_TO_REMOVE) @@ -225,7 +235,7 @@ _remove_images() { "${IMAGES_REMOVER}" 2>&1); then log ERROR "Failed to remove images: ${RMI_MSG}" fi - wait_service_state "${SERVICE_NAME}" --complete; + wait_service_state "${SERVICE_NAME}" docker_service_logs "${SERVICE_NAME}" docker_service_remove "${SERVICE_NAME}" } From 814cc455a20061dc8c543b51fe01646760681413 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Wed, 14 Feb 2024 18:32:07 -0800 Subject: [PATCH 03/19] [tests] Use "run" instead of "call" in tests. "run" starts a subroutine. So each test will have their own independent env. --- tests/gantry_cleanup_images_spec.sh | 36 +++++--- tests/gantry_entrypoint_spec.sh | 44 ++++++--- tests/gantry_filters_spec.sh | 44 +++++---- tests/gantry_job_spec.sh | 39 +++++--- tests/gantry_login_spec.sh | 36 +++++--- tests/gantry_manifest_spec.sh | 65 +++++++------ tests/gantry_multiple_services_spec.sh | 20 ++-- tests/gantry_no_running_tasks_spec.sh | 28 +++--- tests/gantry_notify_spec.sh | 23 +++-- tests/gantry_options_spec.sh | 50 +++++----- tests/gantry_rollback_spec.sh | 52 ++++++++--- tests/gantry_simple_spec.sh | 50 ++++++---- tests/spec_gantry_test_helper.sh | 121 ++++++++++++++++++------- 13 files changed, 387 insertions(+), 221 deletions(-) diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh index 4ce5f4b..0b612b7 100644 --- a/tests/gantry_cleanup_images_spec.sh +++ b/tests/gantry_cleanup_images_spec.sh @@ -22,17 +22,19 @@ Describe 'Cleanup_images' AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_CLEANUP_IMAGES_false" "container_test:true" TEST_NAME="test_CLEANUP_IMAGES_false" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_CLEANUP_IMAGES_false() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_CLEANUP_IMAGES="false" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_CLEANUP_IMAGES_false "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_CLEANUP_IMAGES_false "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -60,19 +62,21 @@ Describe 'Cleanup_images' End Describe "test_CLEANUP_IMAGES_OPTIONS_bad" "container_test:true" TEST_NAME="test_CLEANUP_IMAGES_OPTIONS_bad" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_CLEANUP_IMAGES_OPTIONS_bad() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_CLEANUP_IMAGES="true" # Image remover would fail due to the incorrect option. export GANTRY_CLEANUP_IMAGES_OPTIONS="--incorrect-option" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_CLEANUP_IMAGES_OPTIONS_bad "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_CLEANUP_IMAGES_OPTIONS_bad "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -102,18 +106,20 @@ Describe 'Cleanup_images' End Describe "test_CLEANUP_IMAGES_OPTIONS_good" "container_test:true" TEST_NAME="test_CLEANUP_IMAGES_OPTIONS_good" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_CLEANUP_IMAGES_OPTIONS_good() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_CLEANUP_IMAGES="true" export GANTRY_CLEANUP_IMAGES_OPTIONS="--container-label=test" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_CLEANUP_IMAGES_OPTIONS_good "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_CLEANUP_IMAGES_OPTIONS_good "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_entrypoint_spec.sh b/tests/gantry_entrypoint_spec.sh index a28013b..0ab722e 100644 --- a/tests/gantry_entrypoint_spec.sh +++ b/tests/gantry_entrypoint_spec.sh @@ -21,10 +21,12 @@ Describe 'Entrypoint' AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_DOCKER_HOST_not_swarm_manager" "container_test:false" TEST_NAME="test_DOCKER_HOST_not_swarm_manager" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_DOCKER_HOST_not_swarm_manager() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export DOCKER_HOST="8.8.8.8:53" local RETURN_VALUE=0 run_gantry "${TEST_NAME}" @@ -32,10 +34,10 @@ Describe 'Entrypoint' export DOCKER_HOST= return "${RETURN_VALUE}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_DOCKER_HOST_not_swarm_manager "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_DOCKER_HOST_not_swarm_manager "${TEST_NAME}" "${SERVICE_NAME}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output @@ -65,17 +67,19 @@ Describe 'Entrypoint' End Describe "test_SLEEP_SECONDS_not_a_number" "container_test:false" TEST_NAME="test_SLEEP_SECONDS_not_a_number" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_SLEEP_SECONDS_not_a_number() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_SLEEP_SECONDS="NotANumber" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_SLEEP_SECONDS_not_a_number "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_SLEEP_SECONDS_not_a_number "${TEST_NAME}" "${SERVICE_NAME}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output @@ -107,7 +111,7 @@ Describe 'Entrypoint' Describe "test_IMAGES_TO_REMOVE_none_empty" "container_test:true" # Test the remove image entrypoint. To improve coverage. TEST_NAME="test_IMAGES_TO_REMOVE_none_empty" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0" IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2" @@ -133,7 +137,17 @@ Describe 'Entrypoint' # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks. wait_zero_running_tasks "${SERVICE_NAME0}" # Do not creat the Image IMAGE_WITH_TAG2, to run the test on a non-exist image. + } + test_IMAGES_TO_REMOVE_none_empty() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + local IMAGE_WITH_TAG=${3} + local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0" + local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" + local IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2" + reset_gantry_env "${SERVICE_NAME}" export GANTRY_IMAGES_TO_REMOVE="${IMAGE_WITH_TAG0} ${IMAGE_WITH_TAG1} ${IMAGE_WITH_TAG2}" + run_gantry "${TEST_NAME}" } test_end() { local TEST_NAME=${1} @@ -150,10 +164,10 @@ Describe 'Entrypoint' prune_local_test_image "${IMAGE_WITH_TAG1}" finalize_test "${TEST_NAME}" } - Before "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call run_gantry "${TEST_NAME}" + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_IMAGES_TO_REMOVE_none_empty "${TEST_NAME}" "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" The status should be success The stdout should satisfy display_output The stdout should satisfy spec_expect_message "Removed exited container.*${SERVICE_NAME0}.*${IMAGE_WITH_TAG0}" diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh index f1bd550..dbc54ed 100644 --- a/tests/gantry_filters_spec.sh +++ b/tests/gantry_filters_spec.sh @@ -21,18 +21,19 @@ Describe 'Filters' AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_SERVICES_FILTERS_bad" "container_test:false" TEST_NAME="test_SERVICES_FILTERS_bad" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_SERVICES_FILTERS_bad() { local TEST_NAME=${1} local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_SERVICES_FILTERS="BadFilterValue" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_SERVICES_FILTERS_bad "${TEST_NAME}" "${SERVICE_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_SERVICES_FILTERS_bad "${TEST_NAME}" "${SERVICE_NAME}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output @@ -62,18 +63,19 @@ Describe 'Filters' End Describe "test_SERVICES_EXCLUDED" "container_test:true" TEST_NAME="test_SERVICES_EXCLUDED" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_SERVICES_EXCLUDED() { local TEST_NAME=${1} local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_SERVICES_EXCLUDED="${SERVICE_NAME}" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_SERVICES_EXCLUDED "${TEST_NAME}" "${SERVICE_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_SERVICES_EXCLUDED "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -102,18 +104,19 @@ Describe 'Filters' End Describe "test_SERVICES_EXCLUDED_FILTERS" "container_test:true" TEST_NAME="test_SERVICES_EXCLUDED_FILTERS" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_SERVICES_EXCLUDED_FILTERS() { local TEST_NAME=${1} local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_SERVICES_EXCLUDED_FILTERS="name=${SERVICE_NAME}" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_SERVICES_EXCLUDED_FILTERS "${TEST_NAME}" "${SERVICE_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_SERVICES_EXCLUDED_FILTERS "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -142,18 +145,19 @@ Describe 'Filters' End Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:false" TEST_NAME="test_SERVICES_EXCLUDED_FILTERS_bad" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_SERVICES_EXCLUDED_FILTERS_bad() { local TEST_NAME=${1} local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_SERVICES_EXCLUDED_FILTERS="BadFilterValue" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_SERVICES_EXCLUDED_FILTERS_bad "${TEST_NAME}" "${SERVICE_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_SERVICES_EXCLUDED_FILTERS_bad "${TEST_NAME}" "${SERVICE_NAME}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_job_spec.sh b/tests/gantry_job_spec.sh index 8224341..bbd9bc6 100644 --- a/tests/gantry_job_spec.sh +++ b/tests/gantry_job_spec.sh @@ -21,12 +21,18 @@ Describe 'Job' AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_jobs_skipping" "container_test:true" TEST_NAME="test_jobs_skipping" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - Before "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call run_gantry "${TEST_NAME}" + test_jobs_skipping() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + run_gantry "${TEST_NAME}" + } + BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_jobs_skipping "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -55,19 +61,21 @@ Describe 'Job' End Describe "test_jobs_UPDATE_JOBS_true" "container_test:true" TEST_NAME="test_jobs_UPDATE_JOBS_true" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_jobs_UPDATE_JOBS_true() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_UPDATE_JOBS="true" # The job may not reach the desired "Complete" state and blocking update CLI. So add "--detach=true" export GANTRY_UPDATE_OPTIONS="--detach=true" run_gantry "${TEST_NAME}" } - Before "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_jobs_UPDATE_JOBS_true "${TEST_NAME}" + BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_jobs_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -97,7 +105,7 @@ Describe 'Job' End Describe "test_jobs_UPDATE_JOBS_true_no_running_tasks" "container_test:true" TEST_NAME="test_jobs_UPDATE_JOBS_true_no_running_tasks" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" TASK_SECONDS=15 test_jobs_UPDATE_JOBS_true_no_running_tasks() { @@ -105,14 +113,15 @@ Describe 'Job' local SERVICE_NAME=${2} # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks. wait_zero_running_tasks "${SERVICE_NAME}" + reset_gantry_env "${SERVICE_NAME}" export GANTRY_UPDATE_JOBS="true" run_gantry "${TEST_NAME}" } # The task will finish in ${TASK_SECONDS} seconds - Before "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_jobs_UPDATE_JOBS_true_no_running_tasks "${TEST_NAME}" "${SERVICE_NAME}" + BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_jobs_UPDATE_JOBS_true_no_running_tasks "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh index 85823bc..1fe2f8f 100644 --- a/tests/gantry_login_spec.sh +++ b/tests/gantry_login_spec.sh @@ -22,8 +22,9 @@ Describe 'Login' # Here are just simple login tests. Describe "test_login_config" "container_test:true" TEST_NAME="test_login_config" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_config() { local TEST_NAME=${1} local SERVICE_NAME=${2} @@ -41,6 +42,7 @@ Describe 'Login' docker service update --quiet --label-add "${LABEL}=${CONFIG}" "${SERVICE_NAME}" echo "${USERNAME}" > "${USER_FILE}" echo "${PASSWORD}" > "${PASS_FILE}" + reset_gantry_env "${SERVICE_NAME}" export GANTRY_REGISTRY_CONFIG="${CONFIG}" export GANTRY_REGISTRY_HOST="${REGISTRY}" export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}" @@ -53,10 +55,10 @@ Describe 'Login' [ -d "${CONFIG}" ] && rm -r "${CONFIG}" return "${RETURN_VALUE}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_login_config "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_login_config "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -86,8 +88,9 @@ Describe 'Login' End Describe "test_login_REGISTRY_CONFIGS_FILE" "container_test:true" TEST_NAME="test_login_REGISTRY_CONFIGS_FILE" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_REGISTRY_CONFIGS_FILE() { local TEST_NAME=${1} local SERVICE_NAME=${2} @@ -104,6 +107,7 @@ Describe 'Login' docker service update --quiet --label-add "${LABEL}=${CONFIG}" "${SERVICE_NAME}" echo "# Test comments: CONFIG REGISTRY USERNAME PASSWORD" >> "${CONFIGS_FILE}" echo "${CONFIG} ${REGISTRY} ${USERNAME} ${PASSWORD}" >> "${CONFIGS_FILE}" + reset_gantry_env "${SERVICE_NAME}" export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}" # Since we pass credentials via the configs file, we can use other envs to login to docker hub and check the rate. # However we do not actually check whether we read rates correctly, in case password or usrename for docker hub is not set. @@ -119,10 +123,10 @@ Describe 'Login' [ -d "${CONFIG}" ] && rm -r "${CONFIG}" return "${RETURN_VALUE}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_login_REGISTRY_CONFIGS_FILE "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_login_REGISTRY_CONFIGS_FILE "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -152,8 +156,9 @@ Describe 'Login' End Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:false" TEST_NAME="test_login_REGISTRY_CONFIGS_FILE_bad_format_extra" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_REGISTRY_CONFIGS_FILE_bad_format_extra() { local TEST_NAME=${1} local SERVICE_NAME=${2} @@ -172,6 +177,7 @@ Describe 'Login' echo "${CONFIG} ${REGISTRY} ${USERNAME} ${PASSWORD} Extra" >> "${CONFIGS_FILE}" # Missing an item from the line. echo "${REGISTRY} ${USERNAME} ${PASSWORD}" >> "${CONFIGS_FILE}" + reset_gantry_env "${SERVICE_NAME}" export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}" local RETURN_VALUE= run_gantry "${TEST_NAME}" @@ -180,10 +186,10 @@ Describe 'Login' [ -d "${CONFIG}" ] && rm -r "${CONFIG}" return "${RETURN_VALUE}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_login_REGISTRY_CONFIGS_FILE_bad_format_extra "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_login_REGISTRY_CONFIGS_FILE_bad_format_extra "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh index 309657c..602e60c 100644 --- a/tests/gantry_manifest_spec.sh +++ b/tests/gantry_manifest_spec.sh @@ -21,18 +21,20 @@ Describe 'Manifest_command' AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_MANIFEST_CMD_none" "container_test:true" TEST_NAME="test_MANIFEST_CMD_none" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_MANIFEST_CMD_none() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_MANIFEST_CMD="none" export GANTRY_UPDATE_OPTIONS="--force" run_gantry "${TEST_NAME}" } - Before "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_MANIFEST_CMD_none "${TEST_NAME}" + BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_MANIFEST_CMD_none "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -67,20 +69,22 @@ Describe 'Manifest_command' End Describe "test_MANIFEST_CMD_none_SERVICES_SELF" "container_test:true" TEST_NAME="test_MANIFEST_CMD_none_SERVICES_SELF" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_MANIFEST_CMD_none_SERVICES_SELF() { # If the service is self, it will always run manifest checking. Even if the CMD is set to none local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" # Explicitly set GANTRY_SERVICES_SELF export GANTRY_SERVICES_SELF="${SERVICE_NAME}" export GANTRY_MANIFEST_CMD="none" run_gantry "${TEST_NAME}" } - Before "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_MANIFEST_CMD_none_SERVICES_SELF "${TEST_NAME}" + BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_MANIFEST_CMD_none_SERVICES_SELF "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -109,18 +113,20 @@ Describe 'Manifest_command' End Describe "test_MANIFEST_CMD_manifest" "container_test:true" TEST_NAME="test_MANIFEST_CMD_manifest" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_MANIFEST_CMD_manifest() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_MANIFEST_OPTIONS="--insecure" export GANTRY_MANIFEST_CMD="manifest" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_MANIFEST_CMD_manifest "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_MANIFEST_CMD_manifest "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -149,18 +155,20 @@ Describe 'Manifest_command' End Describe "test_MANIFEST_CMD_unsupported_cmd" "container_test:false" TEST_NAME="test_MANIFEST_CMD_unsupported_cmd" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_MANIFEST_CMD_unsupported_cmd() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_MANIFEST_OPTIONS="--insecure" export GANTRY_MANIFEST_CMD="unsupported_cmd" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_MANIFEST_CMD_unsupported_cmd "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_MANIFEST_CMD_unsupported_cmd "${TEST_NAME}" "${SERVICE_NAME}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output @@ -191,7 +199,7 @@ Describe 'Manifest_command' End Describe "test_MANIFEST_CMD_failure" "container_test:false" TEST_NAME="test_MANIFEST_CMD_failure" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_start() { # This test assumes that the IMAGE_WITH_TAG does not exist on the registry. @@ -203,12 +211,17 @@ Describe 'Manifest_command' # No push image to the registry. Checking new image would fail. build_test_image "${IMAGE_WITH_TAG}" start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" 2>&1 - export GANTRY_SERVICES_FILTERS="name=${SERVICE_NAME}" } - Before "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call run_gantry "${TEST_NAME}" + test_MANIFEST_CMD_failure() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + run_gantry "${TEST_NAME}" + } + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_MANIFEST_CMD_failure "${TEST_NAME}" "${SERVICE_NAME}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_multiple_services_spec.sh b/tests/gantry_multiple_services_spec.sh index d403ef0..dedd02d 100644 --- a/tests/gantry_multiple_services_spec.sh +++ b/tests/gantry_multiple_services_spec.sh @@ -21,7 +21,7 @@ Describe 'Multiple_services' AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_multiple_services_excluded_filters" "container_test:true" TEST_NAME="test_multiple_services_excluded_filters" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" SERVICE_NAME0="${SERVICE_NAME}-0" SERVICE_NAME1="${SERVICE_NAME}-1" @@ -49,11 +49,17 @@ Describe 'Multiple_services' build_and_push_test_image "${IMAGE_WITH_TAG}" start_replicated_service "${SERVICE_NAME4}" "${IMAGE_WITH_TAG}" start_replicated_service "${SERVICE_NAME5}" "${IMAGE_WITH_TAG}" - - export GANTRY_SERVICES_FILTERS="name=${SERVICE_NAME}" + } + test_multiple_services_excluded_filters() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + local SERVICE_NAME1="${SERVICE_NAME}-1" + local SERVICE_NAME2="${SERVICE_NAME}-2" + reset_gantry_env "${SERVICE_NAME}" # test both the list of names and the filters export GANTRY_SERVICES_EXCLUDED="${SERVICE_NAME1}" export GANTRY_SERVICES_EXCLUDED_FILTERS="name=${SERVICE_NAME2}" + run_gantry "${TEST_NAME}" } test_end() { local TEST_NAME=${1} @@ -74,10 +80,10 @@ Describe 'Multiple_services' prune_local_test_image "${IMAGE_WITH_TAG}" finalize_test "${TEST_NAME}" } - Before "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call run_gantry "${TEST_NAME}" + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_multiple_services_excluded_filters "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_no_running_tasks_spec.sh b/tests/gantry_no_running_tasks_spec.sh index 3105dd5..2dc9193 100644 --- a/tests/gantry_no_running_tasks_spec.sh +++ b/tests/gantry_no_running_tasks_spec.sh @@ -21,19 +21,20 @@ Describe "No_Running_Tasks" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_no_running_tasks_replicated" "container_test:true" TEST_NAME="test_no_running_tasks_replicated" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_no_running_tasks_replicated() { local TEST_NAME=${1} local SERVICE_NAME=${2} docker service update --quiet --replicas=0 "${SERVICE_NAME}" wait_zero_running_tasks "${SERVICE_NAME}" + reset_gantry_env "${SERVICE_NAME}" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_no_running_tasks_replicated "${TEST_NAME}" "${SERVICE_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_no_running_tasks_replicated "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -65,7 +66,7 @@ Describe "No_Running_Tasks" End Describe "test_no_running_tasks_global" "container_test:true" TEST_NAME="test_no_running_tasks_global" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_start() { local TEST_NAME=${1} @@ -77,14 +78,19 @@ Describe "No_Running_Tasks" build_and_push_test_image "${IMAGE_WITH_TAG}" "${TASK_SECONDS}" start_global_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" build_and_push_test_image "${IMAGE_WITH_TAG}" - export GANTRY_SERVICES_FILTERS="name=${SERVICE_NAME}" # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks. wait_zero_running_tasks "${SERVICE_NAME}" } - Before "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call run_gantry "${TEST_NAME}" + test_no_running_tasks_global() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + run_gantry "${TEST_NAME}" + } + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_no_running_tasks_global "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh index fef7ef2..555c5fb 100644 --- a/tests/gantry_notify_spec.sh +++ b/tests/gantry_notify_spec.sh @@ -23,7 +23,7 @@ Describe 'Notify' AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_notify_apprise" "container_test:true" TEST_NAME="test_notify_apprise" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_notify_apprise() { local TEST_NAME=${1} @@ -44,6 +44,7 @@ Describe 'Notify' axllent/mailpit \ --smtp "localhost:${SMTP_PORT}" --listen "localhost:${EMAIL_API_PORT}" \ --smtp-auth-accept-any --smtp-auth-allow-insecure + reset_gantry_env "${SERVICE_NAME}" export GANTRY_NOTIFICATION_APPRISE_URL="http://localhost:${APPRISE_PORT}/notify" export GANTRY_NOTIFICATION_TITLE="TEST_TITLE" run_gantry "${TEST_NAME}" @@ -60,10 +61,10 @@ Describe 'Notify' docker_remove "${SERVICE_NAME_MAILPIT}" return "${RETURN_VALUE}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_notify_apprise "${TEST_NAME}" "${SERVICE_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_notify_apprise "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stdout should satisfy spec_expect_message "Subject.*1 services updated 0 failed TEST_TITLE" @@ -94,17 +95,19 @@ Describe 'Notify' End Describe "test_notify_apprise_bad_url" "container_test:true" TEST_NAME="test_notify_apprise_bad_url" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_notify_apprise_bad_url() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_NOTIFICATION_APPRISE_URL="http://bad-url/notify" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_notify_apprise_bad_url "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_notify_apprise_bad_url "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_options_spec.sh b/tests/gantry_options_spec.sh index 2ba626d..0790188 100644 --- a/tests/gantry_options_spec.sh +++ b/tests/gantry_options_spec.sh @@ -21,18 +21,20 @@ Describe 'Options' AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_options_LOG_LEVEL_none" "container_test:true" TEST_NAME="test_options_LOG_LEVEL_none" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_options_LOG_LEVEL_none() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" # Same as test_new_image_yes, except set LOG_LEVEL to NONE export GANTRY_LOG_LEVEL=NONE run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_options_LOG_LEVEL_none "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_options_LOG_LEVEL_none "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" @@ -42,7 +44,7 @@ Describe 'Options' End Describe "test_options_UPDATE_OPTIONS" "container_test:true" TEST_NAME="test_options_UPDATE_OPTIONS" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" _read_service_label() { local SERVICE_NAME="${1}" @@ -54,10 +56,10 @@ Describe 'Options' local SERVICE_NAME=${2} local LABEL="gantry.test" local LABEL_VALUE= - export GANTRY_UPDATE_OPTIONS="--label-add=${LABEL}=${SERVICE_NAME}" - LABEL_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}") echo "Before updating: LABEL_VALUE=${LABEL_VALUE}" + reset_gantry_env "${SERVICE_NAME}" + export GANTRY_UPDATE_OPTIONS="--label-add=${LABEL}=${SERVICE_NAME}" local RETURN_VALUE= run_gantry "${TEST_NAME}" RETURN_VALUE="${?}" @@ -65,10 +67,10 @@ Describe 'Options' echo "After updating: LABEL_VALUE=${LABEL_VALUE}" return "${RETURN_VALUE}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_options_UPDATE_OPTIONS "${TEST_NAME}" "${SERVICE_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_options_UPDATE_OPTIONS "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output # Check an observable difference before and after applying UPDATE_OPTIONS. @@ -100,17 +102,19 @@ Describe 'Options' End Describe "test_options_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false" TEST_NAME="test_options_UPDATE_TIMEOUT_SECONDS_not_a_number" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_options_UPDATE_TIMEOUT_SECONDS_not_a_number() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_options_UPDATE_TIMEOUT_SECONDS_not_a_number "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_options_UPDATE_TIMEOUT_SECONDS_not_a_number "${TEST_NAME}" "${SERVICE_NAME}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output @@ -139,18 +143,20 @@ Describe 'Options' End Describe "test_options_PRE_POST_RUN_CMD" "container_test:true" TEST_NAME="test_options_PRE_POST_RUN_CMD" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_options_PRE_POST_RUN_CMD() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" export GANTRY_PRE_RUN_CMD="echo \"Pre update\"" export GANTRY_POST_RUN_CMD="echo \"Post update\"" run_gantry "${TEST_NAME}" } - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_options_PRE_POST_RUN_CMD "${TEST_NAME}" + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_options_PRE_POST_RUN_CMD "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh index d907e82..174a405 100644 --- a/tests/gantry_rollback_spec.sh +++ b/tests/gantry_rollback_spec.sh @@ -21,12 +21,22 @@ Describe 'Rollback' AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_rollback_due_to_timeout" "container_test:false" TEST_NAME="test_rollback_due_to_timeout" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - Before "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call run_gantry "${TEST_NAME}" + TIMEOUT=3 + test_rollback_due_to_timeout() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + local TIMEOUT=${3} + reset_gantry_env "${SERVICE_NAME}" + # Assume service update won't be done within TIMEOUT second. + export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}" + run_gantry "${TEST_NAME}" + } + BeforeEach "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TIMEOUT}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_rollback_due_to_timeout "${TEST_NAME}" "${SERVICE_NAME}" "${TIMEOUT}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output @@ -54,18 +64,24 @@ Describe 'Rollback' End Describe "test_rollback_failed" "container_test:false" TEST_NAME="test_rollback_failed" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + TIMEOUT=3 test_rollback_failed() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + local TIMEOUT=${3} + reset_gantry_env "${SERVICE_NAME}" + # Assume service update won't be done within TIMEOUT second. + export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}" # Rollback would fail due to the incorrect option. export GANTRY_ROLLBACK_OPTIONS="--incorrect-option" run_gantry "${TEST_NAME}" } - Before "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_rollback_failed "${TEST_NAME}" + BeforeEach "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TIMEOUT}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_rollback_failed "${TEST_NAME}" "${SERVICE_NAME}" "${TIMEOUT}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output @@ -94,17 +110,23 @@ Describe 'Rollback' End Describe "test_rollback_ROLLBACK_ON_FAILURE_false" "container_test:false" TEST_NAME="test_rollback_ROLLBACK_ON_FAILURE_false" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + TIMEOUT=3 test_rollback_ROLLBACK_ON_FAILURE_false() { local TEST_NAME=${1} + local SERVICE_NAME=${2} + local TIMEOUT=${3} + reset_gantry_env "${SERVICE_NAME}" + # Assume service update won't be done within TIMEOUT second. + export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}" export GANTRY_ROLLBACK_ON_FAILURE="false" run_gantry "${TEST_NAME}" } - Before "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call test_rollback_ROLLBACK_ON_FAILURE_false "${TEST_NAME}" + BeforeEach "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TIMEOUT}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_rollback_ROLLBACK_ON_FAILURE_false "${TEST_NAME}" "${SERVICE_NAME}" "${TIMEOUT}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_simple_spec.sh b/tests/gantry_simple_spec.sh index a555f64..4e6c1f0 100644 --- a/tests/gantry_simple_spec.sh +++ b/tests/gantry_simple_spec.sh @@ -21,12 +21,18 @@ Describe 'Simple' AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_new_image_no" "container_test:true" TEST_NAME="test_new_image_no" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - Before "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call run_gantry "${TEST_NAME}" + test_new_image_no() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + run_gantry "${TEST_NAME}" + } + BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_new_image_no "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -54,12 +60,18 @@ Describe 'Simple' End Describe "test_new_image_yes" "container_test:true" TEST_NAME="test_new_image_yes" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - Before "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call run_gantry "${TEST_NAME}" + test_new_image_yes() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + run_gantry "${TEST_NAME}" + } + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_new_image_yes "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -87,24 +99,28 @@ Describe 'Simple' End Describe "test_new_image_no_digest" "container_test:true" TEST_NAME="test_new_image_no_digest" - IMAGE_WITH_TAG=$(get_image_with_tag) + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_start() { local TEST_NAME=${1} local IMAGE_WITH_TAG=${2} local SERVICE_NAME=${3} - initialize_test "${TEST_NAME}" # Start a service with image not available on the registry, the digest will not be available. build_test_image "${IMAGE_WITH_TAG}" start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" 2>&1 # Push a new image to registry, thus manifest command will success. build_and_push_test_image "${IMAGE_WITH_TAG}" - export GANTRY_SERVICES_FILTERS="name=${SERVICE_NAME}" } - Before "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - After "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_gantry' - When call run_gantry "${TEST_NAME}" + test_new_image_no_digest() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + run_gantry "${TEST_NAME}" + } + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_new_image_no_digest "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index bc10baa..7a3e11a 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -63,7 +63,6 @@ common_setup_new_image() { build_and_push_test_image "${IMAGE_WITH_TAG}" start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" build_and_push_test_image "${IMAGE_WITH_TAG}" - export GANTRY_SERVICES_FILTERS="name=${SERVICE_NAME}" } common_setup_no_new_image() { @@ -74,7 +73,6 @@ common_setup_no_new_image() { build_and_push_test_image "${IMAGE_WITH_TAG}" start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" # No image updates after service started. - export GANTRY_SERVICES_FILTERS="name=${SERVICE_NAME}" } common_setup_job() { @@ -87,14 +85,13 @@ common_setup_job() { build_and_push_test_image "${IMAGE_WITH_TAG}" "${TASK_SECONDS}" _start_replicated_job "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" build_and_push_test_image "${IMAGE_WITH_TAG}" - export GANTRY_SERVICES_FILTERS="name=${SERVICE_NAME}" } common_setup_timeout() { local TEST_NAME="${1}" local IMAGE_WITH_TAG="${2}" local SERVICE_NAME="${3}" - local TIMEOUT=3 + local TIMEOUT="${4}" local DOUBLE_TIMEOUT=$((TIMEOUT+TIMEOUT)) initialize_test "${TEST_NAME}" # -1 thus the task runs forever. @@ -102,9 +99,6 @@ common_setup_timeout() { build_and_push_test_image "${IMAGE_WITH_TAG}" "-1" "${DOUBLE_TIMEOUT}" start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" build_and_push_test_image "${IMAGE_WITH_TAG}" - export GANTRY_SERVICES_FILTERS="name=${SERVICE_NAME}" - # Assume service update won't be done within TIMEOUT second. - export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}" } common_cleanup() { @@ -132,7 +126,7 @@ _init_swarm() { return 0 fi echo "Run docker swarm init" - docker swarm init + docker swarm init 2>&1 } # Image for the software under test (SUT) @@ -141,20 +135,53 @@ _get_sut_image() { echo "${SUT_REPO_TAG}" } +_next_available_port() { + local START="${1}" + local LIMIT="${2:-1}" + local END=$((START+LIMIT)) + [ "${END}" -le "${START}" ] && END=$((START+1)) + local PORT="${START}" + while nc -z localhost "${PORT}"; do + PORT=$((PORT+1)) + if [ "${PORT}" -ge "${END}" ]; then + PORT= + return 1 + fi + done + echo "${PORT}" +} + +_get_test_registry_file() { + local SUITE_NAME="${1:?}" + SUITE_NAME=$(echo "${SUITE_NAME}" | tr ' ' '-') + echo "/tmp/TEST_REGISTRY-${SUITE_NAME}" +} + +load_test_registry() { + local SUITE_NAME="${1:?}" + local REGISTRY_FILE= + REGISTRY_FILE=$(_get_test_registry_file "${SUITE_NAME}") + [ ! -r "${REGISTRY_FILE}" ] && return 1 + cat "${REGISTRY_FILE}" +} + _start_registry() { local SUITE_NAME="${1:?}" SUITE_NAME=$(echo "${SUITE_NAME}" | tr ' ' '-') - export REGISTRY_SERVICE_NAME="gantry-test-registry-${SUITE_NAME}" - export TEST_IMAGE="gantry/test" - local TEST_REGISTRY_BASE="127.0.0.1" - local TEST_REGISTRY_PORT="5000" - export TEST_REGISTRY="${TEST_REGISTRY_BASE}:${TEST_REGISTRY_PORT}" + local REGISTRY_SERVICE_NAME="gantry-test-registry-${SUITE_NAME}" + local REGISTRY_BASE="127.0.0.1" + local REGISTRY_PORT="5000" + local TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}" export TEST_USERNAME="gantry" export TEST_PASSWORD="gantry" local REGISTRY_IMAGE="docker.io/registry" local TRIES=0 local MAX_RETRIES=50 - echo -n "Starting registry ${TEST_REGISTRY} " + local PORT_LIMIT=500 + REGISTRY_PORT=$(_next_available_port "${REGISTRY_PORT}" "${PORT_LIMIT}") || return 1 + [ -z "${REGISTRY_PORT}" ] && return 1 + TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}" + echo -n "${SUITE_NAME} starting registry ${TEST_REGISTRY} " # SC2046 (warning): Quote this to prevent word splitting. # shellcheck disable=SC2046 while ! docker service create --quiet \ @@ -163,24 +190,33 @@ _start_registry() { --restart-max-attempts 5 \ $(_location_constraints) \ --mode=replicated \ - -p "${TEST_REGISTRY_PORT}:5000" \ - "${REGISTRY_IMAGE}" 2>/dev/null; do - TEST_REGISTRY_PORT=$((TEST_REGISTRY_PORT+1)) - export TEST_REGISTRY="${TEST_REGISTRY_BASE}:${TEST_REGISTRY_PORT}" - echo -n "Starting registry ${TEST_REGISTRY} again " - if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then - return 1 - fi + -p "${REGISTRY_PORT}:5000" \ + "${REGISTRY_IMAGE}" 2>&1; do + stop_service "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1 + [ "${TRIES}" -ge "${MAX_RETRIES}" ] && echo "_start_registry Reach MAX_RETRIES ${MAX_RETRIES}" && return 1 TRIES=$((TRIES+1)) sleep 1 + REGISTRY_PORT=$(_next_available_port "${REGISTRY_PORT}" "${PORT_LIMIT}") || return 1 + [ -z "${REGISTRY_PORT}" ] && return 1 + TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}" + echo -n "${SUITE_NAME} starting registry ${TEST_REGISTRY} again " done + local REGISTRY_FILE= + REGISTRY_FILE=$(_get_test_registry_file "${SUITE_NAME}") || return 1 + echo "${TEST_REGISTRY}" > "${REGISTRY_FILE}" } _stop_registry() { local SUITE_NAME="${1:?}" SUITE_NAME=$(echo "${SUITE_NAME}" | tr ' ' '-') - echo -n "Removing registry ${TEST_REGISTRY:?} " - stop_service "${REGISTRY_SERVICE_NAME:?}" + local REGISTRY_SERVICE_NAME="gantry-test-registry-${SUITE_NAME}" + local REGISTRY= + REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 + echo -n "Removing registry ${REGISTRY} " + stop_service "${REGISTRY_SERVICE_NAME}" + local REGISTRY_FILE= + REGISTRY_FILE=$(_get_test_registry_file "${SUITE_NAME}") || return 1 + rm "${REGISTRY_FILE}" return 0 } @@ -191,7 +227,7 @@ initialize_all_tests() { SCRIPT_DIR="$(_get_script_dir)" || return 1 source "${SCRIPT_DIR}/../src/lib-common.sh" echo "==============================" - echo "== Starting tests in ${SUITE_NAME}" + echo "== Starting suite ${SUITE_NAME}" echo "==============================" _init_swarm _start_registry "${SUITE_NAME}" @@ -212,6 +248,10 @@ initialize_test() { echo "==============================" echo "== Starting ${TEST_NAME}" echo "==============================" +} + +reset_gantry_env() { + local SERVICE_NAME="${1}" export DOCKER_HOST= export GANTRY_LOG_LEVEL="DEBUG" export GANTRY_NODE_NAME= @@ -230,7 +270,11 @@ initialize_test() { export GANTRY_REGISTRY_USER_FILE= export GANTRY_SERVICES_EXCLUDED= export GANTRY_SERVICES_EXCLUDED_FILTERS= - export GANTRY_SERVICES_FILTERS= + if [ -z "${SERVICE_NAME}" ]; then + export GANTRY_SERVICES_FILTERS= + else + export GANTRY_SERVICES_FILTERS="name=${SERVICE_NAME}" + fi export GANTRY_SERVICES_SELF= export GANTRY_MANIFEST_CMD= export GANTRY_MANIFEST_OPTIONS= @@ -260,8 +304,11 @@ finalize_test() { } get_image_with_tag() { - local IMAGE="${TEST_IMAGE:?}" - local REGISTRY="${TEST_REGISTRY:?}" + local SUITE_NAME="${1:?}" + SUITE_NAME=$(echo "${SUITE_NAME}" | tr ' ' '-') + local IMAGE="gantry/test" + local REGISTRY= + REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 if [ -z "${IMAGE}" ]; then echo "IMAGE is empty." >&2 return 1 @@ -271,7 +318,7 @@ get_image_with_tag() { if [ -n "${REGISTRY}" ]; then IMAGE_WITH_TAG="${REGISTRY}/${IMAGE_WITH_TAG}" fi - IMAGE_WITH_TAG="${IMAGE_WITH_TAG}:for-test-$(unique_id)" + IMAGE_WITH_TAG="${IMAGE_WITH_TAG}:${SUITE_NAME}-$(unique_id)" echo "${IMAGE_WITH_TAG}" } @@ -334,7 +381,7 @@ build_test_image() { echo "FROM alpinelinux/docker-cli:latest" > "${FILE}" echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}" echo -n "Building ${IMAGE_WITH_TAG} " - timeout 300 docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" . + timeout 120 docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" . rm "${FILE}" } @@ -359,6 +406,8 @@ wait_zero_running_tasks() { local NUM_RUNS=1 local REPLICAS= local USED_SECONDS=0 + local TRIES=0 + local MAX_RETRIES=120 echo "Wait until ${SERVICE_NAME} has zero running tasks." while [ "${NUM_RUNS}" -ne 0 ]; do if [ -n "${TIMEOUT_SECONDS}" ] && [ "${USED_SECONDS}" -ge "${TIMEOUT_SECONDS}" ]; then @@ -369,6 +418,8 @@ wait_zero_running_tasks() { _handle_failure "Failed to obtain task states of service ${SERVICE_NAME}: ${REPLICAS}" return 1 fi + [ "${TRIES}" -ge "${MAX_RETRIES}" ] && echo "wait_zero_running_tasks Reach MAX_RETRIES ${MAX_RETRIES}" && return 1 + TRIES=$((TRIES+1)) # https://docs.docker.com/engine/reference/commandline/service_ls/#examples # The REPLICAS is like "5/5" or "1/1 (3/5 completed)" # Get the number before the first "/". @@ -400,7 +451,11 @@ _location_constraints() { _wait_service_state() { local SERVICE_NAME="${1}" local STATE="${2}" + local TRIES=0 + local MAX_RETRIES=120 while ! docker service ps --format "{{.CurrentState}}" "${SERVICE_NAME}" | grep -q "${STATE}"; do + [ "${TRIES}" -ge "${MAX_RETRIES}" ] && echo "_wait_service_state Reach MAX_RETRIES ${MAX_RETRIES}" && return 1 + TRIES=$((TRIES+1)) sleep 1 done } @@ -411,7 +466,7 @@ start_replicated_service() { echo -n "Creating service ${SERVICE_NAME} in replicated mode " # SC2046 (warning): Quote this to prevent word splitting. # shellcheck disable=SC2046 - timeout 300 docker service create --quiet \ + timeout 120 docker service create --quiet \ --name "${SERVICE_NAME}" \ --restart-condition "on-failure" \ --restart-max-attempts 5 \ @@ -426,7 +481,7 @@ start_global_service() { echo -n "Creating service ${SERVICE_NAME} in global mode " # SC2046 (warning): Quote this to prevent word splitting. # shellcheck disable=SC2046 - timeout 300 docker service create --quiet \ + timeout 120 docker service create --quiet \ --name "${SERVICE_NAME}" \ --restart-condition "on-failure" \ --restart-max-attempts 5 \ @@ -441,7 +496,7 @@ _start_replicated_job() { echo -n "Creating service ${SERVICE_NAME} in replicated job mode " # SC2046 (warning): Quote this to prevent word splitting. # shellcheck disable=SC2046 - timeout 300 docker service create --quiet \ + timeout 120 docker service create --quiet \ --name "${SERVICE_NAME}" \ --restart-condition "on-failure" \ --restart-max-attempts 5 \ From 9bf488b5ce34ea58d2c5fcca9a600d4d0916ebb9 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Wed, 14 Feb 2024 18:42:19 -0800 Subject: [PATCH 04/19] [tests] add test_login_file_not_exist --- src/entrypoint.sh | 4 +- src/lib-gantry.sh | 19 ++++++--- tests/gantry_login_spec.sh | 73 +++++++++++++++++++++++++++++--- tests/spec_gantry_test_helper.sh | 2 +- 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/src/entrypoint.sh b/src/entrypoint.sh index cfa5c7b..89c209f 100755 --- a/src/entrypoint.sh +++ b/src/entrypoint.sh @@ -60,7 +60,7 @@ _read_docker_hub_rate() { USER= fi if ! HOST=$(gantry_read_registry_host 2>&1); then - log ERROR "Failed to read HOST: ${HOST}"; + log ERROR "Failed to read registry HOST: ${HOST}"; HOST= fi local USER_AND_PASS= @@ -133,7 +133,7 @@ gantry() { TIME_ELAPSED=$(time_elapsed_since "${START_TIME}") local MESSAGE="Done. Use ${TIME_ELAPSED}. ${ACCUMULATED_ERRORS} errors." local RETURN_VALUE=0 - if [ ${ACCUMULATED_ERRORS} -gt 0 ]; then + if [ "${ACCUMULATED_ERRORS}" -gt 0 ]; then log ERROR "${MESSAGE}" RETURN_VALUE=1 else diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index 55f6a3d..218d60f 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -63,20 +63,24 @@ _authenticate_to_registries() { local ACCUMULATED_ERRORS=0 local CONFIG HOST PASSWORD USER if ! CONFIG=$(read_config GANTRY_REGISTRY_CONFIG 2>&1); then - log ERROR "Failed to read CONFIG: ${CONFIG}" + log ERROR "Failed to read registry CONFIG: ${CONFIG}" ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1)) + CONFIG= fi if ! HOST=$(gantry_read_registry_host 2>&1); then - log ERROR "Failed to read HOST: ${HOST}" + log ERROR "Failed to read registry HOST: ${HOST}" ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1)) + HOST= fi if ! PASSWORD=$(gantry_read_registry_password 2>&1); then - log ERROR "Failed to read PASSWORD: ${PASSWORD}" + log ERROR "Failed to read registry PASSWORD: ${PASSWORD}" ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1)) + PASSWORD= fi if ! USER=$(gantry_read_registry_username 2>&1); then - log ERROR "Failed to read USER: ${USER}" + log ERROR "Failed to read registry USER: ${USER}" ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1)) + USER= fi if [ "${ACCUMULATED_ERRORS}" -gt 0 ]; then log ERROR "Skip logging in due to previous errors." @@ -88,7 +92,10 @@ _authenticate_to_registries() { [ "${ACCUMULATED_ERRORS}" -gt 0 ] && return 1 return 0 fi - [ ! -r "${CONFIGS_FILE}" ] && log ERROR "Failed to read ${CONFIGS_FILE}." && return 1 + if [ ! -r "${CONFIGS_FILE}" ]; then + log ERROR "Failed to read CONFIGS_FILE ${CONFIGS_FILE}." + return 1 + fi local LINE= while read -r LINE; do # skip comments @@ -117,7 +124,7 @@ _authenticate_to_registries() { _login_registry "${USER}" "${PASSWORD}" "${HOST}" "${CONFIG}" ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?)) done < <(cat "${CONFIGS_FILE}"; echo;) - [ ${ACCUMULATED_ERRORS} -gt 0 ] && return 1 + [ "${ACCUMULATED_ERRORS}" -gt 0 ] && return 1 return 0 } diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh index 1fe2f8f..fa48afe 100644 --- a/tests/gantry_login_spec.sh +++ b/tests/gantry_login_spec.sh @@ -31,8 +31,8 @@ Describe 'Login' local REGISTRY=${3} local USERNAME=${4} local PASSWORD=${5} - if [ -z "${USERNAME}" ] || [ -z "${PASSWORD}" ]; then - echo "No USERNAME or PASSWORD provided." >&2 + if [ -z "${REGISTRY}" ] || [ -z "${USERNAME}" ] || [ -z "${PASSWORD}" ]; then + echo "No REGISTRY, USERNAME or PASSWORD provided." >&2 return 1 fi local LABEL="gantry.auth.config" @@ -155,11 +155,11 @@ Describe 'Login' End End Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:false" - TEST_NAME="test_login_REGISTRY_CONFIGS_FILE_bad_format_extra" + TEST_NAME="test_login_REGISTRY_CONFIGS_FILE_bad_format" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 - test_login_REGISTRY_CONFIGS_FILE_bad_format_extra() { + test_login_REGISTRY_CONFIGS_FILE_bad_format() { local TEST_NAME=${1} local SERVICE_NAME=${2} local REGISTRY=${3} @@ -189,7 +189,7 @@ Describe 'Login' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_REGISTRY_CONFIGS_FILE_bad_format_extra "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_REGISTRY_CONFIGS_FILE_bad_format "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output @@ -220,4 +220,67 @@ Describe 'Login' The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End End + Describe "test_login_file_not_exist" "container_test:false" + TEST_NAME="test_login_file_not_exist" + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") + SERVICE_NAME="gantry-test-$(unique_id)" + TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 + test_login_file_not_exist() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + local REGISTRY=${3} + local USERNAME=${4} + local PASSWORD=${5} + if [ -z "${REGISTRY}" ] || [ -z "${USERNAME}" ] || [ -z "${PASSWORD}" ]; then + echo "No REGISTRY, USERNAME or PASSWORD provided." >&2 + return 1 + fi + local LABEL="gantry.auth.config" + CONFIG="C$(unique_id)" + docker service update --quiet --label-add "${LABEL}=${CONFIG}" "${SERVICE_NAME}" + local FILE_NOT_EXIST="/tmp/${CONFIG}" + reset_gantry_env "${SERVICE_NAME}" + export GANTRY_REGISTRY_CONFIG_FILE="${FILE_NOT_EXIST}" + export GANTRY_REGISTRY_CONFIGS_FILE="${FILE_NOT_EXIST}" + export GANTRY_REGISTRY_HOST_FILE="${FILE_NOT_EXIST}" + export GANTRY_REGISTRY_PASSWORD_FILE="${FILE_NOT_EXIST}" + export GANTRY_REGISTRY_USER_FILE="${FILE_NOT_EXIST}" + local RETURN_VALUE= + run_gantry "${TEST_NAME}" + RETURN_VALUE="${?}" + [ -d "${CONFIG}" ] && rm -r "${CONFIG}" + return "${RETURN_VALUE}" + } + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_login_file_not_exist "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + The status should be failure + The stdout should satisfy display_output + The stderr should satisfy display_output + The stderr should satisfy spec_expect_no_message "Logged into registry *${TEST_REGISTRY} for config ${CONFIG}" + The stderr should satisfy spec_expect_message "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}" + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*" + The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${NO_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" + The stderr should satisfy spec_expect_message "${NO_IMAGES_TO_REMOVE}" + The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" + The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" + The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" + End + End End # Describe 'Login' diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index 7a3e11a..449d896 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -543,7 +543,7 @@ _get_entrypoint() { _add_file_to_mount_options() { local MOUNT_OPTIONS="${1}" local FILE="${2}" - if [ -n "${FILE}" ]; then + if [ -n "${FILE}" ] && [ -r "${FILE}" ]; then MOUNT_OPTIONS="${MOUNT_OPTIONS} --mount type=bind,source=${FILE},target=${FILE}" fi echo "${MOUNT_OPTIONS}" From 80b46bfc6c006120cc6dcf8ef6834e240b18d5a7 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Wed, 14 Feb 2024 21:21:31 -0800 Subject: [PATCH 05/19] [workflows] add post build tests. --- .github/workflows/on-push.yml | 63 +++++++++++++++++++++++++++++++- tests/spec_gantry_test_helper.sh | 17 ++++++--- 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index c3ff3a2..94c4fd4 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -30,7 +30,7 @@ jobs: set +e tests: - name: Run tests + name: Test script runs-on: ubuntu-latest strategy: fail-fast: false @@ -111,4 +111,65 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} provenance: false + - name: Extract tag + run: | + TAGS="${{ steps.meta.outputs.tags }}" + for TAG in ${TAGS}; do + if echo "${TAG}" | grep -q "ghcr.io/${{ github.repository }}-development:dev-"; then + echo "TAG=${TAG}" + echo "${TAG}" > tag.txt + break; + fi + done + - name: Store tag + uses: actions/upload-artifact@v4 + with: + name: tag + path: tag.txt + + container_tests: + name: Test container + runs-on: ubuntu-latest + needs: + - build_and_push + strategy: + fail-fast: false + matrix: + test_suit: + - gantry_cleanup_images_spec.sh + - gantry_entrypoint_spec.sh + - gantry_filters_spec.sh + - gantry_job_spec.sh + - gantry_login_spec.sh + - gantry_manifest_spec.sh + - gantry_multiple_services_spec.sh + - gantry_no_running_tasks_spec.sh + - gantry_notify_spec.sh + - gantry_options_spec.sh + - gantry_rollback_spec.sh + - gantry_simple_spec.sh + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + - name: Install shellspec + run: | + mkdir -p ~/shellspec + cd ~/shellspec + git clone https://github.com/shellspec/shellspec.git + ln -s ~/shellspec/shellspec/shellspec /usr/local/bin/shellspec + echo -n "shellspec version: " + shellspec --version + - name: Checkout Code + uses: actions/checkout@v4 + - name: Load tag + uses: actions/download-artifact@v4 + with: + name: tag + - name: Run tests + run: | + export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }} + export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} + export GANTRY_TEST_CONTAINER_REPO_TAG=$(cat tag.txt) + echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}" + bash shellspec --pattern tests/${{ matrix.test_suit }} --tag "container_test:true" diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index 449d896..ce0228b 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -565,11 +565,14 @@ _run_gantry_container() { MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_HOST_FILE}") MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_PASSWORD_FILE}") MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_USER_FILE}") - local RETURN_VALUE= - echo -n "Starting SUT service ${SERVICE_NAME} " + if [ "${GANTRY_LOG_LEVEL}" != "NONE" ]; then + echo "Starting SUT service ${SERVICE_NAME} with image ${SUT_REPO_TAG}." + fi + local RETURN_VALUE=0 + local CMD_OUTPUT= # SC2086 (info): Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - docker service create --quiet --name "${SERVICE_NAME}" \ + if ! CMD_OUTPUT=$(docker service create --name "${SERVICE_NAME}" \ --mode replicated-job --restart-condition=none --network host \ --constraint "node.role==manager" \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ @@ -607,12 +610,14 @@ _run_gantry_container() { --env "GANTRY_NOTIFICATION_TITLE=${GANTRY_NOTIFICATION_TITLE}" \ --env "TZ=${TZ}" \ "${SUT_REPO_TAG}" \ - "${STACK}"; - RETURN_VALUE="${?}" + "${STACK}" 2>&1); then + echo "Failed to create service ${SERVICE_NAME}: ${CMD_OUTPUT}" >&2 + RETURN_VALUE=1 + fi docker service logs --raw "${SERVICE_NAME}" - local CMD_OUTPUT= if ! CMD_OUTPUT=$(docker service rm "${SERVICE_NAME}" 2>&1); then echo "Failed to remove service ${SERVICE_NAME}: ${CMD_OUTPUT}" >&2 + RETURN_VALUE=1 fi return "${RETURN_VALUE}" } From 87ae432dc6a4f624e0f56b6471f45bb3aa4460c5 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Fri, 16 Feb 2024 02:15:40 -0800 Subject: [PATCH 06/19] [gantry] fix the problem about incorrectly obtaining the current container name when running outside the container. This blocks run tests in parallel. --- .github/workflows/on-push.yml | 18 +------ src/lib-gantry.sh | 25 ++++++---- tests/README.md | 7 ++- tests/spec_gantry_test_helper.sh | 81 ++++++++++++++++++++------------ 4 files changed, 73 insertions(+), 58 deletions(-) diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 94c4fd4..00da7e5 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -133,22 +133,6 @@ jobs: runs-on: ubuntu-latest needs: - build_and_push - strategy: - fail-fast: false - matrix: - test_suit: - - gantry_cleanup_images_spec.sh - - gantry_entrypoint_spec.sh - - gantry_filters_spec.sh - - gantry_job_spec.sh - - gantry_login_spec.sh - - gantry_manifest_spec.sh - - gantry_multiple_services_spec.sh - - gantry_no_running_tasks_spec.sh - - gantry_notify_spec.sh - - gantry_options_spec.sh - - gantry_rollback_spec.sh - - gantry_simple_spec.sh steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.0.0 @@ -172,4 +156,4 @@ jobs: export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} export GANTRY_TEST_CONTAINER_REPO_TAG=$(cat tag.txt) echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}" - bash shellspec --pattern tests/${{ matrix.test_suit }} --tag "container_test:true" + bash shellspec --jobs 50 --tag "container_test:true" diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index 218d60f..7ff0716 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -239,8 +239,8 @@ _remove_images() { --mount type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock \ --env "GANTRY_IMAGES_TO_REMOVE=${IMAGES_TO_REMOVE_LIST}" \ ${CLEANUP_IMAGES_OPTIONS} \ - "${IMAGES_REMOVER}" 2>&1); then - log ERROR "Failed to remove images: ${RMI_MSG}" + "${IMAGES_REMOVER}"); then + log ERROR "Failed to remove images." fi wait_service_state "${SERVICE_NAME}" docker_service_logs "${SERVICE_NAME}" @@ -347,19 +347,24 @@ _current_container_name() { GWBRIDGE_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=docker_gwbridge') || return 1; IPS=$(ip route | grep src | sed -n "s/.* src \(\S*\).*$/\1/p"); [ -z "${IPS}" ] && return 0; - local NID; + local NID=; for NID in ${ALL_NETWORKS}; do [ "${NID}" = "${GWBRIDGE_NETWORK}" ] && continue; - local ALL_LOCAL_NAME_AND_IP; - ALL_LOCAL_NAME_AND_IP=$(docker network inspect "${NID}" --format "{{range .Containers}}{{.Name}}={{println .IPv4Address}}{{end}}") || return 1; + local ALL_LOCAL_NAME_AND_IP=; + ALL_LOCAL_NAME_AND_IP=$(docker network inspect "${NID}" --format "{{range .Containers}}{{.Name}}/{{println .IPv4Address}}{{end}}") || return 1; for NAME_AND_IP in ${ALL_LOCAL_NAME_AND_IP}; do [ -z "${NAME_AND_IP}" ] && continue; + # '//' + # '/' (when network mode is host) + local CNAME CIP + CNAME=$(echo "${NAME_AND_IP}" | cut -d/ -f1); + CIP=$(echo "${NAME_AND_IP}" | cut -d/ -f2); + # Unable to find the container IP when network mode is host. + [ -z "${CIP}" ] && continue; for IP in ${IPS}; do - echo "${NAME_AND_IP}" | grep -q "${IP}" || continue; - local NAME; - NAME=$(echo "${NAME_AND_IP}" | sed "s/\(.*\)=${IP}.*$/\1/"); - _static_variable_add_unique_to_list STATIC_VAR_CURRENT_CONTAINER_NAME "${NAME}" - echo "${NAME}"; + [ "${IP}" != "${CIP}" ] && continue; + _static_variable_add_unique_to_list STATIC_VAR_CURRENT_CONTAINER_NAME "${CNAME}" + echo "${CNAME}"; return 0; done done diff --git a/tests/README.md b/tests/README.md index fa6dcf5..434a5f0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -22,7 +22,12 @@ bash shellspec --pattern tests/ bash shellspec --pattern tests/ --example ``` -To generate coverage (need [kcov](https://github.com/SimonKagstrom/kcov) installed): +To run multiple tests in parallel +``` +bash shellspec --jobs 50 +``` + +To generate coverage (require [kcov](https://github.com/SimonKagstrom/kcov) installed): ``` bash shellspec --kcov ``` diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index ce0228b..e23a648 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -168,9 +168,10 @@ load_test_registry() { _start_registry() { local SUITE_NAME="${1:?}" SUITE_NAME=$(echo "${SUITE_NAME}" | tr ' ' '-') + local SUITE_NAME_LENGTH="${#SUITE_NAME}" local REGISTRY_SERVICE_NAME="gantry-test-registry-${SUITE_NAME}" local REGISTRY_BASE="127.0.0.1" - local REGISTRY_PORT="5000" + local REGISTRY_PORT=$((55000+SUITE_NAME_LENGTH*2)) local TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}" export TEST_USERNAME="gantry" export TEST_PASSWORD="gantry" @@ -178,31 +179,44 @@ _start_registry() { local TRIES=0 local MAX_RETRIES=50 local PORT_LIMIT=500 - REGISTRY_PORT=$(_next_available_port "${REGISTRY_PORT}" "${PORT_LIMIT}") || return 1 - [ -z "${REGISTRY_PORT}" ] && return 1 - TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}" - echo -n "${SUITE_NAME} starting registry ${TEST_REGISTRY} " - # SC2046 (warning): Quote this to prevent word splitting. - # shellcheck disable=SC2046 - while ! docker service create --quiet \ - --name "${REGISTRY_SERVICE_NAME}" \ - --restart-condition "on-failure" \ - --restart-max-attempts 5 \ - $(_location_constraints) \ - --mode=replicated \ - -p "${REGISTRY_PORT}:5000" \ - "${REGISTRY_IMAGE}" 2>&1; do + while true; do + if ! REGISTRY_PORT=$(_next_available_port "${REGISTRY_PORT}" "${PORT_LIMIT}" 2>&1); then + echo "_start_registry _next_available_port error: ${REGISTRY_PORT}" >&2 + return 1 + fi + if [ -z "${REGISTRY_PORT}" ]; then + echo "_start_registry _next_available_port error: REGISTRY_PORT is empty." >&2 + return 1 + fi stop_service "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1 - [ "${TRIES}" -ge "${MAX_RETRIES}" ] && echo "_start_registry Reach MAX_RETRIES ${MAX_RETRIES}" && return 1 + TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}" + echo "${SUITE_NAME} starting registry ${TEST_REGISTRY} " + # SC2046 (warning): Quote this to prevent word splitting. + # shellcheck disable=SC2046 + if docker service create --quiet \ + --name "${REGISTRY_SERVICE_NAME}" \ + --restart-condition "on-failure" \ + --restart-max-attempts 5 \ + $(_location_constraints) \ + --mode=replicated \ + -p "${REGISTRY_PORT}:5000" \ + "${REGISTRY_IMAGE}" 2>&1; then + break; + fi + if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then + echo "_start_registry Reach MAX_RETRIES ${MAX_RETRIES}" >&2 + return 1 + fi TRIES=$((TRIES+1)) + REGISTRY_PORT=$((REGISTRY_PORT+1)) sleep 1 - REGISTRY_PORT=$(_next_available_port "${REGISTRY_PORT}" "${PORT_LIMIT}") || return 1 - [ -z "${REGISTRY_PORT}" ] && return 1 - TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}" - echo -n "${SUITE_NAME} starting registry ${TEST_REGISTRY} again " done local REGISTRY_FILE= - REGISTRY_FILE=$(_get_test_registry_file "${SUITE_NAME}") || return 1 + if ! REGISTRY_FILE=$(_get_test_registry_file "${SUITE_NAME}" 2>&1); then + echo "_start_registry _get_test_registry_file error: ${REGISTRY_FILE}" >&2 + return 1 + fi + echo "${SUITE_NAME} uses registry ${TEST_REGISTRY}." echo "${TEST_REGISTRY}" > "${REGISTRY_FILE}" } @@ -212,7 +226,7 @@ _stop_registry() { local REGISTRY_SERVICE_NAME="gantry-test-registry-${SUITE_NAME}" local REGISTRY= REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 - echo -n "Removing registry ${REGISTRY} " + echo "Removing registry ${REGISTRY} " stop_service "${REGISTRY_SERVICE_NAME}" local REGISTRY_FILE= REGISTRY_FILE=$(_get_test_registry_file "${SUITE_NAME}") || return 1 @@ -380,7 +394,7 @@ build_test_image() { FILE=$(mktemp) echo "FROM alpinelinux/docker-cli:latest" > "${FILE}" echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}" - echo -n "Building ${IMAGE_WITH_TAG} " + echo "Building ${IMAGE_WITH_TAG} " timeout 120 docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" . rm "${FILE}" } @@ -390,7 +404,7 @@ build_and_push_test_image() { local TASK_SECONDS="${2}" local EXIT_SECONDS="${3}" build_test_image "${IMAGE_WITH_TAG}" "${TASK_SECONDS}" "${EXIT_SECONDS}" - echo -n "Pushing image " + echo "Pushing image " docker push --quiet "${IMAGE_WITH_TAG}" } @@ -418,7 +432,10 @@ wait_zero_running_tasks() { _handle_failure "Failed to obtain task states of service ${SERVICE_NAME}: ${REPLICAS}" return 1 fi - [ "${TRIES}" -ge "${MAX_RETRIES}" ] && echo "wait_zero_running_tasks Reach MAX_RETRIES ${MAX_RETRIES}" && return 1 + if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then + echo "wait_zero_running_tasks Reach MAX_RETRIES ${MAX_RETRIES}" >&2 + return 1 + fi TRIES=$((TRIES+1)) # https://docs.docker.com/engine/reference/commandline/service_ls/#examples # The REPLICAS is like "5/5" or "1/1 (3/5 completed)" @@ -454,7 +471,10 @@ _wait_service_state() { local TRIES=0 local MAX_RETRIES=120 while ! docker service ps --format "{{.CurrentState}}" "${SERVICE_NAME}" | grep -q "${STATE}"; do - [ "${TRIES}" -ge "${MAX_RETRIES}" ] && echo "_wait_service_state Reach MAX_RETRIES ${MAX_RETRIES}" && return 1 + if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then + echo "_wait_service_state Reach MAX_RETRIES ${MAX_RETRIES}" >&2 + return 1 + fi TRIES=$((TRIES+1)) sleep 1 done @@ -463,7 +483,7 @@ _wait_service_state() { start_replicated_service() { local SERVICE_NAME="${1}" local IMAGE_WITH_TAG="${2}" - echo -n "Creating service ${SERVICE_NAME} in replicated mode " + echo "Creating service ${SERVICE_NAME} in replicated mode " # SC2046 (warning): Quote this to prevent word splitting. # shellcheck disable=SC2046 timeout 120 docker service create --quiet \ @@ -478,7 +498,7 @@ start_replicated_service() { start_global_service() { local SERVICE_NAME="${1}" local IMAGE_WITH_TAG="${2}" - echo -n "Creating service ${SERVICE_NAME} in global mode " + echo "Creating service ${SERVICE_NAME} in global mode " # SC2046 (warning): Quote this to prevent word splitting. # shellcheck disable=SC2046 timeout 120 docker service create --quiet \ @@ -493,7 +513,7 @@ start_global_service() { _start_replicated_job() { local SERVICE_NAME="${1}" local IMAGE_WITH_TAG="${2}" - echo -n "Creating service ${SERVICE_NAME} in replicated job mode " + echo "Creating service ${SERVICE_NAME} in replicated job mode " # SC2046 (warning): Quote this to prevent word splitting. # shellcheck disable=SC2046 timeout 120 docker service create --quiet \ @@ -509,7 +529,7 @@ _start_replicated_job() { stop_service() { local SERVICE_NAME="${1}" - echo -n "Removing service " + echo "Removing service " docker service rm "${SERVICE_NAME}" } @@ -605,6 +625,7 @@ _run_gantry_container() { --env "GANTRY_UPDATE_TIMEOUT_SECONDS=${GANTRY_UPDATE_TIMEOUT_SECONDS}" \ --env "GANTRY_CLEANUP_IMAGES=${GANTRY_CLEANUP_IMAGES}" \ --env "GANTRY_CLEANUP_IMAGES_OPTIONS=${GANTRY_CLEANUP_IMAGES_OPTIONS}" \ + --env "GANTRY_CLEANUP_IMAGES_REMOVER=${GANTRY_CLEANUP_IMAGES_REMOVER}" \ --env "GANTRY_IMAGES_TO_REMOVE=${GANTRY_IMAGES_TO_REMOVE}" \ --env "GANTRY_NOTIFICATION_APPRISE_URL=${GANTRY_NOTIFICATION_APPRISE_URL}" \ --env "GANTRY_NOTIFICATION_TITLE=${GANTRY_NOTIFICATION_TITLE}" \ From 3c5715edbdc9c99d6cd8cf388564f37b31aed656 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Fri, 16 Feb 2024 06:20:43 -0800 Subject: [PATCH 07/19] [lib] improve logging. fix test checks using env variables in subprocess. --- src/entrypoint.sh | 6 +++-- src/lib-common.sh | 37 ++++++++++++++++------------- src/lib-gantry.sh | 31 ++++++++++++++---------- tests/gantry_cleanup_images_spec.sh | 8 +++---- tests/gantry_job_spec.sh | 2 +- tests/gantry_manifest_spec.sh | 8 +++---- tests/gantry_options_spec.sh | 2 +- tests/gantry_rollback_spec.sh | 2 +- 8 files changed, 55 insertions(+), 41 deletions(-) diff --git a/src/entrypoint.sh b/src/entrypoint.sh index 89c209f..be52a14 100755 --- a/src/entrypoint.sh +++ b/src/entrypoint.sh @@ -77,7 +77,9 @@ _read_docker_hub_rate() { gantry() { local PRE_RUN_CMD="${GANTRY_PRE_RUN_CMD:-""}" local POST_RUN_CMD="${GANTRY_POST_RUN_CMD:-""}" - local STACK="${1:-gantry}" + local STACK="${1}" + [ -z "${STACK}" ] && STACK=$(gantry_current_service_name) + [ -z "${STACK}" ] && STACK="gantry" export LOG_SCOPE="${STACK}" local START_TIME= START_TIME=$(date +%s) @@ -131,7 +133,7 @@ gantry() { local TIME_ELAPSED= TIME_ELAPSED=$(time_elapsed_since "${START_TIME}") - local MESSAGE="Done. Use ${TIME_ELAPSED}. ${ACCUMULATED_ERRORS} errors." + local MESSAGE="Done. Use ${TIME_ELAPSED}. ${ACCUMULATED_ERRORS} error(s)." local RETURN_VALUE=0 if [ "${ACCUMULATED_ERRORS}" -gt 0 ]; then log ERROR "${MESSAGE}" diff --git a/src/lib-common.sh b/src/lib-common.sh index 0ef454b..92c84f0 100755 --- a/src/lib-common.sh +++ b/src/lib-common.sh @@ -376,10 +376,13 @@ docker_service_remove() { return 0 fi log INFO "Removing service ${SERVICE_NAME}." - docker service rm "${SERVICE_NAME}" >/dev/null - local RETURN_VALUE=$? + local LOG= + if ! LOG=$(docker service rm "${SERVICE_NAME}" 2>&1); then + log ERROR "Failed to remove docker service ${SERVICE_NAME}: ${LOG}" + return 1 + fi log INFO "Removed service ${SERVICE_NAME}." - return ${RETURN_VALUE} + return 0 } # We do not expect failures when using docker_global_job. @@ -389,10 +392,13 @@ docker_service_remove() { docker_global_job() { local SERVICE_NAME= SERVICE_NAME=$(_get_docker_command_name_arg "${@}") - log INFO "Starting service ${SERVICE_NAME}." - docker service create \ - --mode global-job \ - "${@}" >/dev/null + log INFO "Starting global-job ${SERVICE_NAME}." + local LOG= + if ! LOG=$(docker service create --mode global-job "${@}" 2>&1); then + log ERROR "Failed to create global-job ${SERVICE_NAME}: ${LOG}" + return 1 + fi + return 0 } # A job could fail when using docker_replicated_job. @@ -403,16 +409,17 @@ docker_replicated_job() { IS_DETACH=$(_get_docker_command_detach "${@}") # Add "--detach" to work around https://github.com/docker/cli/issues/2979 # The Docker CLI does not exit on failures. - log INFO "Starting service ${SERVICE_NAME}." - docker service create \ - --mode replicated-job --detach \ - "${@}" >/dev/null - local RETURN_VALUE=$? + log INFO "Starting replicated-job ${SERVICE_NAME}." + local LOG= + if ! LOG=$(docker service create --mode replicated-job --detach "${@}" 2>&1); then + log ERROR "Failed to create replicated-job ${SERVICE_NAME}: ${LOG}" + return 1 + fi # If the command line does not contain '--detach', the function returns til the replicated job is complete. if ! "${IS_DETACH}"; then wait_service_state "${SERVICE_NAME}" --complete || return $? fi - return ${RETURN_VALUE} + return 0 } _container_status() { @@ -438,9 +445,7 @@ docker_run() { local RETRIES=0 local MAX_RETRIES=5 local SLEEP_SECONDS=10 - while ! docker run \ - "${@}" >/dev/null; - do + while ! docker run "${@}" >/dev/null; do if [ ${RETRIES} -ge ${MAX_RETRIES} ]; then echo "Failed to run docker. Reached the max retries ${MAX_RETRIES}." >&2 return 1 diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index 7ff0716..7edc471 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -224,24 +224,21 @@ _remove_images() { log INFO "- ${I}" done local IMAGES_REMOVER= - IMAGES_REMOVER=$(_get_service_image "$(_current_service_name)") + IMAGES_REMOVER=$(_get_service_image "$(gantry_current_service_name)") [ -z "${IMAGES_REMOVER}" ] && IMAGES_REMOVER="${DEFAULT_IMAGES_REMOVER}" log DEBUG "Set IMAGES_REMOVER=${IMAGES_REMOVER}" local IMAGES_TO_REMOVE_LIST= IMAGES_TO_REMOVE_LIST=$(echo "${IMAGES_TO_REMOVE}" | tr '\n' ' ') [ -n "${CLEANUP_IMAGES_OPTIONS}" ] && log DEBUG "Adding options \"${CLEANUP_IMAGES_OPTIONS}\" to the global job ${SERVICE_NAME}." - local RMI_MSG= # SC2086: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - if ! RMI_MSG=$(docker_global_job --name "${SERVICE_NAME}" \ + docker_global_job --name "${SERVICE_NAME}" \ --restart-condition on-failure \ --restart-max-attempts 1 \ --mount type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock \ --env "GANTRY_IMAGES_TO_REMOVE=${IMAGES_TO_REMOVE_LIST}" \ ${CLEANUP_IMAGES_OPTIONS} \ - "${IMAGES_REMOVER}"); then - log ERROR "Failed to remove images." - fi + "${IMAGES_REMOVER}"; wait_service_state "${SERVICE_NAME}" docker_service_logs "${SERVICE_NAME}" docker_service_remove "${SERVICE_NAME}" @@ -341,19 +338,26 @@ _current_container_name() { local CURRENT_CONTAINER_NAME= CURRENT_CONTAINER_NAME=$(_static_variable_read_list STATIC_VAR_CURRENT_CONTAINER_NAME) [ -n "${CURRENT_CONTAINER_NAME}" ] && echo "${CURRENT_CONTAINER_NAME}" && return 0 - local ALL_NETWORKS GWBRIDGE_NETWORK IPS; + local ALL_NETWORKS= ALL_NETWORKS=$(docker network ls --format '{{.ID}}') || return 1; [ -z "${ALL_NETWORKS}" ] && return 0; - GWBRIDGE_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=docker_gwbridge') || return 1; + local IPS=; IPS=$(ip route | grep src | sed -n "s/.* src \(\S*\).*$/\1/p"); [ -z "${IPS}" ] && return 0; + local GWBRIDGE_NETWORK HOST_NETWORK; + GWBRIDGE_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=^docker_gwbridge$') || return 1; + HOST_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=^host$') || return 1; local NID=; for NID in ${ALL_NETWORKS}; do + # The output of gwbridge does not contain the container name. It looks like gateway_8f55496ce4f1/172.18.0.5/16. [ "${NID}" = "${GWBRIDGE_NETWORK}" ] && continue; + # The output of host does not contain an IP. + [ "${NID}" = "${HOST_NETWORK}" ] && continue; local ALL_LOCAL_NAME_AND_IP=; ALL_LOCAL_NAME_AND_IP=$(docker network inspect "${NID}" --format "{{range .Containers}}{{.Name}}/{{println .IPv4Address}}{{end}}") || return 1; for NAME_AND_IP in ${ALL_LOCAL_NAME_AND_IP}; do [ -z "${NAME_AND_IP}" ] && continue; + # NAME_AND_IP will be in one of the following formats: # '//' # '/' (when network mode is host) local CNAME CIP @@ -372,7 +376,7 @@ _current_container_name() { return 0; } -_current_service_name() { +gantry_current_service_name() { local CURRENT_SERVICE_NAME= CURRENT_SERVICE_NAME=$(_static_variable_read_list STATIC_VAR_CURRENT_SERVICE_NAME) [ -n "${CURRENT_SERVICE_NAME}" ] && echo "${CURRENT_SERVICE_NAME}" && return 0 @@ -389,10 +393,10 @@ _service_is_self() { if [ -z "${GANTRY_SERVICES_SELF}" ]; then # If _service_is_self is called inside a subprocess, export won't affect the parent process. # We only want to log it once, thus try to read the value from a static variable firstly. - # The static variable should be set inside the function _current_service_name. If it is set, skip logging. + # The static variable should be set inside the function gantry_current_service_name. If it is set, skip logging. GANTRY_SERVICES_SELF=$(_static_variable_read_list STATIC_VAR_CURRENT_SERVICE_NAME) if [ -z "${GANTRY_SERVICES_SELF}" ]; then - GANTRY_SERVICES_SELF=$(_current_service_name) + GANTRY_SERVICES_SELF=$(gantry_current_service_name) export GANTRY_SERVICES_SELF [ -n "${GANTRY_SERVICES_SELF}" ] && log INFO "Set GANTRY_SERVICES_SELF to ${GANTRY_SERVICES_SELF}." fi @@ -830,6 +834,9 @@ gantry_finalize() { if ! _report_services; then RETURN_VALUE=1 fi - [ -n "${STATIC_VARIABLES_FOLDER}" ] && log DEBUG "Removing STATIC_VARIABLES_FOLDER ${STATIC_VARIABLES_FOLDER}" && rm -r "${STATIC_VARIABLES_FOLDER}" + if [ -n "${STATIC_VARIABLES_FOLDER}" ]; then + log DEBUG "Removing STATIC_VARIABLES_FOLDER ${STATIC_VARIABLES_FOLDER}" + rm -r "${STATIC_VARIABLES_FOLDER}" + fi return "${RETURN_VALUE}" } diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh index 0b612b7..5f1d8a6 100644 --- a/tests/gantry_cleanup_images_spec.sh +++ b/tests/gantry_cleanup_images_spec.sh @@ -98,8 +98,8 @@ Describe 'Cleanup_images' The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*${GANTRY_CLEANUP_IMAGES_OPTIONS}" - The stderr should satisfy spec_expect_message "${FAILED_TO_REMOVE_IMAGE}.*${GANTRY_CLEANUP_IMAGES_OPTIONS}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--incorrect-option" + The stderr should satisfy spec_expect_message "Failed.*--incorrect-option" The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End @@ -141,8 +141,8 @@ Describe 'Cleanup_images' The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*${GANTRY_CLEANUP_IMAGES_OPTIONS}" - The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${GANTRY_CLEANUP_IMAGES_OPTIONS}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--container-label=test" + The stderr should satisfy spec_expect_no_message "Failed.*--container-label=test" The stderr should satisfy spec_expect_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End diff --git a/tests/gantry_job_spec.sh b/tests/gantry_job_spec.sh index bbd9bc6..6a96224 100644 --- a/tests/gantry_job_spec.sh +++ b/tests/gantry_job_spec.sh @@ -85,7 +85,7 @@ Describe 'Job' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*${GANTRY_UPDATE_OPTIONS}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh index 602e60c..8781d33 100644 --- a/tests/gantry_manifest_spec.sh +++ b/tests/gantry_manifest_spec.sh @@ -50,7 +50,7 @@ Describe 'Manifest_command' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*${GANTRY_UPDATE_OPTIONS}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--force" The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -130,7 +130,7 @@ Describe 'Manifest_command' The status should be success The stdout should satisfy display_output The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*${GANTRY_MANIFEST_OPTIONS}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--insecure" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" @@ -173,8 +173,8 @@ Describe 'Manifest_command' The stdout should satisfy display_output The stderr should satisfy display_output # No options are added to the unknwon command. - The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*${GANTRY_MANIFEST_OPTIONS}" - The stderr should satisfy spec_expect_message "Unknown MANIFEST_CMD.*${GANTRY_MANIFEST_CMD}" + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--insecure" + The stderr should satisfy spec_expect_message "Unknown MANIFEST_CMD.*unsupported_cmd" The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" diff --git a/tests/gantry_options_spec.sh b/tests/gantry_options_spec.sh index 0790188..01bcc2c 100644 --- a/tests/gantry_options_spec.sh +++ b/tests/gantry_options_spec.sh @@ -83,7 +83,7 @@ Describe 'Options' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*${GANTRY_UPDATE_OPTIONS}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh index 174a405..b6aba3e 100644 --- a/tests/gantry_rollback_spec.sh +++ b/tests/gantry_rollback_spec.sh @@ -93,7 +93,7 @@ Describe 'Rollback' The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*${GANTRY_ROLLBACK_OPTIONS}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--incorrect-option" The stderr should satisfy spec_expect_message "${ROLLING_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" From 8f24bbf7660fbb64e6e6619b6f403208f319d14c Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Fri, 16 Feb 2024 11:04:25 -0800 Subject: [PATCH 08/19] [tests] move tests, based on option category in readme.md --- .github/workflows/coverage.yml | 2 + .github/workflows/on-pull-request.yml | 11 +- .github/workflows/on-push.yml | 11 +- .github/workflows/on-release.yml | 18 +- tests/gantry_cleanup_images_spec.sh | 73 ++++++- ..._spec.sh => gantry_common_options_spec.sh} | 154 +++++++-------- tests/gantry_entrypoint_spec.sh | 180 ------------------ tests/gantry_filters_spec.sh | 4 +- tests/gantry_login_spec.sh | 4 +- tests/gantry_manifest_spec.sh | 4 +- tests/gantry_notify_spec.sh | 4 +- tests/gantry_rollback_spec.sh | 4 +- ...pec.sh => gantry_service_multiple_spec.sh} | 4 +- ...> gantry_service_no_running_tasks_spec.sh} | 4 +- ..._spec.sh => gantry_service_single_spec.sh} | 4 +- ..._spec.sh => gantry_update_options_spec.sh} | 129 +++++++++++-- tests/spec_gantry_test_helper.sh | 3 + 17 files changed, 289 insertions(+), 324 deletions(-) rename tests/{gantry_options_spec.sh => gantry_common_options_spec.sh} (72%) delete mode 100644 tests/gantry_entrypoint_spec.sh rename tests/{gantry_multiple_services_spec.sh => gantry_service_multiple_spec.sh} (98%) rename tests/{gantry_no_running_tasks_spec.sh => gantry_service_no_running_tasks_spec.sh} (98%) rename tests/{gantry_simple_spec.sh => gantry_service_single_spec.sh} (99%) rename tests/{gantry_job_spec.sh => gantry_update_options_spec.sh} (54%) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ba594cb..7e9a55f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -47,6 +47,8 @@ jobs: - name: Upload coverage reports to Codecov if: ${{ github.ref == 'refs/heads/main' }} uses: codecov/codecov-action@v4 + with: + directory: coverage env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml index 0795e79..f5cc31b 100644 --- a/.github/workflows/on-pull-request.yml +++ b/.github/workflows/on-pull-request.yml @@ -30,17 +30,16 @@ jobs: matrix: test_suit: - gantry_cleanup_images_spec.sh - - gantry_entrypoint_spec.sh + - gantry_common_options_spec.sh - gantry_filters_spec.sh - - gantry_job_spec.sh - gantry_login_spec.sh - gantry_manifest_spec.sh - - gantry_multiple_services_spec.sh - - gantry_no_running_tasks_spec.sh - gantry_notify_spec.sh - - gantry_options_spec.sh - gantry_rollback_spec.sh - - gantry_simple_spec.sh + - gantry_service_multiple_spec.sh + - gantry_service_no_running_tasks_spec.sh + - gantry_service_single_spec.sh + - gantry_update_options_spec.sh steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.0.0 diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 00da7e5..e0a2c21 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -37,17 +37,16 @@ jobs: matrix: test_suit: - gantry_cleanup_images_spec.sh - - gantry_entrypoint_spec.sh + - gantry_common_options_spec.sh - gantry_filters_spec.sh - - gantry_job_spec.sh - gantry_login_spec.sh - gantry_manifest_spec.sh - - gantry_multiple_services_spec.sh - - gantry_no_running_tasks_spec.sh - gantry_notify_spec.sh - - gantry_options_spec.sh - gantry_rollback_spec.sh - - gantry_simple_spec.sh + - gantry_service_multiple_spec.sh + - gantry_service_no_running_tasks_spec.sh + - gantry_service_single_spec.sh + - gantry_update_options_spec.sh steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.0.0 diff --git a/.github/workflows/on-release.yml b/.github/workflows/on-release.yml index b5546a4..19cb497 100644 --- a/.github/workflows/on-release.yml +++ b/.github/workflows/on-release.yml @@ -27,22 +27,6 @@ jobs: tests: name: Run tests runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - test_suit: - - gantry_cleanup_images_spec.sh - - gantry_entrypoint_spec.sh - - gantry_filters_spec.sh - - gantry_job_spec.sh - - gantry_login_spec.sh - - gantry_manifest_spec.sh - - gantry_multiple_services_spec.sh - - gantry_no_running_tasks_spec.sh - - gantry_notify_spec.sh - - gantry_options_spec.sh - - gantry_rollback_spec.sh - - gantry_simple_spec.sh steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.0.0 @@ -60,7 +44,7 @@ jobs: run: | export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }} export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} - bash shellspec --pattern tests/${{ matrix.test_suit }} + bash shellspec --jobs 50 build_and_push: name: Build and push Docker image diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh index 5f1d8a6..c262f2d 100644 --- a/tests/gantry_cleanup_images_spec.sh +++ b/tests/gantry_cleanup_images_spec.sh @@ -16,8 +16,8 @@ # # Test cleanup images related options. -Describe 'Cleanup_images' - SUITE_NAME="Cleanup_images" +Describe 'cleanup-images' + SUITE_NAME="cleanup-images" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_CLEANUP_IMAGES_false" "container_test:true" @@ -147,4 +147,73 @@ Describe 'Cleanup_images' The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End End + Describe "test_IMAGES_TO_REMOVE_none_empty" "container_test:true" + # Test the remove image entrypoint. To improve coverage. + TEST_NAME="test_IMAGES_TO_REMOVE_none_empty" + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") + IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0" + IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" + IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2" + SERVICE_NAME="gantry-test-$(unique_id)" + SERVICE_NAME0="${SERVICE_NAME}-0" + SERVICE_NAME1="${SERVICE_NAME}-1" + test_start() { + local TEST_NAME=${1} + local IMAGE_WITH_TAG=${2} + local SERVICE_NAME=${3} + local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0" + local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" + local IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2" + local SERVICE_NAME0="${SERVICE_NAME}-0" + local SERVICE_NAME1="${SERVICE_NAME}-1" + local TASK_SECONDS=15 + initialize_test "${TEST_NAME}" + # The task will finish in ${TASK_SECONDS} seconds + build_and_push_test_image "${IMAGE_WITH_TAG0}" "${TASK_SECONDS}" + start_global_service "${SERVICE_NAME0}" "${IMAGE_WITH_TAG0}" + build_and_push_test_image "${IMAGE_WITH_TAG1}" + start_global_service "${SERVICE_NAME1}" "${IMAGE_WITH_TAG1}" + # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks. + wait_zero_running_tasks "${SERVICE_NAME0}" + # Do not creat the Image IMAGE_WITH_TAG2, to run the test on a non-exist image. + } + test_IMAGES_TO_REMOVE_none_empty() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + local IMAGE_WITH_TAG=${3} + local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0" + local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" + local IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2" + reset_gantry_env "${SERVICE_NAME}" + export GANTRY_IMAGES_TO_REMOVE="${IMAGE_WITH_TAG0} ${IMAGE_WITH_TAG1} ${IMAGE_WITH_TAG2}" + run_gantry "${TEST_NAME}" + } + test_end() { + local TEST_NAME=${1} + local IMAGE_WITH_TAG=${2} + local SERVICE_NAME=${3} + local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0" + local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" + local SERVICE_NAME0="${SERVICE_NAME}-0" + local SERVICE_NAME1="${SERVICE_NAME}-1" + stop_service "${SERVICE_NAME0}" + stop_service "${SERVICE_NAME1}" + # If run successfully, IMAGE_WITH_TAG0 should already be removed. + prune_local_test_image "${IMAGE_WITH_TAG0}" 2>&1 + prune_local_test_image "${IMAGE_WITH_TAG1}" + finalize_test "${TEST_NAME}" + } + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_IMAGES_TO_REMOVE_none_empty "${TEST_NAME}" "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" + The status should be success + The stdout should satisfy display_output + The stdout should satisfy spec_expect_message "Removed exited container.*${SERVICE_NAME0}.*${IMAGE_WITH_TAG0}" + The stdout should satisfy spec_expect_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG0}" + The stdout should satisfy spec_expect_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG1}" + The stdout should satisfy spec_expect_message "There is no image.*${IMAGE_WITH_TAG2}" + The stderr should satisfy display_output + End + End End # Describe 'Single service' diff --git a/tests/gantry_options_spec.sh b/tests/gantry_common_options_spec.sh similarity index 72% rename from tests/gantry_options_spec.sh rename to tests/gantry_common_options_spec.sh index 01bcc2c..fae2619 100644 --- a/tests/gantry_options_spec.sh +++ b/tests/gantry_common_options_spec.sh @@ -15,173 +15,163 @@ # along with this program. If not, see . # -Describe 'Options' - SUITE_NAME="Options" +Describe 'common-options' + SUITE_NAME="common-options" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" - Describe "test_options_LOG_LEVEL_none" "container_test:true" - TEST_NAME="test_options_LOG_LEVEL_none" + Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:false" + TEST_NAME="test_common_DOCKER_HOST_not_swarm_manager" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - test_options_LOG_LEVEL_none() { + test_common_DOCKER_HOST_not_swarm_manager() { local TEST_NAME=${1} local SERVICE_NAME=${2} reset_gantry_env "${SERVICE_NAME}" - # Same as test_new_image_yes, except set LOG_LEVEL to NONE - export GANTRY_LOG_LEVEL=NONE - run_gantry "${TEST_NAME}" - } - BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_test' - When run test_options_LOG_LEVEL_none "${TEST_NAME}" "${SERVICE_NAME}" - The status should be success - The stdout should satisfy display_output - The stdout should satisfy spec_expect_no_message ".+" - The stderr should satisfy display_output - The stderr should satisfy spec_expect_no_message ".+" - End - End - Describe "test_options_UPDATE_OPTIONS" "container_test:true" - TEST_NAME="test_options_UPDATE_OPTIONS" - IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") - SERVICE_NAME="gantry-test-$(unique_id)" - _read_service_label() { - local SERVICE_NAME="${1}" - local LABEL="${2}" - docker service inspect -f "{{index .Spec.Labels \"${LABEL}\"}}" "${SERVICE_NAME}" - } - test_options_UPDATE_OPTIONS() { - local TEST_NAME=${1} - local SERVICE_NAME=${2} - local LABEL="gantry.test" - local LABEL_VALUE= - LABEL_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}") - echo "Before updating: LABEL_VALUE=${LABEL_VALUE}" - reset_gantry_env "${SERVICE_NAME}" - export GANTRY_UPDATE_OPTIONS="--label-add=${LABEL}=${SERVICE_NAME}" - local RETURN_VALUE= + export DOCKER_HOST="8.8.8.8:53" + local RETURN_VALUE=0 run_gantry "${TEST_NAME}" RETURN_VALUE="${?}" - LABEL_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}") - echo "After updating: LABEL_VALUE=${LABEL_VALUE}" + export DOCKER_HOST= return "${RETURN_VALUE}" } BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_options_UPDATE_OPTIONS "${TEST_NAME}" "${SERVICE_NAME}" - The status should be success + When run test_common_DOCKER_HOST_not_swarm_manager "${TEST_NAME}" "${SERVICE_NAME}" + The status should be failure The stdout should satisfy display_output - # Check an observable difference before and after applying UPDATE_OPTIONS. - The stdout should satisfy spec_expect_no_message "Before updating: LABEL_VALUE=.*${SERVICE_NAME}" - The stdout should satisfy spec_expect_message "After updating: LABEL_VALUE=.*${SERVICE_NAME}" The stderr should satisfy display_output + The stderr should satisfy spec_expect_message "${SKIP_UPDATING_ALL}.*${SKIP_REASON_NOT_SWARM_MANAGER}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}" - The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}" + The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" - The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" + The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}" End End - Describe "test_options_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false" - TEST_NAME="test_options_UPDATE_TIMEOUT_SECONDS_not_a_number" + Describe "test_common_LOG_LEVEL_none" "container_test:true" + TEST_NAME="test_common_LOG_LEVEL_none" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - test_options_UPDATE_TIMEOUT_SECONDS_not_a_number() { + test_common_LOG_LEVEL_none() { local TEST_NAME=${1} local SERVICE_NAME=${2} reset_gantry_env "${SERVICE_NAME}" - export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber" + # Same as test_new_image_yes, except set LOG_LEVEL to NONE + export GANTRY_LOG_LEVEL=NONE run_gantry "${TEST_NAME}" } BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_options_UPDATE_TIMEOUT_SECONDS_not_a_number "${TEST_NAME}" "${SERVICE_NAME}" - The status should be failure + When run test_common_LOG_LEVEL_none "${TEST_NAME}" "${SERVICE_NAME}" + The status should be success + The stdout should satisfy display_output + The stdout should satisfy spec_expect_no_message ".+" + The stderr should satisfy display_output + The stderr should satisfy spec_expect_no_message ".+" + End + End + Describe "test_common_PRE_POST_RUN_CMD" "container_test:true" + TEST_NAME="test_common_PRE_POST_RUN_CMD" + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") + SERVICE_NAME="gantry-test-$(unique_id)" + test_common_PRE_POST_RUN_CMD() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + export GANTRY_PRE_RUN_CMD="echo \"Pre update\"" + export GANTRY_POST_RUN_CMD="echo \"Post update\"" + run_gantry "${TEST_NAME}" + } + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_common_PRE_POST_RUN_CMD "${TEST_NAME}" "${SERVICE_NAME}" + The status should be success The stdout should satisfy display_output The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "GANTRY_UPDATE_TIMEOUT_SECONDS must be a number.*" + The stderr should satisfy spec_expect_message "Pre update" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_ERRORS}" - The stderr should satisfy spec_expect_message "${NO_IMAGES_TO_REMOVE}" - The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" + The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" + The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_message "Post update" End End - Describe "test_options_PRE_POST_RUN_CMD" "container_test:true" - TEST_NAME="test_options_PRE_POST_RUN_CMD" + Describe "test_common_SLEEP_SECONDS_not_a_number" "container_test:false" + TEST_NAME="test_common_SLEEP_SECONDS_not_a_number" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - test_options_PRE_POST_RUN_CMD() { + test_common_SLEEP_SECONDS_not_a_number() { local TEST_NAME=${1} local SERVICE_NAME=${2} reset_gantry_env "${SERVICE_NAME}" - export GANTRY_PRE_RUN_CMD="echo \"Pre update\"" - export GANTRY_POST_RUN_CMD="echo \"Post update\"" + export GANTRY_SLEEP_SECONDS="NotANumber" run_gantry "${TEST_NAME}" } BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_options_PRE_POST_RUN_CMD "${TEST_NAME}" "${SERVICE_NAME}" - The status should be success + When run test_common_SLEEP_SECONDS_not_a_number "${TEST_NAME}" "${SERVICE_NAME}" + The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "Pre update" + The stderr should satisfy spec_expect_message "GANTRY_SLEEP_SECONDS must be a number.*" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING_ALL}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}" + The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" - The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" + The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" - The stderr should satisfy spec_expect_message "Post update" + The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}" End End End # Describe 'Single service' diff --git a/tests/gantry_entrypoint_spec.sh b/tests/gantry_entrypoint_spec.sh deleted file mode 100644 index 0ab722e..0000000 --- a/tests/gantry_entrypoint_spec.sh +++ /dev/null @@ -1,180 +0,0 @@ -#!/bin/bash spellspec -# Copyright (C) 2024 Shizun Ge -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -Describe 'Entrypoint' - SUITE_NAME="Entrypoint" - BeforeAll "initialize_all_tests ${SUITE_NAME}" - AfterAll "finish_all_tests ${SUITE_NAME}" - Describe "test_DOCKER_HOST_not_swarm_manager" "container_test:false" - TEST_NAME="test_DOCKER_HOST_not_swarm_manager" - IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") - SERVICE_NAME="gantry-test-$(unique_id)" - test_DOCKER_HOST_not_swarm_manager() { - local TEST_NAME=${1} - local SERVICE_NAME=${2} - reset_gantry_env "${SERVICE_NAME}" - export DOCKER_HOST="8.8.8.8:53" - local RETURN_VALUE=0 - run_gantry "${TEST_NAME}" - RETURN_VALUE="${?}" - export DOCKER_HOST= - return "${RETURN_VALUE}" - } - BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_test' - When run test_DOCKER_HOST_not_swarm_manager "${TEST_NAME}" "${SERVICE_NAME}" - The status should be failure - The stdout should satisfy display_output - The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "${SKIP_UPDATING_ALL}.*${SKIP_REASON_NOT_SWARM_MANAGER}" - The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" - The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" - The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" - The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" - The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" - The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}" - End - End - Describe "test_SLEEP_SECONDS_not_a_number" "container_test:false" - TEST_NAME="test_SLEEP_SECONDS_not_a_number" - IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") - SERVICE_NAME="gantry-test-$(unique_id)" - test_SLEEP_SECONDS_not_a_number() { - local TEST_NAME=${1} - local SERVICE_NAME=${2} - reset_gantry_env "${SERVICE_NAME}" - export GANTRY_SLEEP_SECONDS="NotANumber" - run_gantry "${TEST_NAME}" - } - BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_test' - When run test_SLEEP_SECONDS_not_a_number "${TEST_NAME}" "${SERVICE_NAME}" - The status should be failure - The stdout should satisfy display_output - The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "GANTRY_SLEEP_SECONDS must be a number.*" - The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING_ALL}" - The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" - The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" - The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" - The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" - The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" - The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}" - End - End - Describe "test_IMAGES_TO_REMOVE_none_empty" "container_test:true" - # Test the remove image entrypoint. To improve coverage. - TEST_NAME="test_IMAGES_TO_REMOVE_none_empty" - IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") - IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0" - IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" - IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2" - SERVICE_NAME="gantry-test-$(unique_id)" - SERVICE_NAME0="${SERVICE_NAME}-0" - SERVICE_NAME1="${SERVICE_NAME}-1" - test_start() { - local TEST_NAME=${1} - local IMAGE_WITH_TAG=${2} - local SERVICE_NAME=${3} - local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0" - local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" - local IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2" - local SERVICE_NAME0="${SERVICE_NAME}-0" - local SERVICE_NAME1="${SERVICE_NAME}-1" - local TASK_SECONDS=15 - initialize_test "${TEST_NAME}" - # The task will finish in ${TASK_SECONDS} seconds - build_and_push_test_image "${IMAGE_WITH_TAG0}" "${TASK_SECONDS}" - start_global_service "${SERVICE_NAME0}" "${IMAGE_WITH_TAG0}" - build_and_push_test_image "${IMAGE_WITH_TAG1}" - start_global_service "${SERVICE_NAME1}" "${IMAGE_WITH_TAG1}" - # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks. - wait_zero_running_tasks "${SERVICE_NAME0}" - # Do not creat the Image IMAGE_WITH_TAG2, to run the test on a non-exist image. - } - test_IMAGES_TO_REMOVE_none_empty() { - local TEST_NAME=${1} - local SERVICE_NAME=${2} - local IMAGE_WITH_TAG=${3} - local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0" - local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" - local IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2" - reset_gantry_env "${SERVICE_NAME}" - export GANTRY_IMAGES_TO_REMOVE="${IMAGE_WITH_TAG0} ${IMAGE_WITH_TAG1} ${IMAGE_WITH_TAG2}" - run_gantry "${TEST_NAME}" - } - test_end() { - local TEST_NAME=${1} - local IMAGE_WITH_TAG=${2} - local SERVICE_NAME=${3} - local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0" - local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" - local SERVICE_NAME0="${SERVICE_NAME}-0" - local SERVICE_NAME1="${SERVICE_NAME}-1" - stop_service "${SERVICE_NAME0}" - stop_service "${SERVICE_NAME1}" - # If run successfully, IMAGE_WITH_TAG0 should already be removed. - prune_local_test_image "${IMAGE_WITH_TAG0}" 2>&1 - prune_local_test_image "${IMAGE_WITH_TAG1}" - finalize_test "${TEST_NAME}" - } - BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_test' - When run test_IMAGES_TO_REMOVE_none_empty "${TEST_NAME}" "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" - The status should be success - The stdout should satisfy display_output - The stdout should satisfy spec_expect_message "Removed exited container.*${SERVICE_NAME0}.*${IMAGE_WITH_TAG0}" - The stdout should satisfy spec_expect_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG0}" - The stdout should satisfy spec_expect_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG1}" - The stdout should satisfy spec_expect_message "There is no image.*${IMAGE_WITH_TAG2}" - The stderr should satisfy display_output - End - End -End # Describe 'Single service' diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh index dbc54ed..708ef6b 100644 --- a/tests/gantry_filters_spec.sh +++ b/tests/gantry_filters_spec.sh @@ -15,8 +15,8 @@ # along with this program. If not, see . # -Describe 'Filters' - SUITE_NAME="Filters" +Describe 'filters' + SUITE_NAME="filters" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_SERVICES_FILTERS_bad" "container_test:false" diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh index fa48afe..0a1f521 100644 --- a/tests/gantry_login_spec.sh +++ b/tests/gantry_login_spec.sh @@ -15,8 +15,8 @@ # along with this program. If not, see . # -Describe 'Login' - SUITE_NAME="Login" +Describe 'login' + SUITE_NAME="login" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" # Here are just simple login tests. diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh index 8781d33..7baafd7 100644 --- a/tests/gantry_manifest_spec.sh +++ b/tests/gantry_manifest_spec.sh @@ -15,8 +15,8 @@ # along with this program. If not, see . # -Describe 'Manifest_command' - SUITE_NAME="Manifest_command" +Describe 'manifest-command' + SUITE_NAME="manifest-command" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_MANIFEST_CMD_none" "container_test:true" diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh index 555c5fb..716c15e 100644 --- a/tests/gantry_notify_spec.sh +++ b/tests/gantry_notify_spec.sh @@ -17,8 +17,8 @@ export SKIP_NOTIFY_APPRISE="Skip sending notification via Apprise" -Describe 'Notify' - SUITE_NAME="Notify" +Describe 'notify' + SUITE_NAME="notify" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_notify_apprise" "container_test:true" diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh index b6aba3e..f7adc0a 100644 --- a/tests/gantry_rollback_spec.sh +++ b/tests/gantry_rollback_spec.sh @@ -15,8 +15,8 @@ # along with this program. If not, see . # -Describe 'Rollback' - SUITE_NAME="Rollback" +Describe 'rollback' + SUITE_NAME="rollback" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_rollback_due_to_timeout" "container_test:false" diff --git a/tests/gantry_multiple_services_spec.sh b/tests/gantry_service_multiple_spec.sh similarity index 98% rename from tests/gantry_multiple_services_spec.sh rename to tests/gantry_service_multiple_spec.sh index dedd02d..2594e56 100644 --- a/tests/gantry_multiple_services_spec.sh +++ b/tests/gantry_service_multiple_spec.sh @@ -15,8 +15,8 @@ # along with this program. If not, see . # -Describe 'Multiple_services' - SUITE_NAME="Multiple_services" +Describe 'service-multiple-services' + SUITE_NAME="service-multiple-services" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_multiple_services_excluded_filters" "container_test:true" diff --git a/tests/gantry_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh similarity index 98% rename from tests/gantry_no_running_tasks_spec.sh rename to tests/gantry_service_no_running_tasks_spec.sh index 2dc9193..5b6d298 100644 --- a/tests/gantry_no_running_tasks_spec.sh +++ b/tests/gantry_service_no_running_tasks_spec.sh @@ -15,8 +15,8 @@ # along with this program. If not, see . # -Describe "No_Running_Tasks" - SUITE_NAME="No_Running_Tasks" +Describe "service-no-running-tasks" + SUITE_NAME="service-no-running-tasks" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_no_running_tasks_replicated" "container_test:true" diff --git a/tests/gantry_simple_spec.sh b/tests/gantry_service_single_spec.sh similarity index 99% rename from tests/gantry_simple_spec.sh rename to tests/gantry_service_single_spec.sh index 4e6c1f0..fc4db8d 100644 --- a/tests/gantry_simple_spec.sh +++ b/tests/gantry_service_single_spec.sh @@ -15,8 +15,8 @@ # along with this program. If not, see . # -Describe 'Simple' - SUITE_NAME="Simple" +Describe 'service-single-service' + SUITE_NAME="service-single-service" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_new_image_no" "container_test:true" diff --git a/tests/gantry_job_spec.sh b/tests/gantry_update_options_spec.sh similarity index 54% rename from tests/gantry_job_spec.sh rename to tests/gantry_update_options_spec.sh index 6a96224..c62af04 100644 --- a/tests/gantry_job_spec.sh +++ b/tests/gantry_update_options_spec.sh @@ -15,15 +15,15 @@ # along with this program. If not, see . # -Describe 'Job' - SUITE_NAME="Job" +Describe 'update-options' + SUITE_NAME="update-options" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" - Describe "test_jobs_skipping" "container_test:true" - TEST_NAME="test_jobs_skipping" + Describe "test_update_jobs_skipping" "container_test:true" + TEST_NAME="test_update_jobs_skipping" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - test_jobs_skipping() { + test_update_jobs_skipping() { local TEST_NAME=${1} local SERVICE_NAME=${2} reset_gantry_env "${SERVICE_NAME}" @@ -32,7 +32,7 @@ Describe 'Job' BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_jobs_skipping "${TEST_NAME}" "${SERVICE_NAME}" + When run test_update_jobs_skipping "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -59,11 +59,11 @@ Describe 'Job' The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End End - Describe "test_jobs_UPDATE_JOBS_true" "container_test:true" - TEST_NAME="test_jobs_UPDATE_JOBS_true" + Describe "test_update_jobs_UPDATE_JOBS_true" "container_test:true" + TEST_NAME="test_update_jobs_UPDATE_JOBS_true" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - test_jobs_UPDATE_JOBS_true() { + test_update_jobs_UPDATE_JOBS_true() { local TEST_NAME=${1} local SERVICE_NAME=${2} reset_gantry_env "${SERVICE_NAME}" @@ -75,7 +75,7 @@ Describe 'Job' BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_jobs_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}" + When run test_update_jobs_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -103,12 +103,12 @@ Describe 'Job' The stderr should satisfy spec_expect_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End End - Describe "test_jobs_UPDATE_JOBS_true_no_running_tasks" "container_test:true" - TEST_NAME="test_jobs_UPDATE_JOBS_true_no_running_tasks" + Describe "test_update_jobs_no_running_tasks" "container_test:true" + TEST_NAME="test_update_jobs_no_running_tasks" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" TASK_SECONDS=15 - test_jobs_UPDATE_JOBS_true_no_running_tasks() { + test_update_jobs_no_running_tasks() { local TEST_NAME=${1} local SERVICE_NAME=${2} # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks. @@ -121,7 +121,7 @@ Describe 'Job' BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_jobs_UPDATE_JOBS_true_no_running_tasks "${TEST_NAME}" "${SERVICE_NAME}" + When run test_update_jobs_no_running_tasks "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -150,4 +150,103 @@ Describe 'Job' The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End End -End # Describe 'Job' + Describe "test_update_UPDATE_OPTIONS" "container_test:true" + TEST_NAME="test_update_UPDATE_OPTIONS" + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") + SERVICE_NAME="gantry-test-$(unique_id)" + _read_service_label() { + local SERVICE_NAME="${1}" + local LABEL="${2}" + docker service inspect -f "{{index .Spec.Labels \"${LABEL}\"}}" "${SERVICE_NAME}" + } + test_update_UPDATE_OPTIONS() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + local LABEL="gantry.test" + local LABEL_VALUE= + LABEL_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}") + echo "Before updating: LABEL_VALUE=${LABEL_VALUE}" + reset_gantry_env "${SERVICE_NAME}" + export GANTRY_UPDATE_OPTIONS="--label-add=${LABEL}=${SERVICE_NAME}" + local RETURN_VALUE= + run_gantry "${TEST_NAME}" + RETURN_VALUE="${?}" + LABEL_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}") + echo "After updating: LABEL_VALUE=${LABEL_VALUE}" + return "${RETURN_VALUE}" + } + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_update_UPDATE_OPTIONS "${TEST_NAME}" "${SERVICE_NAME}" + The status should be success + The stdout should satisfy display_output + # Check an observable difference before and after applying UPDATE_OPTIONS. + The stdout should satisfy spec_expect_no_message "Before updating: LABEL_VALUE=.*${SERVICE_NAME}" + The stdout should satisfy spec_expect_message "After updating: LABEL_VALUE=.*${SERVICE_NAME}" + The stderr should satisfy display_output + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" + The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" + The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" + The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" + The stderr should satisfy spec_expect_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" + End + End + Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false" + TEST_NAME="test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") + SERVICE_NAME="gantry-test-$(unique_id)" + test_update_UPDATE_TIMEOUT_SECONDS_not_a_number() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber" + run_gantry "${TEST_NAME}" + } + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_update_UPDATE_TIMEOUT_SECONDS_not_a_number "${TEST_NAME}" "${SERVICE_NAME}" + The status should be failure + The stdout should satisfy display_output + The stderr should satisfy display_output + The stderr should satisfy spec_expect_message "GANTRY_UPDATE_TIMEOUT_SECONDS must be a number.*" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" + The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${NO_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_ERRORS}" + The stderr should satisfy spec_expect_message "${NO_IMAGES_TO_REMOVE}" + The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" + The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" + The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" + End + End +End # Describe 'Single service' diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index e23a648..ae70055 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -483,6 +483,7 @@ _wait_service_state() { start_replicated_service() { local SERVICE_NAME="${1}" local IMAGE_WITH_TAG="${2}" + [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63} echo "Creating service ${SERVICE_NAME} in replicated mode " # SC2046 (warning): Quote this to prevent word splitting. # shellcheck disable=SC2046 @@ -498,6 +499,7 @@ start_replicated_service() { start_global_service() { local SERVICE_NAME="${1}" local IMAGE_WITH_TAG="${2}" + [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63} echo "Creating service ${SERVICE_NAME} in global mode " # SC2046 (warning): Quote this to prevent word splitting. # shellcheck disable=SC2046 @@ -513,6 +515,7 @@ start_global_service() { _start_replicated_job() { local SERVICE_NAME="${1}" local IMAGE_WITH_TAG="${2}" + [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63} echo "Creating service ${SERVICE_NAME} in replicated job mode " # SC2046 (warning): Quote this to prevent word splitting. # shellcheck disable=SC2046 From 91c67d867398a2e2ccae4d63a3e4f67f4e0f3cdb Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Fri, 16 Feb 2024 21:46:06 -0800 Subject: [PATCH 09/19] add lock to static variables. --- src/lib-gantry.sh | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index 7edc471..34ba9a3 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -138,7 +138,19 @@ _send_notification() { notify_summary "${TYPE}" "${TITLE}" "${BODY}" } -_static_variable_read_list() { +_lock() { + local NAME="${1}" + local LOCK_NAME="${STATIC_VARIABLES_FOLDER}/${NAME}-LOCK" + while ! mkdir "${LOCK_NAME}" >/dev/null 2>&1; do sleep 0.001; done +} + +_unlock() { + local NAME="${1}" + local LOCK_NAME="${STATIC_VARIABLES_FOLDER}/${NAME}-LOCK" + rm -r "${LOCK_NAME}" >/dev/null 2>&1 +} + +_static_variable_read_list_core() { local LIST_NAME="${1}" [ -z "${LIST_NAME}" ] && log ERROR "LIST_NAME is empty." && return 1 local FILE_NAME="${STATIC_VARIABLES_FOLDER}/${LIST_NAME}" @@ -146,16 +158,35 @@ _static_variable_read_list() { cat "${FILE_NAME}" } -# Add unique value to a static variable which holds a list. -_static_variable_add_unique_to_list() { +_static_variable_add_unique_to_list_core() { local LIST_NAME="${1}" local VALUE="${2}" [ -z "${LIST_NAME}" ] && log ERROR "LIST_NAME is empty." && return 1 local FILE_NAME="${STATIC_VARIABLES_FOLDER}/${LIST_NAME}" local OLD_LIST NEW_LIST - OLD_LIST=$(_static_variable_read_list "${LIST_NAME}") + OLD_LIST=$(_static_variable_read_list_core "${LIST_NAME}") NEW_LIST=$(add_unique_to_list "${OLD_LIST}" "${VALUE}") echo "${NEW_LIST}" > "${FILE_NAME}" + +} + +_static_variable_read_list() { + local LIST_NAME="${1}" + _lock "${LIST_NAME}" + _static_variable_read_list_core "${@}" + local RETURN_VALUE=$? + _unlock "${LIST_NAME}" + return "${RETURN_VALUE}" +} + +# Add unique value to a static variable which holds a list. +_static_variable_add_unique_to_list() { + local LIST_NAME="${1}" + _lock "${LIST_NAME}" + _static_variable_add_unique_to_list_core "${@}" + local RETURN_VALUE=$? + _unlock "${LIST_NAME}" + return "${RETURN_VALUE}" } _remove_container() { From 8e41c2cb6e6c8b291efa3f0583fa18d6a3d77ac3 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Fri, 16 Feb 2024 22:19:40 -0800 Subject: [PATCH 10/19] add gantry_service_parallel_spec.sh --- .github/workflows/on-pull-request.yml | 1 + .github/workflows/on-push.yml | 1 + tests/gantry_service_parallel_spec.sh | 206 ++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 tests/gantry_service_parallel_spec.sh diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml index f5cc31b..2c13a7c 100644 --- a/.github/workflows/on-pull-request.yml +++ b/.github/workflows/on-pull-request.yml @@ -38,6 +38,7 @@ jobs: - gantry_rollback_spec.sh - gantry_service_multiple_spec.sh - gantry_service_no_running_tasks_spec.sh + - gantry_service_parallel_spec.sh - gantry_service_single_spec.sh - gantry_update_options_spec.sh steps: diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index e0a2c21..843408f 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -45,6 +45,7 @@ jobs: - gantry_rollback_spec.sh - gantry_service_multiple_spec.sh - gantry_service_no_running_tasks_spec.sh + - gantry_service_parallel_spec.sh - gantry_service_single_spec.sh - gantry_update_options_spec.sh steps: diff --git a/tests/gantry_service_parallel_spec.sh b/tests/gantry_service_parallel_spec.sh new file mode 100644 index 0000000..adea17b --- /dev/null +++ b/tests/gantry_service_parallel_spec.sh @@ -0,0 +1,206 @@ +#!/bin/bash spellspec +# Copyright (C) 2024 Shizun Ge +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +Describe 'service-parallel' + SUITE_NAME="service-parallel" + BeforeAll "initialize_all_tests ${SUITE_NAME}" + AfterAll "finish_all_tests ${SUITE_NAME}" + Describe "test_service_parallel_less_workers" "container_test:true" + TEST_NAME="test_service_parallel_less_workers" + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") + SERVICE_NAME="gantry-test-$(unique_id)" + test_start() { + local TEST_NAME=${1} + local IMAGE_WITH_TAG=${2} + local SERVICE_NAME=${3} + initialize_test "${TEST_NAME}" + build_and_push_test_image "${IMAGE_WITH_TAG}" + local NUM= + local PIDS= + for NUM in $(seq 0 6); do + local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" & + PIDS="${!} ${PIDS}" + done + # SC2086 (info): Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + wait ${PIDS} + build_and_push_test_image "${IMAGE_WITH_TAG}" + PIDS= + for NUM in $(seq 7 9); do + local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" & + PIDS="${!} ${PIDS}" + done + # SC2086 (info): Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + wait ${PIDS} + } + test_service_parallel_less_workers() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + export GANTRY_UPDATE_WORKERS=5 + run_gantry "${TEST_NAME}" + } + test_end() { + local TEST_NAME=${1} + local IMAGE_WITH_TAG=${2} + local SERVICE_NAME=${3} + local NUM= + for NUM in $(seq 0 9); do + local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + stop_service "${SERVICE_NAME_NUM}" + done + prune_local_test_image "${IMAGE_WITH_TAG}" + finalize_test "${TEST_NAME}" + } + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_service_parallel_less_workers "${TEST_NAME}" "${SERVICE_NAME}" + The status should be success + The stdout should satisfy display_output + The stderr should satisfy display_output + SERVICE_NAME_NUM="${SERVICE_NAME}-0" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" + The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME_NUM}" + for NUM in $(seq 1 6); do + SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}.*${PERFORM_REASON_KNOWN_NEWER_IMAGE}" + The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME_NUM}" + done + SERVICE_NAME_NUM="${SERVICE_NAME}-7" + The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}.*${SKIP_REASON_CURRENT_IS_LATEST}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME_NUM}" + for NUM in $(seq 8 9); do + SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}.*${SKIP_REASON_NO_KNOWN_NEWER_IMAGE}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME_NUM}" + done + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_NO_NEW_IMAGES}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" + The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" + The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" + The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" + The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" + The stderr should satisfy spec_expect_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" + End + End + Describe "test_service_parallel_more_workers" "container_test:true" + TEST_NAME="test_service_parallel_more_workers" + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") + SERVICE_NAME="gantry-test-$(unique_id)" + test_start() { + local TEST_NAME=${1} + local IMAGE_WITH_TAG=${2} + local SERVICE_NAME=${3} + initialize_test "${TEST_NAME}" + build_and_push_test_image "${IMAGE_WITH_TAG}" + local NUM= + local PIDS= + for NUM in $(seq 0 4); do + local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" & + PIDS="${!} ${PIDS}" + done + # SC2086 (info): Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + wait ${PIDS} + build_and_push_test_image "${IMAGE_WITH_TAG}" + PIDS= + for NUM in $(seq 5 8); do + local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" & + PIDS="${!} ${PIDS}" + done + # SC2086 (info): Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + wait ${PIDS} + } + test_service_parallel_more_workers() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + export GANTRY_UPDATE_WORKERS=50 + run_gantry "${TEST_NAME}" + } + test_end() { + local TEST_NAME=${1} + local IMAGE_WITH_TAG=${2} + local SERVICE_NAME=${3} + local NUM= + for NUM in $(seq 0 8); do + local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + stop_service "${SERVICE_NAME_NUM}" + done + prune_local_test_image "${IMAGE_WITH_TAG}" + finalize_test "${TEST_NAME}" + } + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_service_parallel_more_workers "${TEST_NAME}" "${SERVICE_NAME}" + The status should be success + The stdout should satisfy display_output + The stderr should satisfy display_output + SERVICE_NAME_NUM="${SERVICE_NAME}-0" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" + The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME_NUM}" + for NUM in $(seq 1 4); do + SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}.*${PERFORM_REASON_KNOWN_NEWER_IMAGE}" + The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME_NUM}" + done + SERVICE_NAME_NUM="${SERVICE_NAME}-5" + The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}.*${SKIP_REASON_CURRENT_IS_LATEST}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME_NUM}" + for NUM in $(seq 6 8); do + SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}.*${SKIP_REASON_NO_KNOWN_NEWER_IMAGE}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME_NUM}" + done + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_NO_NEW_IMAGES}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" + The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" + The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" + The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" + The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" + The stderr should satisfy spec_expect_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" + End + End +End # Describe 'Multiple services' From 746e490dfe26d3090717897b848d4f311ff9fa57 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Fri, 16 Feb 2024 23:28:12 -0800 Subject: [PATCH 11/19] run updates in parallel --- .gitignore | 1 + README.md | 1 + docs/migration.md | 1 + src/lib-gantry.sh | 66 +++++++++++++++++++---- tests/gantry_service_parallel_spec.sh | 75 ++++++++++++++++++++++----- tests/spec_gantry_test_helper.sh | 2 + 6 files changed, 124 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 158663a..348c7ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .shellspec-quick.log +coverage diff --git a/README.md b/README.md index 8883d1f..2fb2d96 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ You can configure the most behaviors of *Gantry* via environment variables. | GANTRY_ROLLBACK_ON_FAILURE | true | Set to `true` to enable rollback when updating fails. Set to `false` to disable the rollback. | | GANTRY_ROLLBACK_OPTIONS | | [Options](https://docs.docker.com/engine/reference/commandline/service_update/#options) added to the `docker service update --rollback` command. | | GANTRY_UPDATE_JOBS | false | Set to `true` to update replicated-job or global-job. Set to `false` to disable updating jobs. | +| GANTRY_UPDATE_NUM_WORKERS | 1 | The maximum number of updates that can run in parallel. | | GANTRY_UPDATE_OPTIONS | | [Options](https://docs.docker.com/engine/reference/commandline/service_update/#options) added to the `docker service update` command. | | GANTRY_UPDATE_TIMEOUT_SECONDS | 300 | Error out if updating of a single service takes longer than the given time. | diff --git a/docs/migration.md b/docs/migration.md index 98adc49..3ade0fd 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -65,6 +65,7 @@ The label on the services to select config to enable authentication is renamed t | GANTRY_SERVICES_EXCLUDED_FILTERS | | GANTRY_SERVICES_SELF | | GANTRY_UPDATE_JOBS | +| GANTRY_UPDATE_NUM_WORKERS | ### License diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index 34ba9a3..96b43b7 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -167,7 +167,19 @@ _static_variable_add_unique_to_list_core() { OLD_LIST=$(_static_variable_read_list_core "${LIST_NAME}") NEW_LIST=$(add_unique_to_list "${OLD_LIST}" "${VALUE}") echo "${NEW_LIST}" > "${FILE_NAME}" - +} + +_static_variable_pop_list_core() { + local LIST_NAME="${1}" + [ -z "${LIST_NAME}" ] && log ERROR "LIST_NAME is empty." && return 1 + local FILE_NAME="${STATIC_VARIABLES_FOLDER}/${LIST_NAME}" + [ ! -e "${FILE_NAME}" ] && touch "${FILE_NAME}" + local ITEM= + ITEM=$(head -1 "${FILE_NAME}") + local NEW_LIST= + NEW_LIST=$(tail -n+2 "${FILE_NAME}") + echo "${NEW_LIST}" > "${FILE_NAME}" + echo "${ITEM}" } _static_variable_read_list() { @@ -189,6 +201,16 @@ _static_variable_add_unique_to_list() { return "${RETURN_VALUE}" } +# echo the first item in the list and remove the first item from the list. +_static_variable_pop_list() { + local LIST_NAME="${1}" + _lock "${LIST_NAME}" + _static_variable_pop_list_core "${@}" + local RETURN_VALUE=$? + _unlock "${LIST_NAME}" + return "${RETURN_VALUE}" +} + _remove_container() { local IMAGE="${1}"; local STATUS="${2}"; @@ -751,6 +773,25 @@ _update_single_service() { return 0 } +# To run update in parallel +_update_worker() { + local INDEX="${1}" + local STATIC_VAR_LIST_NAME="${2}" + local OLD_LOG_SCOPE="${LOG_SCOPE}" + LOG_SCOPE=$(attach_tag_to_log_scope "worker${INDEX}") + export LOG_SCOPE + local SERVICE_AND_IMAGE= + local SERVICE IMAGE + while true; do + SERVICE_AND_IMAGE=$(_static_variable_pop_list "${STATIC_VAR_LIST_NAME}") + [ -z "${SERVICE_AND_IMAGE}" ] && break; + SERVICE=$(echo "${SERVICE_AND_IMAGE}" | cut -d ' ' -f 1) + IMAGE=$(echo "${SERVICE_AND_IMAGE}" | cut -d ' ' -f 2) + _update_single_service "${SERVICE}" "${IMAGE}" + done + export LOG_SCOPE="${OLD_LOG_SCOPE}" +} + _get_services_filted() { local SERVICES_FILTERS="${1}" local SERVICES= @@ -818,7 +859,12 @@ gantry_get_services_list() { } gantry_update_services_list() { + local NUM_WORKERS="${GANTRY_UPDATE_NUM_WORKERS:-1}" local LIST="${*}" + if ! is_number "${NUM_WORKERS}"; then + log ERROR "GANTRY_UPDATE_NUM_WORKERS must be a number. Got \"${GANTRY_UPDATE_NUM_WORKERS}\"." + return 1; + fi local NUM= NUM=$(_get_number_of_elements "${LIST}") log INFO "Inspecting ${NUM} service(s)." @@ -836,15 +882,15 @@ gantry_update_services_list() { _report_services_from_static_variable STATIC_VAR_SERVICES_NO_NEW_IMAGE "No new images for" | log_lines INFO _report_services_from_static_variable STATIC_VAR_SERVICES_TO_UPDATE "Updating" | log_lines INFO - local SERVICES_AND_IMAGES_TO_UPDATE= - SERVICES_AND_IMAGES_TO_UPDATE=$(_static_variable_read_list STATIC_VAR_SERVICES_AND_IMAGES_TO_UPDATE) - while read -r SERVICE_AND_IMAGE; do - [ -z "${SERVICE_AND_IMAGE}" ] && continue - local SERVICE IMAGE - SERVICE=$(echo "${SERVICE_AND_IMAGE}" | cut -d ' ' -f 1) - IMAGE=$(echo "${SERVICE_AND_IMAGE}" | cut -d ' ' -f 2) - _update_single_service "${SERVICE}" "${IMAGE}" - done < <(echo "${SERVICES_AND_IMAGES_TO_UPDATE}") + log DEBUG "NUM_WORKERS=${NUM_WORKERS}" + local PIDS= + for INDEX in $(seq 0 $((NUM_WORKERS-1)) ); do + _update_worker "${INDEX}" STATIC_VAR_SERVICES_AND_IMAGES_TO_UPDATE & + PIDS="${!} ${PIDS}" + done + # SC2086 (info): Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + wait ${PIDS} local RETURN_VALUE=0 local FAILED_NUM= diff --git a/tests/gantry_service_parallel_spec.sh b/tests/gantry_service_parallel_spec.sh index adea17b..9f60c0c 100644 --- a/tests/gantry_service_parallel_spec.sh +++ b/tests/gantry_service_parallel_spec.sh @@ -19,8 +19,8 @@ Describe 'service-parallel' SUITE_NAME="service-parallel" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" - Describe "test_service_parallel_less_workers" "container_test:true" - TEST_NAME="test_service_parallel_less_workers" + Describe "test_parallel_less_workers" "container_test:true" + TEST_NAME="test_parallel_less_workers" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_start() { @@ -50,11 +50,11 @@ Describe 'service-parallel' # shellcheck disable=SC2086 wait ${PIDS} } - test_service_parallel_less_workers() { + test_parallel_less_workers() { local TEST_NAME=${1} local SERVICE_NAME=${2} reset_gantry_env "${SERVICE_NAME}" - export GANTRY_UPDATE_WORKERS=5 + export GANTRY_UPDATE_NUM_WORKERS=5 run_gantry "${TEST_NAME}" } test_end() { @@ -62,17 +62,22 @@ Describe 'service-parallel' local IMAGE_WITH_TAG=${2} local SERVICE_NAME=${3} local NUM= + local PIDS= for NUM in $(seq 0 9); do local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - stop_service "${SERVICE_NAME_NUM}" + stop_service "${SERVICE_NAME_NUM}" & + PIDS="${!} ${PIDS}" done + # SC2086 (info): Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + wait ${PIDS} prune_local_test_image "${IMAGE_WITH_TAG}" finalize_test "${TEST_NAME}" } BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_service_parallel_less_workers "${TEST_NAME}" "${SERVICE_NAME}" + When run test_parallel_less_workers "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -111,8 +116,8 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End End - Describe "test_service_parallel_more_workers" "container_test:true" - TEST_NAME="test_service_parallel_more_workers" + Describe "test_parallel_more_workers" "container_test:true" + TEST_NAME="test_parallel_more_workers" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" test_start() { @@ -142,11 +147,11 @@ Describe 'service-parallel' # shellcheck disable=SC2086 wait ${PIDS} } - test_service_parallel_more_workers() { + test_parallel_more_workers() { local TEST_NAME=${1} local SERVICE_NAME=${2} reset_gantry_env "${SERVICE_NAME}" - export GANTRY_UPDATE_WORKERS=50 + export GANTRY_UPDATE_NUM_WORKERS=50 run_gantry "${TEST_NAME}" } test_end() { @@ -154,17 +159,22 @@ Describe 'service-parallel' local IMAGE_WITH_TAG=${2} local SERVICE_NAME=${3} local NUM= + local PIDS= for NUM in $(seq 0 8); do local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - stop_service "${SERVICE_NAME_NUM}" + stop_service "${SERVICE_NAME_NUM}" & + PIDS="${!} ${PIDS}" done + # SC2086 (info): Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + wait ${PIDS} prune_local_test_image "${IMAGE_WITH_TAG}" finalize_test "${TEST_NAME}" } BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_service_parallel_more_workers "${TEST_NAME}" "${SERVICE_NAME}" + When run test_parallel_more_workers "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -203,4 +213,45 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End End + Describe "test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number" "container_test:false" + TEST_NAME="test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number" + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") + SERVICE_NAME="gantry-test-$(unique_id)" + test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number() { + local TEST_NAME=${1} + local SERVICE_NAME=${2} + reset_gantry_env "${SERVICE_NAME}" + export GANTRY_UPDATE_NUM_WORKERS="NotANumber" + run_gantry "${TEST_NAME}" + } + BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number "${TEST_NAME}" "${SERVICE_NAME}" + The status should be failure + The stdout should satisfy display_output + The stderr should satisfy display_output + The stderr should satisfy spec_expect_message "GANTRY_UPDATE_NUM_WORKERS must be a number.*" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}" + The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${NO_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" + The stderr should satisfy spec_expect_message "${NO_IMAGES_TO_REMOVE}" + The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" + The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" + The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" + End + End End # Describe 'Multiple services' diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index ae70055..58f26f9 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -296,6 +296,7 @@ reset_gantry_env() { export GANTRY_UPDATE_JOBS= export GANTRY_UPDATE_OPTIONS= export GANTRY_UPDATE_TIMEOUT_SECONDS= + export GANTRY_UPDATE_NUM_WORKERS= export GANTRY_CLEANUP_IMAGES= export GANTRY_CLEANUP_IMAGES_OPTIONS= export GANTRY_CLEANUP_IMAGES_REMOVER=ghcr.io/shizunge/gantry-development @@ -626,6 +627,7 @@ _run_gantry_container() { --env "GANTRY_UPDATE_JOBS=${GANTRY_UPDATE_JOBS}" \ --env "GANTRY_UPDATE_OPTIONS=${GANTRY_UPDATE_OPTIONS}" \ --env "GANTRY_UPDATE_TIMEOUT_SECONDS=${GANTRY_UPDATE_TIMEOUT_SECONDS}" \ + --env "GANTRY_UPDATE_NUM_WORKERS=${GANTRY_UPDATE_NUM_WORKERS}" \ --env "GANTRY_CLEANUP_IMAGES=${GANTRY_CLEANUP_IMAGES}" \ --env "GANTRY_CLEANUP_IMAGES_OPTIONS=${GANTRY_CLEANUP_IMAGES_OPTIONS}" \ --env "GANTRY_CLEANUP_IMAGES_REMOVER=${GANTRY_CLEANUP_IMAGES_REMOVER}" \ From aece1eac19e7326255def3c6f87030c22bc16bee Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Sat, 17 Feb 2024 01:10:32 -0800 Subject: [PATCH 12/19] [tests] check exact number of services updated. --- tests/gantry_cleanup_images_spec.sh | 6 +++--- tests/gantry_common_options_spec.sh | 2 +- tests/gantry_login_spec.sh | 4 ++-- tests/gantry_manifest_spec.sh | 2 +- tests/gantry_notify_spec.sh | 4 ++-- tests/gantry_service_multiple_spec.sh | 2 +- tests/gantry_service_no_running_tasks_spec.sh | 4 ++-- tests/gantry_service_parallel_spec.sh | 4 ++-- tests/gantry_service_single_spec.sh | 4 ++-- tests/gantry_update_options_spec.sh | 6 +++--- tests/spec_gantry_test_helper.sh | 5 +++-- 11 files changed, 22 insertions(+), 21 deletions(-) diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh index c262f2d..2915dcf 100644 --- a/tests/gantry_cleanup_images_spec.sh +++ b/tests/gantry_cleanup_images_spec.sh @@ -50,7 +50,7 @@ Describe 'cleanup-images' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -92,7 +92,7 @@ Describe 'cleanup-images' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -135,7 +135,7 @@ Describe 'cleanup-images' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh index fae2619..9c57620 100644 --- a/tests/gantry_common_options_spec.sh +++ b/tests/gantry_common_options_spec.sh @@ -120,7 +120,7 @@ Describe 'common-options' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh index 0a1f521..9002fbd 100644 --- a/tests/gantry_login_spec.sh +++ b/tests/gantry_login_spec.sh @@ -76,7 +76,7 @@ Describe 'login' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -144,7 +144,7 @@ Describe 'login' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh index 7baafd7..a97b7a7 100644 --- a/tests/gantry_manifest_spec.sh +++ b/tests/gantry_manifest_spec.sh @@ -143,7 +143,7 @@ Describe 'manifest-command' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh index 716c15e..45afe42 100644 --- a/tests/gantry_notify_spec.sh +++ b/tests/gantry_notify_spec.sh @@ -81,7 +81,7 @@ Describe 'notify' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -123,7 +123,7 @@ Describe 'notify' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/gantry_service_multiple_spec.sh b/tests/gantry_service_multiple_spec.sh index 2594e56..fab8ca5 100644 --- a/tests/gantry_service_multiple_spec.sh +++ b/tests/gantry_service_multiple_spec.sh @@ -114,7 +114,7 @@ Describe 'service-multiple-services' The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME4}" The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME5}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "2 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh index 5b6d298..5864a5b 100644 --- a/tests/gantry_service_no_running_tasks_spec.sh +++ b/tests/gantry_service_no_running_tasks_spec.sh @@ -54,7 +54,7 @@ Describe "service-no-running-tasks" The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -111,7 +111,7 @@ Describe "service-no-running-tasks" The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/gantry_service_parallel_spec.sh b/tests/gantry_service_parallel_spec.sh index 9f60c0c..6659d1d 100644 --- a/tests/gantry_service_parallel_spec.sh +++ b/tests/gantry_service_parallel_spec.sh @@ -106,7 +106,7 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "7 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -203,7 +203,7 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "5 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh index fc4db8d..c9672f6 100644 --- a/tests/gantry_service_single_spec.sh +++ b/tests/gantry_service_single_spec.sh @@ -87,7 +87,7 @@ Describe 'service-single-service' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -136,7 +136,7 @@ Describe 'service-single-service' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh index c62af04..7f530e8 100644 --- a/tests/gantry_update_options_spec.sh +++ b/tests/gantry_update_options_spec.sh @@ -92,7 +92,7 @@ Describe 'update-options' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -140,7 +140,7 @@ Describe 'update-options' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -198,7 +198,7 @@ Describe 'update-options' The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index 58f26f9..0c4f03f 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -42,6 +42,7 @@ export FAILED_TO_ROLLBACK="Failed to roll back" export ROLLED_BACK="Rolled back" export NO_SERVICES_UPDATED="No services updated" export NUM_SERVICES_UPDATED="[1-9] service\(s\) updated" +export SERVICES_UPDATED="service\(s\) updated" export NUM_SERVICES_UPDATE_FAILED="[1-9] service\(s\) update failed" export NUM_SERVICES_ERRORS="Skip updating [1-9] service\(s\) due to error\(s\)" export NO_IMAGES_TO_REMOVE="No images to remove" @@ -294,9 +295,9 @@ reset_gantry_env() { export GANTRY_MANIFEST_OPTIONS= export GANTRY_ROLLBACK_OPTIONS= export GANTRY_UPDATE_JOBS= + export GANTRY_UPDATE_NUM_WORKERS= export GANTRY_UPDATE_OPTIONS= export GANTRY_UPDATE_TIMEOUT_SECONDS= - export GANTRY_UPDATE_NUM_WORKERS= export GANTRY_CLEANUP_IMAGES= export GANTRY_CLEANUP_IMAGES_OPTIONS= export GANTRY_CLEANUP_IMAGES_REMOVER=ghcr.io/shizunge/gantry-development @@ -625,9 +626,9 @@ _run_gantry_container() { --env "GANTRY_ROLLBACK_ON_FAILURE=${GANTRY_ROLLBACK_ON_FAILURE}" \ --env "GANTRY_ROLLBACK_OPTIONS=${GANTRY_ROLLBACK_OPTIONS}" \ --env "GANTRY_UPDATE_JOBS=${GANTRY_UPDATE_JOBS}" \ + --env "GANTRY_UPDATE_NUM_WORKERS=${GANTRY_UPDATE_NUM_WORKERS}" \ --env "GANTRY_UPDATE_OPTIONS=${GANTRY_UPDATE_OPTIONS}" \ --env "GANTRY_UPDATE_TIMEOUT_SECONDS=${GANTRY_UPDATE_TIMEOUT_SECONDS}" \ - --env "GANTRY_UPDATE_NUM_WORKERS=${GANTRY_UPDATE_NUM_WORKERS}" \ --env "GANTRY_CLEANUP_IMAGES=${GANTRY_CLEANUP_IMAGES}" \ --env "GANTRY_CLEANUP_IMAGES_OPTIONS=${GANTRY_CLEANUP_IMAGES_OPTIONS}" \ --env "GANTRY_CLEANUP_IMAGES_REMOVER=${GANTRY_CLEANUP_IMAGES_REMOVER}" \ From 21878c6116da98e800bda72eb79f26867fc7dcd2 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Sat, 17 Feb 2024 14:21:12 -0800 Subject: [PATCH 13/19] [gantry] update logging & reporting --- .github/workflows/on-pull-request.yml | 2 +- .github/workflows/on-push.yml | 2 +- src/entrypoint.sh | 9 +-- src/lib-gantry.sh | 63 +++++++++++++------ tests/gantry_common_options_spec.sh | 2 +- ...rallel_spec.sh => gantry_parallel_spec.sh} | 2 +- tests/gantry_update_options_spec.sh | 2 +- tests/spec_gantry_test_helper.sh | 3 +- 8 files changed, 54 insertions(+), 31 deletions(-) rename tests/{gantry_service_parallel_spec.sh => gantry_parallel_spec.sh} (99%) diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml index 2c13a7c..38ae7f6 100644 --- a/.github/workflows/on-pull-request.yml +++ b/.github/workflows/on-pull-request.yml @@ -35,10 +35,10 @@ jobs: - gantry_login_spec.sh - gantry_manifest_spec.sh - gantry_notify_spec.sh + - gantry_parallel_spec.sh - gantry_rollback_spec.sh - gantry_service_multiple_spec.sh - gantry_service_no_running_tasks_spec.sh - - gantry_service_parallel_spec.sh - gantry_service_single_spec.sh - gantry_update_options_spec.sh steps: diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 843408f..ceef5bd 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -42,10 +42,10 @@ jobs: - gantry_login_spec.sh - gantry_manifest_spec.sh - gantry_notify_spec.sh + - gantry_parallel_spec.sh - gantry_rollback_spec.sh - gantry_service_multiple_spec.sh - gantry_service_no_running_tasks_spec.sh - - gantry_service_parallel_spec.sh - gantry_service_single_spec.sh - gantry_update_options_spec.sh steps: diff --git a/src/entrypoint.sh b/src/entrypoint.sh index be52a14..3593d87 100755 --- a/src/entrypoint.sh +++ b/src/entrypoint.sh @@ -117,7 +117,7 @@ gantry() { gantry_update_services_list "${SERVICES_LIST}" ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?)) else - log WARN "Skip updating all services due to previous errors." + log WARN "Skip updating all services due to previous error(s)." fi local DOCKER_HUB_RATE_AFTER= @@ -149,12 +149,9 @@ main() { LOG_LEVEL="${GANTRY_LOG_LEVEL:-${LOG_LEVEL}}" NODE_NAME="${GANTRY_NODE_NAME:-${NODE_NAME}}" export LOG_LEVEL NODE_NAME + local INTERVAL_SECONDS= + INTERVAL_SECONDS=$(gantry_read_number GANTRY_SLEEP_SECONDS 0) || return 1 local IMAGES_TO_REMOVE="${GANTRY_IMAGES_TO_REMOVE:-""}" - local INTERVAL_SECONDS="${GANTRY_SLEEP_SECONDS:-0}" - if ! is_number "${INTERVAL_SECONDS}"; then - log ERROR "GANTRY_SLEEP_SECONDS must be a number. Got \"${GANTRY_SLEEP_SECONDS}\"." - return 1; - fi if [ -n "${IMAGES_TO_REMOVE}" ]; then # Image remover runs as a global job. The log will be collected via docker commands then formatted. # Redefine the log function for the formater. diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index 96b43b7..a8d3603 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -15,6 +15,25 @@ # along with this program. If not, see . # +# Read a number from an environment variable. Log an error is it is not a number. +gantry_read_number() { + local VNAME="${1}" + local DEFAULT_VALUE="${2}" + if ! is_number "${DEFAULT_VALUE}"; then + log ERROR "DEFAULT_VALUE must be a number. Got \"${DEFAULT_VALUE}\"." + return 1 + fi + local READ_VALUE VALUE + READ_VALUE=$(read_env "${VNAME}" "${DEFAULT_VALUE}") + VALUE="${READ_VALUE}" + [ -z "${VALUE}" ] && VALUE="${DEFAULT_VALUE}" + if ! is_number "${VALUE}"; then + log ERROR "${VNAME} must be a number. Got \"${READ_VALUE}\"." + return 1; + fi + echo "${VALUE}" +} + _login_registry() { local USER="${1}" local PASSWORD="${2}" @@ -83,7 +102,7 @@ _authenticate_to_registries() { USER= fi if [ "${ACCUMULATED_ERRORS}" -gt 0 ]; then - log ERROR "Skip logging in due to previous errors." + log ERROR "Skip logging in due to previous error(s)." else _login_registry "${USER}" "${PASSWORD}" "${HOST}" "${CONFIG}" ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?)) @@ -347,6 +366,11 @@ _get_number_of_elements_in_static_variable() { } _report_services() { + local STACK="${1:-gantry}" + # ACCUMULATED_ERRORS is the number of errors that are not caused by updating. + local ACCUMULATED_ERRORS="${2:-0}" + if ! is_number "${ACCUMULATED_ERRORS}"; then log WARN "ACCUMULATED_ERRORS \"${ACCUMULATED_ERRORS}\" is not a number." && ACCUMULATED_ERRORS=0; fi + local UPDATED_MSG= UPDATED_MSG=$(_report_services_from_static_variable STATIC_VAR_SERVICES_UPDATED "" "updated" "No services updated.") echo "${UPDATED_MSG}" | log_lines INFO @@ -360,17 +384,20 @@ _report_services() { echo "${ERROR_MSG}" | log_lines ERROR # Send notification - local UPDATED_NUM FAILED_NUM ERROR_NUM TOTAL_ERROR_NUM - UPDATED_NUM=$(_get_number_of_elements_in_static_variable STATIC_VAR_SERVICES_UPDATED) - FAILED_NUM=$(_get_number_of_elements_in_static_variable STATIC_VAR_SERVICES_UPDATE_FAILED) - ERROR_NUM=$(_get_number_of_elements_in_static_variable STATIC_VAR_SERVICES_UPDATE_INPUT_ERROR) - TOTAL_ERROR_NUM=$((FAILED_NUM+ERROR_NUM)) + local NUM_UPDATED NUM_FAILED NUM_ERRORS + NUM_UPDATED=$(_get_number_of_elements_in_static_variable STATIC_VAR_SERVICES_UPDATED) + NUM_FAILED=$(_get_number_of_elements_in_static_variable STATIC_VAR_SERVICES_UPDATE_FAILED) + NUM_ERRORS=$(_get_number_of_elements_in_static_variable STATIC_VAR_SERVICES_UPDATE_INPUT_ERROR) + if [ "${NUM_FAILED}" -eq 0 ] && [ "${NUM_ERRORS}" -eq 0 ]; then + NUM_ERRORS="${ACCUMULATED_ERRORS}" + fi + local NUM_TOTAL_ERRORS=$((NUM_FAILED+NUM_ERRORS)) local TYPE="success" - [ "${TOTAL_ERROR_NUM}" -ne "0" ] && TYPE="failure" + [ "${NUM_TOTAL_ERRORS}" -ne "0" ] && TYPE="failure" local ERROR_STRING= - [ "${ERROR_NUM}" -ne "0" ] && ERROR_STRING=" ${ERROR_NUM} errors" + [ "${NUM_ERRORS}" -ne "0" ] && ERROR_STRING=" ${NUM_TOTAL_ERRORS} error(s)" local TITLE BODY - TITLE="[gantry] ${UPDATED_NUM} services updated ${FAILED_NUM} failed${ERROR_STRING}" + TITLE="[${STACK}] ${NUM_UPDATED} services updated ${NUM_FAILED} failed${ERROR_STRING}" BODY=$(echo -e "${UPDATED_MSG}\n${FAILED_MSG}\n${ERROR_MSG}") _send_notification "${TYPE}" "${TITLE}" "${BODY}" } @@ -565,7 +592,6 @@ _get_image_info() { return 1 fi echo "${MSG}" - return 0 } # echo nothing if we found no new images. @@ -729,12 +755,12 @@ _rollback_service() { # return 0 when there is no error or failure. # return 1 when there are error(s) or failure(s). _update_single_service() { - local UPDATE_TIMEOUT_SECONDS="${GANTRY_UPDATE_TIMEOUT_SECONDS:-300}" + local INPUT_ERROR=0 + local UPDATE_TIMEOUT_SECONDS= + UPDATE_TIMEOUT_SECONDS=$(gantry_read_number GANTRY_UPDATE_TIMEOUT_SECONDS 300) || INPUT_ERROR=1 local UPDATE_OPTIONS="${GANTRY_UPDATE_OPTIONS:-""}" local SERVICE_NAME="${1}" local IMAGE="${2}" - local INPUT_ERROR=0 - if ! is_number "${UPDATE_TIMEOUT_SECONDS}"; then log ERROR "GANTRY_UPDATE_TIMEOUT_SECONDS must be a number. Got \"${GANTRY_UPDATE_TIMEOUT_SECONDS}\"."; INPUT_ERROR=1; fi [ -z "${SERVICE_NAME}" ] && log ERROR "Updating service: SERVICE_NAME must not be empty." && INPUT_ERROR=1 [ -z "${IMAGE}" ] && log ERROR "Updating ${SERVICE_NAME}: IMAGE must not be empty." && INPUT_ERROR=1 if [ "${INPUT_ERROR}" -ne "0" ]; then @@ -859,12 +885,9 @@ gantry_get_services_list() { } gantry_update_services_list() { - local NUM_WORKERS="${GANTRY_UPDATE_NUM_WORKERS:-1}" + local NUM_WORKERS= + NUM_WORKERS=$(gantry_read_number GANTRY_UPDATE_NUM_WORKERS 1) || return 1 local LIST="${*}" - if ! is_number "${NUM_WORKERS}"; then - log ERROR "GANTRY_UPDATE_NUM_WORKERS must be a number. Got \"${GANTRY_UPDATE_NUM_WORKERS}\"." - return 1; - fi local NUM= NUM=$(_get_number_of_elements "${LIST}") log INFO "Inspecting ${NUM} service(s)." @@ -885,6 +908,7 @@ gantry_update_services_list() { log DEBUG "NUM_WORKERS=${NUM_WORKERS}" local PIDS= for INDEX in $(seq 0 $((NUM_WORKERS-1)) ); do + # All workers subscribe to the same list now. _update_worker "${INDEX}" STATIC_VAR_SERVICES_AND_IMAGES_TO_UPDATE & PIDS="${!} ${PIDS}" done @@ -904,11 +928,12 @@ gantry_update_services_list() { gantry_finalize() { local STACK="${1:-gantry}" + local NUM_ERRORS="${2}" local RETURN_VALUE=0 if ! _remove_images "${STACK}_image-remover"; then RETURN_VALUE=1 fi - if ! _report_services; then + if ! _report_services "${STACK}" "${NUM_ERRORS}"; then RETURN_VALUE=1 fi if [ -n "${STATIC_VARIABLES_FOLDER}" ]; then diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh index 9c57620..c07f6e4 100644 --- a/tests/gantry_common_options_spec.sh +++ b/tests/gantry_common_options_spec.sh @@ -149,7 +149,7 @@ Describe 'common-options' The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "GANTRY_SLEEP_SECONDS must be a number.*" + The stderr should satisfy spec_expect_message "GANTRY_SLEEP_SECONDS ${MUST_BE_A_NUMBER}.*" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING_ALL}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" diff --git a/tests/gantry_service_parallel_spec.sh b/tests/gantry_parallel_spec.sh similarity index 99% rename from tests/gantry_service_parallel_spec.sh rename to tests/gantry_parallel_spec.sh index 6659d1d..06347d0 100644 --- a/tests/gantry_service_parallel_spec.sh +++ b/tests/gantry_parallel_spec.sh @@ -231,7 +231,7 @@ Describe 'service-parallel' The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "GANTRY_UPDATE_NUM_WORKERS must be a number.*" + The stderr should satisfy spec_expect_message "GANTRY_UPDATE_NUM_WORKERS ${MUST_BE_A_NUMBER}.*" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh index 7f530e8..a546c11 100644 --- a/tests/gantry_update_options_spec.sh +++ b/tests/gantry_update_options_spec.sh @@ -226,7 +226,7 @@ Describe 'update-options' The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "GANTRY_UPDATE_TIMEOUT_SECONDS must be a number.*" + The stderr should satisfy spec_expect_message "GANTRY_UPDATE_TIMEOUT_SECONDS ${MUST_BE_A_NUMBER}.*" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index 0c4f03f..5741050 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -16,9 +16,10 @@ # # Constant strings for checks. +export MUST_BE_A_NUMBER="must be a number" export SKIP_UPDATING_ALL="Skip updating all services" export SKIP_REASON_NOT_SWARM_MANAGER="is not a swarm manager" -export SKIP_REASON_PREVIOUS_ERRORS="due to previous errors" +export SKIP_REASON_PREVIOUS_ERRORS="due to previous error\(s\)" export SKIP_UPDATING="Skip updating" export SKIP_REASON_IS_JOB="because it is in .*job mode" export SKIP_REASON_NO_KNOWN_NEWER_IMAGE="because there is no known newer version of image" From 0bd5b6f4c29d70c551393e0e8b009f3b2bb4d8ed Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Sat, 17 Feb 2024 19:59:44 -0800 Subject: [PATCH 14/19] [tests] be able to handle more than 10 test services. --- tests/gantry_parallel_spec.sh | 65 +++++++++++--------------------- tests/spec_gantry_test_helper.sh | 16 ++++---- 2 files changed, 31 insertions(+), 50 deletions(-) diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh index 06347d0..d4146f1 100644 --- a/tests/gantry_parallel_spec.sh +++ b/tests/gantry_parallel_spec.sh @@ -82,24 +82,24 @@ Describe 'service-parallel' The stdout should satisfy display_output The stderr should satisfy display_output SERVICE_NAME_NUM="${SERVICE_NAME}-0" - The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}" - The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" - The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} " + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} .*${PERFORM_REASON_HAS_NEWER_IMAGE}" + The stderr should satisfy spec_expect_message "${UPDATED}.* ${SERVICE_NAME_NUM}" for NUM in $(seq 1 6); do SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}" - The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}.*${PERFORM_REASON_KNOWN_NEWER_IMAGE}" - The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} " + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} .*${PERFORM_REASON_KNOWN_NEWER_IMAGE}" + The stderr should satisfy spec_expect_message "${UPDATED}.* ${SERVICE_NAME_NUM}" done SERVICE_NAME_NUM="${SERVICE_NAME}-7" - The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}.*${SKIP_REASON_CURRENT_IS_LATEST}" - The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}" - The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} .*${SKIP_REASON_CURRENT_IS_LATEST}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} " + The stderr should satisfy spec_expect_no_message "${UPDATED}.* ${SERVICE_NAME_NUM}" for NUM in $(seq 8 9); do SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}.*${SKIP_REASON_NO_KNOWN_NEWER_IMAGE}" - The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}" - The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} .*${SKIP_REASON_NO_KNOWN_NEWER_IMAGE}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} " + The stderr should satisfy spec_expect_no_message "${UPDATED}.* ${SERVICE_NAME_NUM} " done The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" @@ -128,7 +128,7 @@ Describe 'service-parallel' build_and_push_test_image "${IMAGE_WITH_TAG}" local NUM= local PIDS= - for NUM in $(seq 0 4); do + for NUM in $(seq 0 10); do local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" & PIDS="${!} ${PIDS}" @@ -137,15 +137,6 @@ Describe 'service-parallel' # shellcheck disable=SC2086 wait ${PIDS} build_and_push_test_image "${IMAGE_WITH_TAG}" - PIDS= - for NUM in $(seq 5 8); do - local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" & - PIDS="${!} ${PIDS}" - done - # SC2086 (info): Double quote to prevent globbing and word splitting. - # shellcheck disable=SC2086 - wait ${PIDS} } test_parallel_more_workers() { local TEST_NAME=${1} @@ -160,7 +151,7 @@ Describe 'service-parallel' local SERVICE_NAME=${3} local NUM= local PIDS= - for NUM in $(seq 0 8); do + for NUM in $(seq 0 10); do local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" stop_service "${SERVICE_NAME_NUM}" & PIDS="${!} ${PIDS}" @@ -179,31 +170,21 @@ Describe 'service-parallel' The stdout should satisfy display_output The stderr should satisfy display_output SERVICE_NAME_NUM="${SERVICE_NAME}-0" - The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}" - The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" - The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME_NUM}" - for NUM in $(seq 1 4); do - SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}" - The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}.*${PERFORM_REASON_KNOWN_NEWER_IMAGE}" - The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME_NUM}" - done - SERVICE_NAME_NUM="${SERVICE_NAME}-5" - The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}.*${SKIP_REASON_CURRENT_IS_LATEST}" - The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}" - The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME_NUM}" - for NUM in $(seq 6 8); do + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} " + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} .*${PERFORM_REASON_HAS_NEWER_IMAGE}" + The stderr should satisfy spec_expect_message "${UPDATED}.* ${SERVICE_NAME_NUM}" + for NUM in $(seq 1 10); do SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME_NUM}.*${SKIP_REASON_NO_KNOWN_NEWER_IMAGE}" - The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_NUM}" - The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME_NUM}" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} " + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} .*${PERFORM_REASON_KNOWN_NEWER_IMAGE}" + The stderr should satisfy spec_expect_message "${UPDATED}.* ${SERVICE_NAME_NUM}" done The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" - The stderr should satisfy spec_expect_message "${NUM_SERVICES_NO_NEW_IMAGES}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "5 ${SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "11 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index 5741050..dba86a1 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -32,22 +32,22 @@ export PERFORM_REASON_DIGEST_IS_EMPTY="because DIGEST is empty" export PERFORM_REASON_HAS_NEWER_IMAGE="because there is a newer version" export IMAGE_NOT_EXIST="does not exist or it is not available" export ADDING_OPTIONS="Adding options" -export NUM_SERVICES_SKIP_JOBS="Skip updating [1-9] service\(s\) due to they are job\(s\)" -export NUM_SERVICES_INSPECT_FAILURE="Failed to inspect [1-9] service\(s\)" -export NUM_SERVICES_NO_NEW_IMAGES="No new images for [1-9] service\(s\)" -export NUM_SERVICES_UPDATING="Updating [1-9] service\(s\)" +export NUM_SERVICES_SKIP_JOBS="Skip updating [0-9]+ service\(s\) due to they are job\(s\)" +export NUM_SERVICES_INSPECT_FAILURE="Failed to inspect [0-9]+ service\(s\)" +export NUM_SERVICES_NO_NEW_IMAGES="No new images for [0-9]+ service\(s\)" +export NUM_SERVICES_UPDATING="Updating [0-9]+ service\(s\)" export NO_UPDATES="No updates" export UPDATED="UPDATED" export ROLLING_BACK="Rolling back" export FAILED_TO_ROLLBACK="Failed to roll back" export ROLLED_BACK="Rolled back" export NO_SERVICES_UPDATED="No services updated" -export NUM_SERVICES_UPDATED="[1-9] service\(s\) updated" +export NUM_SERVICES_UPDATED="[0-9]+ service\(s\) updated" export SERVICES_UPDATED="service\(s\) updated" -export NUM_SERVICES_UPDATE_FAILED="[1-9] service\(s\) update failed" -export NUM_SERVICES_ERRORS="Skip updating [1-9] service\(s\) due to error\(s\)" +export NUM_SERVICES_UPDATE_FAILED="[0-9]+ service\(s\) update failed" +export NUM_SERVICES_ERRORS="Skip updating [0-9]+ service\(s\) due to error\(s\)" export NO_IMAGES_TO_REMOVE="No images to remove" -export REMOVING_NUM_IMAGES="Removing [1-9] image\(s\)" +export REMOVING_NUM_IMAGES="Removing [0-9]+ image\(s\)" export SKIP_REMOVING_IMAGES="Skip removing images" export REMOVED_IMAGE="Removed image" export FAILED_TO_REMOVE_IMAGE="Failed to remove image" From 666a4e8c44a6f47767a23d1f2b8c8f7d78438b1e Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Sat, 17 Feb 2024 22:24:14 -0800 Subject: [PATCH 15/19] [gantry] fix getting number of running tasks and getting mode. --- src/lib-gantry.sh | 44 +++++++++++++----- tests/gantry_service_no_running_tasks_spec.sh | 45 ++++++++++++++++--- tests/gantry_update_options_spec.sh | 31 +++++++++++-- tests/spec_gantry_test_helper.sh | 7 ++- 4 files changed, 104 insertions(+), 23 deletions(-) diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index a8d3603..559eac1 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -388,7 +388,7 @@ _report_services() { NUM_UPDATED=$(_get_number_of_elements_in_static_variable STATIC_VAR_SERVICES_UPDATED) NUM_FAILED=$(_get_number_of_elements_in_static_variable STATIC_VAR_SERVICES_UPDATE_FAILED) NUM_ERRORS=$(_get_number_of_elements_in_static_variable STATIC_VAR_SERVICES_UPDATE_INPUT_ERROR) - if [ "${NUM_FAILED}" -eq 0 ] && [ "${NUM_ERRORS}" -eq 0 ]; then + if [ "${NUM_FAILED}" = "0" ] && [ "${NUM_ERRORS}" = "0" ]; then NUM_ERRORS="${ACCUMULATED_ERRORS}" fi local NUM_TOTAL_ERRORS=$((NUM_FAILED+NUM_ERRORS)) @@ -414,10 +414,16 @@ _in_list() { return 1 } +# echo the name of the current container. +# echo nothing if unable to find the name. +# return 1 when there is an error. _current_container_name() { local CURRENT_CONTAINER_NAME= CURRENT_CONTAINER_NAME=$(_static_variable_read_list STATIC_VAR_CURRENT_CONTAINER_NAME) [ -n "${CURRENT_CONTAINER_NAME}" ] && echo "${CURRENT_CONTAINER_NAME}" && return 0 + local NO_CURRENT_CONTAINER_NAME= + NO_CURRENT_CONTAINER_NAME=$(_static_variable_read_list STATIC_VAR_NO_CURRENT_CONTAINER_NAME) + [ -n "${NO_CURRENT_CONTAINER_NAME}" ] && return 0 local ALL_NETWORKS= ALL_NETWORKS=$(docker network ls --format '{{.ID}}') || return 1; [ -z "${ALL_NETWORKS}" ] && return 0; @@ -453,6 +459,8 @@ _current_container_name() { done done done + # Explicitly set that we cannot find the name of current container. + _static_variable_add_unique_to_list STATIC_VAR_NO_CURRENT_CONTAINER_NAME "NO_CURRENT_CONTAINER_NAME" return 0; } @@ -464,7 +472,9 @@ gantry_current_service_name() { CNAME=$(_current_container_name) || return 1 [ -z "${CNAME}" ] && return 0 local SNAME= - SNAME=$(docker container inspect "${CNAME}" --format '{{range $key,$value := .Config.Labels}}{{$key}}={{println $value}}{{end}}' | grep "com.docker.swarm.service.name" | sed "s/com.docker.swarm.service.name=\(.*\)$/\1/") || return 1 + SNAME=$(docker container inspect "${CNAME}" --format '{{range $key,$value := .Config.Labels}}{{$key}}={{println $value}}{{end}}' \ + | grep "com.docker.swarm.service.name" \ + | sed -n "s/com.docker.swarm.service.name=\(.*\)$/\1/p") || return 1 _static_variable_add_unique_to_list STATIC_VAR_CURRENT_SERVICE_NAME "${SNAME}" echo "${SNAME}" } @@ -501,10 +511,15 @@ _get_service_previous_image() { _get_service_mode() { local SERVICE_NAME="${1}" local MODE= - if ! MODE=$(docker service ls --filter "name=${SERVICE_NAME}" --format '{{.Mode}}' 2>&1); then + if ! MODE=$(docker service ls --filter "name=${SERVICE_NAME}" --format '{{.Mode}} {{.Name}}' 2>&1); then log ERROR "Failed to obtain the mode of the service ${SERVICE_NAME}: ${MODE}" return 1 fi + # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name + # See https://docs.docker.com/engine/reference/commandline/service_ls/#name + # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985 + # We do an extra step to to perform the exact match. + MODE=$(echo "${MODE}" | sed -n "s/\(.*\) ${SERVICE_NAME}$/\1/p") echo "${MODE}" } @@ -693,10 +708,15 @@ _inspect_service() { _get_number_of_running_tasks() { local SERVICE_NAME="${1}" local REPLICAS= - if ! REPLICAS=$(docker service ls --filter "name=${SERVICE_NAME}" --format '{{.Replicas}}' 2>&1); then + if ! REPLICAS=$(docker service ls --filter "name=${SERVICE_NAME}" --format '{{.Replicas}} {{.Name}}' 2>&1); then log ERROR "Failed to obtain task states of service ${SERVICE_NAME}: ${REPLICAS}" return 1 fi + # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name + # See https://docs.docker.com/engine/reference/commandline/service_ls/#name + # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985 + # We do an extra step to to perform the exact match. + REPLICAS=$(echo "${REPLICAS}" | sed -n "s/\(.*\) ${SERVICE_NAME}$/\1/p") # https://docs.docker.com/engine/reference/commandline/service_ls/#examples # The REPLICAS is like "5/5" or "1/1 (3/5 completed)" # Get the number before the first "/". @@ -714,7 +734,7 @@ _get_service_update_additional_options() { return 1 fi local OPTIONS= - if [ "${NUM_RUNS}" -eq 0 ]; then + if [ "${NUM_RUNS}" = "0" ]; then # Add "--detach=true" when there is no running tasks. # https://github.com/docker/cli/issues/627 OPTIONS="${OPTIONS} --detach=true" @@ -739,8 +759,8 @@ _rollback_service() { return 0 fi log INFO "Rolling back ${SERVICE_NAME}." - [ -n "${ADDITIONAL_OPTIONS}" ] && log DEBUG "Adding options \"${ADDITIONAL_OPTIONS}\" to the command \"docker service update --rollback\"." - [ -n "${ROLLBACK_OPTIONS}" ] && log DEBUG "Adding options \"${ROLLBACK_OPTIONS}\" to the command \"docker service update --rollback\"." + [ -n "${ADDITIONAL_OPTIONS}" ] && log DEBUG "Adding options \"${ADDITIONAL_OPTIONS}\" to the command \"docker service update --rollback\" for ${SERVICE_NAME}." + [ -n "${ROLLBACK_OPTIONS}" ] && log DEBUG "Adding options \"${ROLLBACK_OPTIONS}\" to the command \"docker service update --rollback\" for ${SERVICE_NAME}." local ROLLBACK_MSG= # Add "-quiet" to suppress progress output. # SC2086: Double quote to prevent globbing and word splitting. @@ -772,9 +792,9 @@ _update_single_service() { local ADDITIONAL_OPTIONS= DOCKER_CONFIG=$(_get_config_from_service "${SERVICE}") ADDITIONAL_OPTIONS=$(_get_service_update_additional_options "${SERVICE_NAME}") - [ -n "${DOCKER_CONFIG}" ] && log DEBUG "Adding options \"${DOCKER_CONFIG}\" to docker commands." - [ -n "${ADDITIONAL_OPTIONS}" ] && log DEBUG "Adding options \"${ADDITIONAL_OPTIONS}\" to the command \"docker service update\"." - [ -n "${UPDATE_OPTIONS}" ] && log DEBUG "Adding options \"${UPDATE_OPTIONS}\" to the command \"docker service update\"." + [ -n "${DOCKER_CONFIG}" ] && log DEBUG "Adding options \"${DOCKER_CONFIG}\" to docker commands for ${SERVICE_NAME}." + [ -n "${ADDITIONAL_OPTIONS}" ] && log DEBUG "Adding options \"${ADDITIONAL_OPTIONS}\" to the command \"docker service update\" for ${SERVICE_NAME}." + [ -n "${UPDATE_OPTIONS}" ] && log DEBUG "Adding options \"${UPDATE_OPTIONS}\" to the command \"docker service update\" for ${SERVICE_NAME}." local UPDATE_MSG= # Add "-quiet" to suppress progress output. # SC2086: Double quote to prevent globbing and word splitting. @@ -871,8 +891,8 @@ gantry_get_services_list() { continue fi # Add self to the first of the list. - if _service_is_self "${S}"; then - HAS_SELF=${S} + if [ -z "${HAS_SELF}" ] && _service_is_self "${S}"; then + HAS_SELF="${S}" continue fi LIST="${LIST} ${S}" diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh index 5864a5b..c34a895 100644 --- a/tests/gantry_service_no_running_tasks_spec.sh +++ b/tests/gantry_service_no_running_tasks_spec.sh @@ -20,19 +20,48 @@ Describe "service-no-running-tasks" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_no_running_tasks_replicated" "container_test:true" + # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name + # See https://docs.docker.com/engine/reference/commandline/service_ls/#name + # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985 + # This test also checks whether we do an extra step to to perform the exact match. TEST_NAME="test_no_running_tasks_replicated" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix" + test_start() { + local TEST_NAME="${1}" + local IMAGE_WITH_TAG="${2}" + local SERVICE_NAME="${3}" + local SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix" + initialize_test "${TEST_NAME}" + build_and_push_test_image "${IMAGE_WITH_TAG}" + start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" + start_replicated_service "${SERVICE_NAME_SUFFIX}" "${IMAGE_WITH_TAG}" + build_and_push_test_image "${IMAGE_WITH_TAG}" + } test_no_running_tasks_replicated() { local TEST_NAME=${1} local SERVICE_NAME=${2} + local SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix" + # Set running tasks to 0 for SERVICE_NAME. + # But keep tasks running for SERVICE_NAME_SUFFIX. docker service update --quiet --replicas=0 "${SERVICE_NAME}" wait_zero_running_tasks "${SERVICE_NAME}" reset_gantry_env "${SERVICE_NAME}" run_gantry "${TEST_NAME}" } - BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + test_end() { + local TEST_NAME="${1}" + local IMAGE_WITH_TAG="${2}" + local SERVICE_NAME="${3}" + local SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix" + stop_service "${SERVICE_NAME}" + stop_service "${SERVICE_NAME_SUFFIX}" + prune_local_test_image "${IMAGE_WITH_TAG}" + finalize_test "${TEST_NAME}" + } + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' When run test_no_running_tasks_replicated "${TEST_NAME}" "${SERVICE_NAME}" The status should be success @@ -40,10 +69,14 @@ Describe "service-no-running-tasks" The stderr should satisfy display_output # Add "--detach=true" when there is no running tasks. # https://github.com/docker/cli/issues/627 - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--replicas=0" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}\." + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--replicas=0.*${SERVICE_NAME}\." + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME_SUFFIX}\." + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--replicas=0.*${SERVICE_NAME_SUFFIX}\." The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME_SUFFIX}" + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME_SUFFIX}.*${PERFORM_REASON_KNOWN_NEWER_IMAGE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" @@ -54,7 +87,7 @@ Describe "service-no-running-tasks" The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "1 ${SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "2 ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -97,7 +130,7 @@ Describe "service-no-running-tasks" # Add "--detach=true" when there is no running tasks. # https://github.com/docker/cli/issues/627 The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true" - # Cannot add "--replicas" to global + # Cannot add "--replicas" to global mode The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--replicas=0" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh index a546c11..5b147af 100644 --- a/tests/gantry_update_options_spec.sh +++ b/tests/gantry_update_options_spec.sh @@ -20,28 +20,51 @@ Describe 'update-options' BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" Describe "test_update_jobs_skipping" "container_test:true" + # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name + # See https://docs.docker.com/engine/reference/commandline/service_ls/#name + # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985 + # This test also checks whether we do an extra step to to perform the exact match. TEST_NAME="test_update_jobs_skipping" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix" + test_start() { + local TEST_NAME="${1}" + local IMAGE_WITH_TAG="${2}" + local SERVICE_NAME="${3}" + local SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix" + common_setup_job "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME_SUFFIX}" + start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" + } test_update_jobs_skipping() { local TEST_NAME=${1} local SERVICE_NAME=${2} reset_gantry_env "${SERVICE_NAME}" run_gantry "${TEST_NAME}" } - BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + test_end() { + local TEST_NAME="${1}" + local IMAGE_WITH_TAG="${2}" + local SERVICE_NAME="${3}" + local SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix" + stop_service "${SERVICE_NAME}" + common_cleanup "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME_SUFFIX}" + } + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' When run test_update_jobs_skipping "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output # Check whether it is a job before checking whether there is a new image. - The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_IS_JOB}" + The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME_SUFFIX}.*${SKIP_REASON_IS_JOB}" + The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_SUFFIX}" + The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_CURRENT_IS_LATEST}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_SKIP_JOBS}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}" The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index dba86a1..1cdc9af 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -431,10 +431,15 @@ wait_zero_running_tasks() { _handle_failure "Services ${SERVICE_NAME} does not stop after ${TIMEOUT_SECONDS} seconds." return 1 fi - if ! REPLICAS=$(docker service ls --filter "name=${SERVICE_NAME}" --format '{{.Replicas}}' 2>&1); then + if ! REPLICAS=$(docker service ls --filter "name=${SERVICE_NAME}" --format '{{.Replicas}} {{.Name}}' 2>&1); then _handle_failure "Failed to obtain task states of service ${SERVICE_NAME}: ${REPLICAS}" return 1 fi + # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name + # See https://docs.docker.com/engine/reference/commandline/service_ls/#name + # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985 + # We do an extra step to to perform the exact match. + REPLICAS=$(echo "${REPLICAS}" | sed -n "s/\(.*\) ${SERVICE_NAME}$/\1/p") if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then echo "wait_zero_running_tasks Reach MAX_RETRIES ${MAX_RETRIES}" >&2 return 1 From 06b2d8e86751c2257fb6713f1b47c0ac8ac317a2 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Sat, 17 Feb 2024 23:05:53 -0800 Subject: [PATCH 16/19] [tests] add test_SERVICES_EXCLUDED_multiple_services --- tests/gantry_filters_spec.sh | 68 +++++++-------------- tests/gantry_parallel_spec.sh | 100 +++++++------------------------ tests/spec_gantry_test_helper.sh | 39 ++++++++++++ 3 files changed, 82 insertions(+), 125 deletions(-) diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh index 708ef6b..18bd98e 100644 --- a/tests/gantry_filters_spec.sh +++ b/tests/gantry_filters_spec.sh @@ -61,62 +61,34 @@ Describe 'filters' The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" End End - Describe "test_SERVICES_EXCLUDED" "container_test:true" - TEST_NAME="test_SERVICES_EXCLUDED" + Describe "test_SERVICES_EXCLUDED_multiple_services" "container_test:true" + TEST_NAME="test_SERVICES_EXCLUDED_multiple_services" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - test_SERVICES_EXCLUDED() { + MAX_SERVICES_NUM=10 + test_SERVICES_EXCLUDED_multiple_services() { local TEST_NAME=${1} local SERVICE_NAME=${2} + local MAX_SERVICES_NUM=${3} + local NUM_SERVICES_EXCLUDED=$((MAX_SERVICES_NUM/2)) + local NUM_SERVICES_EXCLUDED_FILTER_START=$((NUM_SERVICES_EXCLUDED+1)) reset_gantry_env "${SERVICE_NAME}" - export GANTRY_SERVICES_EXCLUDED="${SERVICE_NAME}" + for NUM in $(seq 0 "${NUM_SERVICES_EXCLUDED}"); do + local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + export GANTRY_SERVICES_EXCLUDED="${GANTRY_SERVICES_EXCLUDED} ${SERVICE_NAME_NUM}" + done + local LABEL="gantry.test" + for NUM in $(seq "${NUM_SERVICES_EXCLUDED_FILTER_START}" "${MAX_SERVICES_NUM}"); do + local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + docker service update --quiet --label-add "${LABEL}=true" "${SERVICE_NAME_NUM}" + done + export GANTRY_SERVICES_EXCLUDED_FILTERS="label=${LABEL}=true" run_gantry "${TEST_NAME}" } - BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - It 'run_test' - When run test_SERVICES_EXCLUDED "${TEST_NAME}" "${SERVICE_NAME}" - The status should be success - The stdout should satisfy display_output - The stderr should satisfy display_output - The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING_ALL}" - The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" - The stderr should satisfy spec_expect_message "${NO_IMAGES_TO_REMOVE}" - The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" - The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" - The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" - End - End - Describe "test_SERVICES_EXCLUDED_FILTERS" "container_test:true" - TEST_NAME="test_SERVICES_EXCLUDED_FILTERS" - IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") - SERVICE_NAME="gantry-test-$(unique_id)" - test_SERVICES_EXCLUDED_FILTERS() { - local TEST_NAME=${1} - local SERVICE_NAME=${2} - reset_gantry_env "${SERVICE_NAME}" - export GANTRY_SERVICES_EXCLUDED_FILTERS="name=${SERVICE_NAME}" - run_gantry "${TEST_NAME}" - } - BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + BeforeEach "common_setup_new_image_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}" + AfterEach "common_cleanup_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}" It 'run_test' - When run test_SERVICES_EXCLUDED_FILTERS "${TEST_NAME}" "${SERVICE_NAME}" + When run test_SERVICES_EXCLUDED_multiple_services "${TEST_NAME}" "${SERVICE_NAME}" "${MAX_SERVICES_NUM}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh index d4146f1..44a146b 100644 --- a/tests/gantry_parallel_spec.sh +++ b/tests/gantry_parallel_spec.sh @@ -23,25 +23,20 @@ Describe 'service-parallel' TEST_NAME="test_parallel_less_workers" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + MAX_SERVICES_NUM=6 + MAX_NO_NEW_IMAGE=3 test_start() { local TEST_NAME=${1} local IMAGE_WITH_TAG=${2} local SERVICE_NAME=${3} - initialize_test "${TEST_NAME}" - build_and_push_test_image "${IMAGE_WITH_TAG}" + local MAX_SERVICES_NUM=${4} + local MAX_NO_NEW_IMAGE=${5} + common_setup_new_image_multiple "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME}" "${MAX_SERVICES_NUM}" + local NO_NEW_IAMGE_START=$((MAX_SERVICES_NUM+1)) + local NO_NEW_IAMGE_END=$((MAX_SERVICES_NUM+MAX_NO_NEW_IMAGE)) local NUM= local PIDS= - for NUM in $(seq 0 6); do - local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" & - PIDS="${!} ${PIDS}" - done - # SC2086 (info): Double quote to prevent globbing and word splitting. - # shellcheck disable=SC2086 - wait ${PIDS} - build_and_push_test_image "${IMAGE_WITH_TAG}" - PIDS= - for NUM in $(seq 7 9); do + for NUM in $(seq "${NO_NEW_IAMGE_START}" "${NO_NEW_IAMGE_END}"); do local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" & PIDS="${!} ${PIDS}" @@ -53,31 +48,15 @@ Describe 'service-parallel' test_parallel_less_workers() { local TEST_NAME=${1} local SERVICE_NAME=${2} + local MAX_SERVICES_NUM=${3} reset_gantry_env "${SERVICE_NAME}" - export GANTRY_UPDATE_NUM_WORKERS=5 + export GANTRY_UPDATE_NUM_WORKERS=$((MAX_SERVICES_NUM/2+1)) run_gantry "${TEST_NAME}" } - test_end() { - local TEST_NAME=${1} - local IMAGE_WITH_TAG=${2} - local SERVICE_NAME=${3} - local NUM= - local PIDS= - for NUM in $(seq 0 9); do - local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - stop_service "${SERVICE_NAME_NUM}" & - PIDS="${!} ${PIDS}" - done - # SC2086 (info): Double quote to prevent globbing and word splitting. - # shellcheck disable=SC2086 - wait ${PIDS} - prune_local_test_image "${IMAGE_WITH_TAG}" - finalize_test "${TEST_NAME}" - } - BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM} ${MAX_NO_NEW_IMAGE}" + AfterEach "common_cleanup_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} $((MAX_SERVICES_NUM+MAX_NO_NEW_IMAGE))" It 'run_test' - When run test_parallel_less_workers "${TEST_NAME}" "${SERVICE_NAME}" + When run test_parallel_less_workers "${TEST_NAME}" "${SERVICE_NAME}" "${MAX_SERVICES_NUM}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -85,17 +64,17 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} " The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} .*${PERFORM_REASON_HAS_NEWER_IMAGE}" The stderr should satisfy spec_expect_message "${UPDATED}.* ${SERVICE_NAME_NUM}" - for NUM in $(seq 1 6); do + for NUM in $(seq 1 "${MAX_SERVICES_NUM}"); do SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} " The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} .*${PERFORM_REASON_KNOWN_NEWER_IMAGE}" The stderr should satisfy spec_expect_message "${UPDATED}.* ${SERVICE_NAME_NUM}" done - SERVICE_NAME_NUM="${SERVICE_NAME}-7" + SERVICE_NAME_NUM="${SERVICE_NAME}-$((MAX_SERVICES_NUM+1))" The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} .*${SKIP_REASON_CURRENT_IS_LATEST}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} " The stderr should satisfy spec_expect_no_message "${UPDATED}.* ${SERVICE_NAME_NUM}" - for NUM in $(seq 8 9); do + for NUM in $(seq "$((MAX_SERVICES_NUM+2))" "$((MAX_SERVICES_NUM+MAX_NO_NEW_IMAGE))"); do SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} .*${SKIP_REASON_NO_KNOWN_NEWER_IMAGE}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} " @@ -120,52 +99,19 @@ Describe 'service-parallel' TEST_NAME="test_parallel_more_workers" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" - test_start() { - local TEST_NAME=${1} - local IMAGE_WITH_TAG=${2} - local SERVICE_NAME=${3} - initialize_test "${TEST_NAME}" - build_and_push_test_image "${IMAGE_WITH_TAG}" - local NUM= - local PIDS= - for NUM in $(seq 0 10); do - local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" & - PIDS="${!} ${PIDS}" - done - # SC2086 (info): Double quote to prevent globbing and word splitting. - # shellcheck disable=SC2086 - wait ${PIDS} - build_and_push_test_image "${IMAGE_WITH_TAG}" - } + MAX_SERVICES_NUM=10 test_parallel_more_workers() { local TEST_NAME=${1} local SERVICE_NAME=${2} + local MAX_SERVICES_NUM=${3} reset_gantry_env "${SERVICE_NAME}" - export GANTRY_UPDATE_NUM_WORKERS=50 + export GANTRY_UPDATE_NUM_WORKERS=$((MAX_SERVICES_NUM*3)) run_gantry "${TEST_NAME}" } - test_end() { - local TEST_NAME=${1} - local IMAGE_WITH_TAG=${2} - local SERVICE_NAME=${3} - local NUM= - local PIDS= - for NUM in $(seq 0 10); do - local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" - stop_service "${SERVICE_NAME_NUM}" & - PIDS="${!} ${PIDS}" - done - # SC2086 (info): Double quote to prevent globbing and word splitting. - # shellcheck disable=SC2086 - wait ${PIDS} - prune_local_test_image "${IMAGE_WITH_TAG}" - finalize_test "${TEST_NAME}" - } - BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" - AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + BeforeEach "common_setup_new_image_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}" + AfterEach "common_cleanup_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}" It 'run_test' - When run test_parallel_more_workers "${TEST_NAME}" "${SERVICE_NAME}" + When run test_parallel_more_workers "${TEST_NAME}" "${SERVICE_NAME}" "${MAX_SERVICES_NUM}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -173,7 +119,7 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} " The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} .*${PERFORM_REASON_HAS_NEWER_IMAGE}" The stderr should satisfy spec_expect_message "${UPDATED}.* ${SERVICE_NAME_NUM}" - for NUM in $(seq 1 10); do + for NUM in $(seq 1 "${MAX_SERVICES_NUM}"); do SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} " The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.* ${SERVICE_NAME_NUM} .*${PERFORM_REASON_KNOWN_NEWER_IMAGE}" diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index 1cdc9af..e5ecb87 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -67,6 +67,26 @@ common_setup_new_image() { build_and_push_test_image "${IMAGE_WITH_TAG}" } +common_setup_new_image_multiple() { + local TEST_NAME="${1}" + local IMAGE_WITH_TAG="${2}" + local SERVICE_NAME="${3}" + local MAX_SERVICES_NUM="${4}" + initialize_test "${TEST_NAME}" + build_and_push_test_image "${IMAGE_WITH_TAG}" + local NUM= + local PIDS= + for NUM in $(seq 0 "${MAX_SERVICES_NUM}"); do + local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" & + PIDS="${!} ${PIDS}" + done + # SC2086 (info): Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + wait ${PIDS} + build_and_push_test_image "${IMAGE_WITH_TAG}" +} + common_setup_no_new_image() { local TEST_NAME="${1}" local IMAGE_WITH_TAG="${2}" @@ -112,6 +132,25 @@ common_cleanup() { finalize_test "${TEST_NAME}" } +common_cleanup_multiple() { + local TEST_NAME="${1}" + local IMAGE_WITH_TAG="${2}" + local SERVICE_NAME="${3}" + local MAX_SERVICES_NUM="${4}" + local NUM= + local PIDS= + for NUM in $(seq 0 "${MAX_SERVICES_NUM}"); do + local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}" + stop_service "${SERVICE_NAME_NUM}" & + PIDS="${!} ${PIDS}" + done + # SC2086 (info): Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + wait ${PIDS} + prune_local_test_image "${IMAGE_WITH_TAG}" + finalize_test "${TEST_NAME}" +} + spec_expect_message() { _expect_message "${spec_expect_message:-""}" "${1}" } From 44febdcb859e2d3063ced9c8ce7c85be2f0e8894 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Sun, 18 Feb 2024 00:32:55 -0800 Subject: [PATCH 17/19] [shellcheck] disable SC2317 for tests shellcheck incorrectly fired this checks on only one test: test_no_running_tasks_replicated --- tests/.shellcheckrc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/.shellcheckrc b/tests/.shellcheckrc index e09e775..1b94c5c 100644 --- a/tests/.shellcheckrc +++ b/tests/.shellcheckrc @@ -36,3 +36,6 @@ disable=SC3057 # SC2154 (warning): VAR is referenced but not assigned. disable=SC2154 + +# SC2317 (info): Command appears to be unreachable. Check usage (or ignore if invoked indirectly). +disable=SC2317 \ No newline at end of file From 7b8ee71f3e80152d5bf158347d09854e8ba5faa1 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Sun, 18 Feb 2024 00:49:44 -0800 Subject: [PATCH 18/19] [gantry] fix logging about GANTRY_SERVICES_SELF; improve reporting not-a-number errors. --- src/lib-gantry.sh | 22 +++++++++++++++------- tests/gantry_parallel_spec.sh | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index 559eac1..1562f32 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -482,12 +482,12 @@ gantry_current_service_name() { _service_is_self() { if [ -z "${GANTRY_SERVICES_SELF}" ]; then # If _service_is_self is called inside a subprocess, export won't affect the parent process. - # We only want to log it once, thus try to read the value from a static variable firstly. - # The static variable should be set inside the function gantry_current_service_name. If it is set, skip logging. - GANTRY_SERVICES_SELF=$(_static_variable_read_list STATIC_VAR_CURRENT_SERVICE_NAME) + # Use a static variable to preserve the value between processes. And we only want to log the value is set once. + GANTRY_SERVICES_SELF=$(_static_variable_read_list STATIC_VAR_SERVICES_SELF) if [ -z "${GANTRY_SERVICES_SELF}" ]; then GANTRY_SERVICES_SELF=$(gantry_current_service_name) export GANTRY_SERVICES_SELF + _static_variable_add_unique_to_list STATIC_VAR_SERVICES_SELF "${GANTRY_SERVICES_SELF}" [ -n "${GANTRY_SERVICES_SELF}" ] && log INFO "Set GANTRY_SERVICES_SELF to ${GANTRY_SERVICES_SELF}." fi fi @@ -775,13 +775,17 @@ _rollback_service() { # return 0 when there is no error or failure. # return 1 when there are error(s) or failure(s). _update_single_service() { - local INPUT_ERROR=0 local UPDATE_TIMEOUT_SECONDS= - UPDATE_TIMEOUT_SECONDS=$(gantry_read_number GANTRY_UPDATE_TIMEOUT_SECONDS 300) || INPUT_ERROR=1 + if ! UPDATE_TIMEOUT_SECONDS=$(gantry_read_number GANTRY_UPDATE_TIMEOUT_SECONDS 300); then + local ERROR_SERVICE="GANTRY_UPDATE_TIMEOUT_SECONDS-is-not-a-number" + _static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATE_INPUT_ERROR "${ERROR_SERVICE}" + return 1 + fi local UPDATE_OPTIONS="${GANTRY_UPDATE_OPTIONS:-""}" local SERVICE_NAME="${1}" local IMAGE="${2}" - [ -z "${SERVICE_NAME}" ] && log ERROR "Updating service: SERVICE_NAME must not be empty." && INPUT_ERROR=1 + local INPUT_ERROR=0 + [ -z "${SERVICE_NAME}" ] && log ERROR "Updating service: SERVICE_NAME must not be empty." && INPUT_ERROR=1 && SERVICE_NAME="unknown-service-name" [ -z "${IMAGE}" ] && log ERROR "Updating ${SERVICE_NAME}: IMAGE must not be empty." && INPUT_ERROR=1 if [ "${INPUT_ERROR}" -ne "0" ]; then _static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATE_INPUT_ERROR "${SERVICE_NAME}" @@ -906,7 +910,11 @@ gantry_get_services_list() { gantry_update_services_list() { local NUM_WORKERS= - NUM_WORKERS=$(gantry_read_number GANTRY_UPDATE_NUM_WORKERS 1) || return 1 + if ! NUM_WORKERS=$(gantry_read_number GANTRY_UPDATE_NUM_WORKERS 1); then + local ERROR_SERVICE="GANTRY_UPDATE_NUM_WORKERS-is-not-a-number" + _static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATE_INPUT_ERROR "${ERROR_SERVICE}" + return 1 + fi local LIST="${*}" local NUM= NUM=$(_get_number_of_elements "${LIST}") diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh index 44a146b..d1af52e 100644 --- a/tests/gantry_parallel_spec.sh +++ b/tests/gantry_parallel_spec.sh @@ -173,7 +173,7 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_message "${NO_SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" - The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_message "${NO_IMAGES_TO_REMOVE}" The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}" The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" From c1a21ba7435355624c46e3b2ab9540be940ae314 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Sun, 18 Feb 2024 08:39:49 -0800 Subject: [PATCH 19/19] [tests] update checks for ADDING_OPTIONS. fix using subprocess variables in login tests. --- src/lib-gantry.sh | 2 +- tests/gantry_cleanup_images_spec.sh | 4 +- tests/gantry_login_spec.sh | 51 +++++++++++-------- tests/gantry_manifest_spec.sh | 6 +-- tests/gantry_parallel_spec.sh | 4 +- tests/gantry_rollback_spec.sh | 2 +- tests/gantry_service_no_running_tasks_spec.sh | 6 +-- tests/gantry_update_options_spec.sh | 6 +-- 8 files changed, 44 insertions(+), 37 deletions(-) diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index 1562f32..8bd8473 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -651,7 +651,7 @@ _inspect_image() { fi local DOCKER_CONFIG= DOCKER_CONFIG=$(_get_config_from_service "${SERVICE}") - [ -n "${DOCKER_CONFIG}" ] && log DEBUG "Adding options \"${DOCKER_CONFIG}\" to docker commands." + [ -n "${DOCKER_CONFIG}" ] && log DEBUG "Adding options \"${DOCKER_CONFIG}\" to docker commands for ${SERVICE_NAME}." local IMAGE_INFO= if ! IMAGE_INFO=$(_get_image_info "${MANIFEST_CMD}" "${IMAGE}" "${DOCKER_CONFIG}"); then log DEBUG "Skip updating ${SERVICE_NAME} because there is a failure to obtain the manifest from the registry of image ${IMAGE}." diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh index 2915dcf..a7a1a32 100644 --- a/tests/gantry_cleanup_images_spec.sh +++ b/tests/gantry_cleanup_images_spec.sh @@ -98,7 +98,7 @@ Describe 'cleanup-images' The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--incorrect-option" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--incorrect-option.*" The stderr should satisfy spec_expect_message "Failed.*--incorrect-option" The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" @@ -141,7 +141,7 @@ Describe 'cleanup-images' The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--container-label=test" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--container-label=test.*" The stderr should satisfy spec_expect_no_message "Failed.*--container-label=test" The stderr should satisfy spec_expect_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh index 9002fbd..6e1341a 100644 --- a/tests/gantry_login_spec.sh +++ b/tests/gantry_login_spec.sh @@ -24,19 +24,21 @@ Describe 'login' TEST_NAME="test_login_config" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_config() { local TEST_NAME=${1} local SERVICE_NAME=${2} - local REGISTRY=${3} - local USERNAME=${4} - local PASSWORD=${5} + local CONFIG=${3} + local REGISTRY=${4} + local USERNAME=${5} + local PASSWORD=${6} if [ -z "${REGISTRY}" ] || [ -z "${USERNAME}" ] || [ -z "${PASSWORD}" ]; then echo "No REGISTRY, USERNAME or PASSWORD provided." >&2 return 1 fi local LABEL="gantry.auth.config" - CONFIG="C$(unique_id)" + local USER_FILE PASS_FILE USER_FILE=$(mktemp) PASS_FILE=$(mktemp) docker service update --quiet --label-add "${LABEL}=${CONFIG}" "${SERVICE_NAME}" @@ -58,7 +60,7 @@ Describe 'login' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_config "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -69,7 +71,7 @@ Describe 'login' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${CONFIG}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -90,19 +92,21 @@ Describe 'login' TEST_NAME="test_login_REGISTRY_CONFIGS_FILE" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_REGISTRY_CONFIGS_FILE() { local TEST_NAME=${1} local SERVICE_NAME=${2} - local REGISTRY=${3} - local USERNAME=${4} - local PASSWORD=${5} + local CONFIG=${3} + local REGISTRY=${4} + local USERNAME=${5} + local PASSWORD=${6} if [ -z "${REGISTRY}" ] || [ -z "${USERNAME}" ] || [ -z "${PASSWORD}" ]; then echo "No REGISTRY, USERNAME or PASSWORD provided." >&2 return 1 fi local LABEL="gantry.auth.config" - CONFIG="C$(unique_id)" + local CONFIGS_FILE= CONFIGS_FILE=$(mktemp) docker service update --quiet --label-add "${LABEL}=${CONFIG}" "${SERVICE_NAME}" echo "# Test comments: CONFIG REGISTRY USERNAME PASSWORD" >> "${CONFIGS_FILE}" @@ -126,7 +130,7 @@ Describe 'login' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_REGISTRY_CONFIGS_FILE "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_REGISTRY_CONFIGS_FILE "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be success The stdout should satisfy display_output The stderr should satisfy display_output @@ -137,7 +141,7 @@ Describe 'login' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${CONFIG}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -158,19 +162,21 @@ Describe 'login' TEST_NAME="test_login_REGISTRY_CONFIGS_FILE_bad_format" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_REGISTRY_CONFIGS_FILE_bad_format() { local TEST_NAME=${1} local SERVICE_NAME=${2} - local REGISTRY=${3} - local USERNAME=${4} - local PASSWORD=${5} + local CONFIG=${3} + local REGISTRY=${4} + local USERNAME=${5} + local PASSWORD=${6} if [ -z "${REGISTRY}" ] || [ -z "${USERNAME}" ] || [ -z "${PASSWORD}" ]; then echo "No REGISTRY, USERNAME or PASSWORD provided." >&2 return 1 fi local LABEL="gantry.auth.config" - CONFIG="C$(unique_id)" + local CONFIGS_FILE= CONFIGS_FILE=$(mktemp) docker service update --quiet --label-add "${LABEL}=${CONFIG}" "${SERVICE_NAME}" # Add an extra item to the line. @@ -189,7 +195,7 @@ Describe 'login' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_REGISTRY_CONFIGS_FILE_bad_format "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_REGISTRY_CONFIGS_FILE_bad_format "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output @@ -224,19 +230,20 @@ Describe 'login' TEST_NAME="test_login_file_not_exist" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME="gantry-test-$(unique_id)" + CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_file_not_exist() { local TEST_NAME=${1} local SERVICE_NAME=${2} - local REGISTRY=${3} - local USERNAME=${4} - local PASSWORD=${5} + local CONFIG=${3} + local REGISTRY=${4} + local USERNAME=${5} + local PASSWORD=${6} if [ -z "${REGISTRY}" ] || [ -z "${USERNAME}" ] || [ -z "${PASSWORD}" ]; then echo "No REGISTRY, USERNAME or PASSWORD provided." >&2 return 1 fi local LABEL="gantry.auth.config" - CONFIG="C$(unique_id)" docker service update --quiet --label-add "${LABEL}=${CONFIG}" "${SERVICE_NAME}" local FILE_NOT_EXIST="/tmp/${CONFIG}" reset_gantry_env "${SERVICE_NAME}" @@ -254,7 +261,7 @@ Describe 'login' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_file_not_exist "${TEST_NAME}" "${SERVICE_NAME}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_file_not_exist "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stderr should satisfy display_output diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh index a97b7a7..8dde54b 100644 --- a/tests/gantry_manifest_spec.sh +++ b/tests/gantry_manifest_spec.sh @@ -50,7 +50,7 @@ Describe 'manifest-command' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--force" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--force.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -130,7 +130,7 @@ Describe 'manifest-command' The status should be success The stdout should satisfy display_output The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--insecure" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--insecure.*" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" @@ -173,7 +173,7 @@ Describe 'manifest-command' The stdout should satisfy display_output The stderr should satisfy display_output # No options are added to the unknwon command. - The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--insecure" + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--insecure.*" The stderr should satisfy spec_expect_message "Unknown MANIFEST_CMD.*unsupported_cmd" The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh index d1af52e..35d2b2c 100644 --- a/tests/gantry_parallel_spec.sh +++ b/tests/gantry_parallel_spec.sh @@ -85,7 +85,7 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "7 ${SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "$((MAX_SERVICES_NUM+1)) ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" @@ -130,7 +130,7 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" - The stderr should satisfy spec_expect_message "11 ${SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "$((MAX_SERVICES_NUM+1)) ${SERVICES_UPDATED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh index f7adc0a..fe6f3b0 100644 --- a/tests/gantry_rollback_spec.sh +++ b/tests/gantry_rollback_spec.sh @@ -93,7 +93,7 @@ Describe 'rollback' The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--incorrect-option" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--incorrect-option.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${ROLLING_BACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh index c34a895..3fb9ad9 100644 --- a/tests/gantry_service_no_running_tasks_spec.sh +++ b/tests/gantry_service_no_running_tasks_spec.sh @@ -71,8 +71,8 @@ Describe "service-no-running-tasks" # https://github.com/docker/cli/issues/627 The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}\." The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--replicas=0.*${SERVICE_NAME}\." - The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME_SUFFIX}\." - The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--replicas=0.*${SERVICE_NAME_SUFFIX}\." + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME_SUFFIX}" + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--replicas=0.*${SERVICE_NAME_SUFFIX}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME_SUFFIX}" @@ -129,7 +129,7 @@ Describe "service-no-running-tasks" The stderr should satisfy display_output # Add "--detach=true" when there is no running tasks. # https://github.com/docker/cli/issues/627 - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}" # Cannot add "--replicas" to global mode The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--replicas=0" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh index 5b147af..05f3689 100644 --- a/tests/gantry_update_options_spec.sh +++ b/tests/gantry_update_options_spec.sh @@ -108,7 +108,7 @@ Describe 'update-options' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -148,7 +148,7 @@ Describe 'update-options' The status should be success The stdout should satisfy display_output The stderr should satisfy display_output - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}" # Cannot add "--replicas" to replicated job The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--replicas=0" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" @@ -214,7 +214,7 @@ Describe 'update-options' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"