linux-grpc-test-long-run

#!/bin/bash
#
# Copyright (C) Extensible Service Proxy Authors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
################################################################################
#

# This script runs a grpc long-running test.
#
# This script requires bazel build //test/grpc:all
# and auth_token_gen.

if [[ "$(uname)" != "Linux" ]]; then
  echo "Run on Linux only."
#  exit 1
fi

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
. ${ROOT}/script/all-utilities \
  || { echo "Cannot load Bash utilities" ; exit 1 ; }

API_KEY=''
SERVICE_NAME='echo-dot-esp-grpc-load-test.appspot.com'
HTTP_HOST=''
GRPC_HOST=''
DURATION_IN_HOUR=0

REQUEST_COUNT=10000

while getopts :a:g:h:l:s: arg; do
  case ${arg} in
    a) API_KEY="${OPTARG}";;
    g) GRPC_HOST="${OPTARG}";;
    h) HTTP_HOST="${OPTARG}";;
    l) DURATION_IN_HOUR="${OPTARG}";;
    s) SERVICE_NAME="${OPTARG}";;
    *) echo "Invalid option: -${OPTARG}";;
  esac
done

if ! [[ -n "${API_KEY}" ]]; then
  set_api_keys;
  API_KEY="${ESP_GRPC_LOAD_TEST_API_KEY}"
fi

[[ -n "${GRPC_HOST}" ]] || error_exit 'Please specify a host with -h option.'

# Nginx default max_concurrent_streams is 128.
# If CONCURRENT > 128, some requests will fail with RST_STREAM.
CONCURRENT_TYPES=(50 128)
CONCURRENT_TYPES_SIZE=${#CONCURRENT_TYPES[@]}
STREAM_COUNT_TYPES=(10 50 100)
STREAM_COUNT_TYPES_SIZE=${#STREAM_COUNT_TYPES[@]}
# For now, total transfer size per stream direction should not pass 1 MB
RANDOM_PAYLOAD_SIZE_TYPES=(1024 20000)
RANDOM_PAYLOAD_SIZE_TYPES_SIZE=${#RANDOM_PAYLOAD_SIZE_TYPES[@]}
ALL_CONFIG_TYPES=$((CONCURRENT_TYPES_SIZE * STREAM_COUNT_TYPES_SIZE * RANDOM_PAYLOAD_SIZE_TYPES_SIZE))

function generate_run_config() {
  local run=${1}
  CONCURRENT=${CONCURRENT_TYPES[$((run % CONCURRENT_TYPES_SIZE))]}
  echo concurrent="${CONCURRENT}"

  STREAM_COUNT_RUN=$((run / CONCURRENT_TYPES_SIZE))
  STREAM_COUNT=${STREAM_COUNT_TYPES[$((STREAM_COUNT_RUN % STREAM_COUNT_TYPES_SIZE))]}
  echo stream_count="${STREAM_COUNT}"

  RANDOM_PAYLOAD_SIZE_RUN=$((STREAM_COUNT_RUN / STREAM_COUNT_TYPES_SIZE))
  RANDOM_PAYLOAD_SIZE=${RANDOM_PAYLOAD_SIZE_TYPES[$((RANDOM_PAYLOAD_SIZE_RUN % RANDOM_PAYLOAD_SIZE_TYPES_SIZE))]}
  echo random_payload_size="${RANDOM_PAYLOAD_SIZE}"
}

function grpc_test_pass_through() {
  echo "Starting grpc pass through stress test at $(date)."

  local tmp_file="$(mktemp)"
  local failures=0
  for run in $(seq 1 ${ALL_CONFIG_TYPES}); do
      generate_run_config $((run - 1))
      # Generating token for each run, that they expire in 1 hour.
      local AUTH_TOKEN=$("${ROOT}/client/custom/gen-auth-token.sh" -a "${SERVICE_NAME}")
      "${ROOT}/test/grpc/grpc_stress_input.py" \
          --server="${GRPC_HOST}" \
          --api_key="${API_KEY}" \
          --auth_token="${AUTH_TOKEN}" \
          --request_count="${REQUEST_COUNT}" \
          --concurrent="${CONCURRENT}" \
          --requests_per_stream="${STREAM_COUNT}" \
          --random_payload_max_size="${RANDOM_PAYLOAD_SIZE}" \
          --random_payload_max_size="${RANDOM_PAYLOAD_SIZE}" > "${tmp_file}"
      # gRPC test client occasionally aborted. Retry up to 5 times.
      local count=0
      while : ; do
          cat "${tmp_file}" |"${ROOT}/bazel-bin/test/grpc/grpc-test-client"
          local status=$?
          if [[ "$status" == "0" ]]; then
              break
          fi
          if [[ "$status" != "134" ]] || [[ ${count} -gt 5 ]]; then
              ((failures++))
              break
          fi
          ((count++))
          echo "Test client crashed, Retry the test: ${count}"
      done
  done
  return $failures
}

function grpc_test_transcode() {
  echo "Starting grpc transcode stress test at $(date)."

  # Generating token for each run, that they expire in 1 hour.
  local AUTH_TOKEN=$("${ROOT}/client/custom/gen-auth-token.sh" -a "${SERVICE_NAME}")
  ./esp_client.py \
      --test=stress \
      --host="${HTTP_HOST}" \
      --api_key="${API_KEY}" \
      --auth_token="${AUTH_TOKEN}" \
      --test_data=grpc_test_data.json
}

function grpc_test_transcode_fuzzing() {
  STATUS_HOST="http:$(echo ${HTTP_HOST}|awk -F : '{ print $2 }'):8090"

  # Generating token for each run, that they expire in 1 hour.
  local AUTH_TOKEN=$("${ROOT}/client/custom/gen-auth-token.sh" -a "${SERVICE_NAME}")

  echo "Starting grpc transcode fuzz test at $(date)."
  ./esp_transcoding_fuzz_test.py \
      --address="${HTTP_HOST}" \
      --status_address="${STATUS_HOST}" \
      --api_key="${API_KEY}" \
      --auth_token="${AUTH_TOKEN}" \
      --runs=3
}

# Issue a request to allow Endpoints-Runtime to fetch metadata.
# If sending N requests concurrently, N-1 requests will fail while the
# first request is fetching the metadata.
cat <<EOF|"${ROOT}/bazel-bin/test/grpc/grpc-test-client"
server_addr: "${GRPC_HOST}"
plans {
  echo {
    request {
      text: "Hello, world!"
    }
  }
}
EOF

END_TIME=$(date +"%s")
END_TIME=$((END_TIME + DURATION_IN_HOUR * 60 * 60))
RUN_COUNT=0
GRPC_STRESS_FAILURES=0
HTTP_STRESS_FAILURES=0

echo "GRPC_HOST: ${GRPC_HOST}"
detect_memory_leak_init "http://${GRPC_HOST}"

# ${ROOT}/test/client/esp_client.py needs to run at that folder.
pushd ${ROOT}/test/client > /dev/null

while true; do
  echo "Starting test run ${RUN_COUNT} at $(date)."
  echo "Failures so far: pass-through: ${GRPC_STRESS_FAILURES}, transcode: ${HTTP_STRESS_FAILURES}."

  #######################
  # Insert tests here
  #######################
  ((RUN_COUNT++))

  grpc_test_pass_through || ((GRPC_STRESS_FAILURES++))

  if [[ -n ${HTTP_HOST} ]]; then
    grpc_test_transcode || ((HTTP_STRESS_FAILURES++))
    grpc_test_transcode_fuzzing || ((HTTP_STRESS_FAILURES++))
  fi
  #######################
  # End of test suite
  #######################

  detect_memory_leak_check ${RUN_COUNT}

  # Break if test has run long enough.
  [[ $(date +"%s") -lt ${END_TIME} ]] || break
done

popd > /dev/null

echo "Finished ${RUN_COUNT} test runs."
echo "Failures: pass-through: ${GRPC_STRESS_FAILURES}, transcode: ${HTTP_STRESS_FAILURES}."

# We fail the test if memory increase is large.
detect_memory_leak_final && MEMORY_LEAK=0 || MEMORY_LEAK=1

# Only mark test as failed if any pass-through tests failed.
# This is to be consistent with other http stress tests.
# All failure will be analyzed by release-engineers.

exit $((${GRPC_STRESS_FAILURES} + ${HTTP_STRESS_FAILURES} + ${MEMORY_LEAK}))