From 37fbff5c070cd899a8e4e1e18ee9ff78def93a22 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Tue, 6 Dec 2022 12:11:58 -0800 Subject: [PATCH] [dev] Standardize e2e tests (#915) * Standardize e2e tests * Ensure dss and monitoring images are current * Fix container name in local log collection --- .github/workflows/CI.md | 9 +- .github/workflows/ci.yml | 10 +- .gitignore | 1 - Makefile | 36 ++++++- build/deploy/db_schemas/README.md | 1 - build/dev/probe_local_instance.sh | 53 ----------- build/dev/standalone_instance.md | 5 +- build/dev/wait_for_local_dss.sh | 47 ++++++++++ monitoring/prober/README.md | 24 +++-- monitoring/prober/run_locally.sh | 60 ++++++++++++ test/README.md | 37 ++++---- test/docker_e2e.sh | 150 ------------------------------ 12 files changed, 190 insertions(+), 243 deletions(-) delete mode 100755 build/dev/probe_local_instance.sh create mode 100755 build/dev/wait_for_local_dss.sh create mode 100755 monitoring/prober/run_locally.sh delete mode 100755 test/docker_e2e.sh diff --git a/.github/workflows/CI.md b/.github/workflows/CI.md index 9346be3613..04cef37aeb 100644 --- a/.github/workflows/CI.md +++ b/.github/workflows/CI.md @@ -26,12 +26,13 @@ Before a pull request can be merged into the master branch, it must pass all aut ### Build `monitoring` image (`make build-monitoring`) -### End-to-end test (`make test-e2e`) +### Tear down any pre-existing local DSS instance (`make down-locally`) -Steps: +### Start local DSS instance (`make start-locally`) + +### Probe local DSS instance (`make probe-locally`) -* `make start-locally` (build/dev/run_locally.sh) -* Run pytest in monitoring/prober (in `monitoring` container) +### Bring down local DSS instance (`make down-locally`) ## `monitoring` tests (`make check-monitoring`) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16770c823b..2665badc3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,8 +58,14 @@ jobs: run: make build-dss - name: Build monitoring image run: make build-monitoring - - name: End-to-end test - run: make test-e2e + - name: Tear down any pre-existing local DSS instance + run: make down-locally + - name: Start local DSS instance + run: make start-locally + - name: Probe local DSS instance + run: make probe-locally + - name: Bring down local DSS instance + run: make down-locally monitoring-tests: name: monitoring tests diff --git a/.gitignore b/.gitignore index 3e3107cc6e..9268591f88 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,6 @@ coverage.xml .pytest_cache/ e2e_test_result test_result -probe_local_instance_test_result.xml # Translations *.mo diff --git a/Makefile b/Makefile index 3bc47ccca1..215f56ae4a 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,10 @@ shell-lint: go-lint: echo "===== Checking Go lint (except for *.gen.go files) =====" && docker run --rm -v $(CURDIR):/dss -w /dss golangci/golangci-lint:v1.50.1 golangci-lint run --timeout 5m --skip-dirs /dss/build/workspace --skip-files '.*\.gen\.go' -v -E gofmt,bodyclose,rowserrcheck,misspell,golint,staticcheck,vet +# This mirrors the hygiene-tests continuous integration workflow job (.github/workflows/ci.yml) +.PHONY: hygiene-tests +hygiene-tests: python-lint check-hygiene validate-uss-qualifier-docs shell-lint go-lint + # --- Targets to autogenerate Go code for OpenAPI-defined interfaces --- .PHONY: apis apis: example_apis dummy_oauth_api dss_apis @@ -130,18 +134,36 @@ build-monitoring: cd monitoring && make build .PHONY: test-e2e -test-e2e: - test/docker_e2e.sh +test-e2e: down-locally build-dss build-monitoring start-locally probe-locally collect-local-logs down-locally tag: scripts/tag.sh $(UPSTREAM_OWNER)/dss/v$(VERSION) +.PHONY: start-locally start-locally: - build/dev/run_locally.sh + build/dev/run_locally.sh up -d + build/dev/wait_for_local_dss.sh + +.PHONY: probe-locally +probe-locally: + monitoring/prober/run_locally.sh +.PHONY: collect-local-logs +collect-local-logs: + docker logs dss_sandbox_local-dss-core-service_1 2> core-service-for-testing.log + +.PHONY: stop-locally stop-locally: build/dev/run_locally.sh stop +.PHONY: down-locally +down-locally: + build/dev/run_locally.sh down + +# This mirrors the dss-tests continuous integration workflow job (.github/workflows/ci.yml) +.PHONY: dss-tests +dss-tests: evaluate-tanka test-go-units test-go-units-crdb build-dss build-monitoring down-locally start-locally probe-locally collect-local-logs down-locally + .PHONY: check-monitoring check-monitoring: cd monitoring && make test @@ -150,3 +172,11 @@ check-monitoring: evaluate-tanka: docker container run -v $(CURDIR)/build/jsonnetfile.json:/build/jsonnetfile.json -v $(CURDIR)/build/deploy:/build/deploy grafana/tanka show --dangerous-allow-redirect /build/deploy/examples/minimum docker container run -v $(CURDIR)/build/jsonnetfile.json:/build/jsonnetfile.json -v $(CURDIR)/build/deploy:/build/deploy grafana/tanka show --dangerous-allow-redirect /build/deploy/examples/schema_manager + +# This mirrors the monitoring-tests continuous integration workflow job (.github/workflows/ci.yml) +.PHONY: monitoring-tests +monitoring-tests: check-monitoring + +# This reproduces the entire continuous integration workflow (.github/workflows/ci.yml) +.PHONY: presubmit +presubmit: hygiene-tests dss-tests monitoring-tests diff --git a/build/deploy/db_schemas/README.md b/build/deploy/db_schemas/README.md index 5b5d57727f..3a950d1011 100644 --- a/build/deploy/db_schemas/README.md +++ b/build/deploy/db_schemas/README.md @@ -19,5 +19,4 @@ places: * [DSS main.jsonnet](../examples/minimum/main.jsonnet) * [Schema manager main.jsonnet](../examples/schema_manager/main.jsonnet) * scd_ or rid_ bootstrapper.sh in [dev/startup](../../dev/startup) -* [docker_e2e.sh](../../../test/docker_e2e.sh) * /pkg/{rid|scd}/store/cockroach/store.go diff --git a/build/dev/probe_local_instance.sh b/build/dev/probe_local_instance.sh deleted file mode 100755 index e105e0b46b..0000000000 --- a/build/dev/probe_local_instance.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -echo "Re/Create probe_local_instance_test_result file" -RESULTFILE="$(pwd)/probe_local_instance_test_result.xml" -touch "${RESULTFILE}" -CORE_CONTAINER="dss_sandbox_local-dss-core-service_1" -OAUTH_CONTAINER="dss_sandbox_local-dss-dummy-oauth_1" -declare -a localhost_containers=("$CORE_CONTAINER" "$OAUTH_CONTAINER") - -for container_name in "${localhost_containers[@]}"; do - if [ "$( docker container inspect -f '{{.State.Status}}' "$container_name" )" == "running" ]; then - echo "$container_name available!" - else - echo "Error: $container_name not running. Execute 'run_locally.sh up' before running probe_local_instance.sh"; - exit 1; - fi -done - -OS=$(uname) -if [[ $OS == "Darwin" ]]; then - # OSX uses BSD readlink - BASEDIR="$(dirname "$0")/../.." -else - BASEDIR=$(readlink -e "$(dirname "$0")/../..") -fi - -echo "probe_local_instance base directory is ${BASEDIR}" -cd "${BASEDIR}" - -sleep 1 -echo " -------------- PYTEST -------------- " -echo "Building monitoring (Integration Test) image" -docker build -q --rm -f monitoring/Dockerfile monitoring -t interuss/monitoring - -echo "Finally Begin Testing" -docker run --network dss_sandbox_default \ - --link $OAUTH_CONTAINER:oauth \ - --link $CORE_CONTAINER:core-service \ - -v "${RESULTFILE}:/app/test_result" \ - -w /app/monitoring/prober \ - interuss/monitoring \ - pytest \ - "${1:-.}" \ - -rsx \ - --junitxml=/app/test_result \ - --dss-endpoint http://core-service:8082 \ - --rid-auth "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \ - --rid-v2-auth "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \ - --scd-auth1 "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \ - --scd-auth2 "DummyOAuth(http://oauth:8085/token,sub=fake_uss2)" \ - --scd-api-version 1.0.0 diff --git a/build/dev/standalone_instance.md b/build/dev/standalone_instance.md index 7cb83c5d91..c4de84b169 100644 --- a/build/dev/standalone_instance.md +++ b/build/dev/standalone_instance.md @@ -130,10 +130,7 @@ target version parameter to `migrate_local_db.sh`. ### Prober -* `probe_local_instance.sh` runs the end-to-end [prober](../../monitoring/prober) -integration test, similar to [docker_e2e.sh](../../test/docker_e2e.sh), but using -the DSS instance already deployed locally instead of also deploying a local -instance as docker_e2e.sh does +* `monitoring/prober/run_locally.sh` runs the [prober](../../monitoring/prober) integration tests using the DSS instance already deployed locally via [`run_locally.sh`](run_locally.sh) ### RID diff --git a/build/dev/wait_for_local_dss.sh b/build/dev/wait_for_local_dss.sh new file mode 100755 index 0000000000..fa52153bc8 --- /dev/null +++ b/build/dev/wait_for_local_dss.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +OAUTH_CONTAINER="dss_sandbox_local-dss-dummy-oauth_1" +CORE_SERVICE_CONTAINER="dss_sandbox_local-dss-core-service_1" +declare -a localhost_containers=("$OAUTH_CONTAINER" "$CORE_SERVICE_CONTAINER") + +for container_name in "${localhost_containers[@]}"; do + last_message="" + while true; do + if [ "$( docker container inspect -f '{{.State.Status}}' "${container_name}" 2>/dev/null)" = "running" ]; then + break + fi + new_message="Waiting for ${container_name} container to start..." + if [ "${new_message}" = "${last_message}" ]; then + printf "." + else + printf '..%s..' "${new_message}" + last_message="${new_message}" + fi + sleep 3 + done + if [ -n "${last_message}" ]; then + echo "" + fi +done + +last_message="" +while true; do + health_status="$( docker container inspect -f '{{.State.Health.Status}}' "${CORE_SERVICE_CONTAINER}" )" + if [ "${health_status}" = "healthy" ]; then + break + else + new_message="Waiting for ${CORE_SERVICE_CONTAINER} to be available (currently ${health_status})..." + if [ "${new_message}" = "${last_message}" ]; then + printf "." + else + printf '..%s..' "${new_message}" + last_message="${new_message}" + fi + sleep 3 + fi +done +if [ -n "${last_message}" ]; then + echo "" +fi + +echo "Local DSS instance is now available." diff --git a/monitoring/prober/README.md b/monitoring/prober/README.md index 01c743e40b..f4568f8c8e 100644 --- a/monitoring/prober/README.md +++ b/monitoring/prober/README.md @@ -2,7 +2,7 @@ This directory contains integration tests that can be run against a live DSS instance. These integration tests are also run as part of -[docker_e2e.sh](../../test/docker_e2e.sh) which is a self-contained system +[`make test-e2e`](../../Makefile) which is a self-contained system integration test suite. ## Authorization @@ -27,19 +27,19 @@ the tests that depend on that authorization will be skipped. Example: `--rid-auth "UsernamePassword(https://example.com/token, username=uss1, password=uss1, client_id=uss1)"` -## Running pytest via Docker +## Running prober via Docker This approach takes slightly longer to execute due to building the prober image, but it requires no setup and generally obtains more reproducible results than running locally. From the root of this repo: First, build the monitoring image: -(from [`monitoring`](../) working directory) +(from the repo root) ```shell script -docker image build . -t interuss/monitoring +monitoring/build.sh ``` -...then run it: +...then run prober: ```shell script docker run --rm interuss/monitoring \ @@ -51,7 +51,7 @@ docker run --rm interuss/monitoring \ [--scd-auth2 ] ``` -## Running pytest locally +## Running prober locally This approach will save the time of building the prober Docker container, but requires more setup and may not be fully-reproducible due to system differences in Python, etc. @@ -75,19 +75,25 @@ pytest \ -vv . ``` +## Running prober on a local DSS deployment + +A local DSS ecosystem can be deployed with `make start-locally` from the repo +root. Then, prober can be run (from the repo root) with +`monitoring/prober/run_locally.sh` + ## Specifying specific tests ### All the tests in a folder -E.g.: `test/docker_e2e.sh scd/` +E.g.: `monitoring/prober/run_locally.sh scd/` ### All the tests in a file -`test/docker_e2e.sh scd/test_constraint_simple.py` +`monitoring/prober/run_locally.sh scd/test_constraint_simple.py` ### A specific test -`test/docker_e2e.sh scd/test_constraint_simple.py::test_ensure_clean_workspace` +`monitoring/prober/run_locally.sh scd/test_constraint_simple.py::test_ensure_clean_workspace` ## Writing prober tests This project strives to make prober tests as easy as possible to create in order diff --git a/monitoring/prober/run_locally.sh b/monitoring/prober/run_locally.sh new file mode 100755 index 0000000000..924aa8302e --- /dev/null +++ b/monitoring/prober/run_locally.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +set -eo pipefail +set -x + +# Find and change to repo root directory +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")" +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "${BASEDIR}/../.." || exit 1 + +CORE_SERVICE_CONTAINER="dss_sandbox_local-dss-core-service_1" +OAUTH_CONTAINER="dss_sandbox_local-dss-dummy-oauth_1" +declare -a localhost_containers=("$CORE_SERVICE_CONTAINER" "$OAUTH_CONTAINER") + +for container_name in "${localhost_containers[@]}"; do + if [ "$( docker container inspect -f '{{.State.Status}}' "$container_name" )" == "running" ]; then + echo "$container_name available!" + else + echo '#########################################################################' + echo '## Prerequisite to run this command is: ##' + echo '## Local DSS instance + Dummy OAuth server (/build/dev/run_locally.sh) ##' + echo '#########################################################################' + echo "Error: $container_name not running. Execute 'build/dev/run_locally.sh up' before running monitoring/prober/run_locally.sh"; + exit 1; + fi +done + +echo "Re/Create e2e_test_result file" +RESULTFILE="$(pwd)/e2e_test_result" +touch "${RESULTFILE}" +cat /dev/null > "${RESULTFILE}" + +if ! docker run --link "$OAUTH_CONTAINER":oauth \ + --link "$CORE_SERVICE_CONTAINER":core-service \ + --network dss_sandbox_default \ + -v "${RESULTFILE}:/app/test_result" \ + -w /app/monitoring/prober \ + interuss/monitoring \ + pytest \ + "${1:-.}" \ + -rsx \ + --junitxml=/app/test_result \ + --dss-endpoint http://core-service:8082 \ + --rid-auth "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \ + --rid-v2-auth "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \ + --scd-auth1 "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \ + --scd-auth2 "DummyOAuth(http://oauth:8085/token,sub=fake_uss2)" \ + --scd-api-version 1.0.0; then + + if [ "$CI" == "true" ]; then + echo "=== END OF TEST RESULTS ===" + echo "Dumping core-service logs" + docker logs "$CORE_SERVICE_CONTAINER" + fi +fi diff --git a/test/README.md b/test/README.md index 91972316ab..5cf9a909b6 100644 --- a/test/README.md +++ b/test/README.md @@ -19,26 +19,31 @@ make test-cockroach ## Integration tests For tests that benefit from being run in a fully-constructed environment, the -[`docker_e2e.sh`](docker_e2e.sh) script in this folder sets up a full -environment and runs a set of tests in that environment. Docker is the only -prerequisite to running this end-to-end test on your local system. +`make test-e2e` from the repo root folder sets up a full environment and runs +the prober tests in that environment. Docker is the only prerequisite to +running this end-to-end test on your local system. + +For repeated tests without changes to the DSS, the local DSS instance can be +brought up initially with `make start-locally`, then the prober tests can be run +repeatedly with `make probe-locally` without needing to rerun +`make start-locally`. To capture DSS logs, run `make collect-local-logs`. To +bring down the local DSS instance at the conclusion of testing, run +`make stop-locally` or `make down-locally`. ### Running a subset of tests -To test a specific test in the [prober](../monitoring/prober) test suite, -simply add its name as the first argument to `docker_e2e.sh`. For example: +To run a specific test in the [prober](../monitoring/prober) test suite, +simply add its name as the first argument to the script to run prober locally +(this is the same script `make probe-locally` uses). For example: ```shell script -./docker_e2e.sh scd/test_constraint_simple.py -./docker_e2e.sh scd/test_constraint_simple.py::test_constraint_does_not_exist_get +./monitoring/prober/run_locally.sh scd/test_constraint_simple.py +./monitoring/prober/run_locally.sh scd/test_constraint_simple.py::test_ensure_clean_workspace ``` ### Examining Core Service logs -After a `docker_e2e.sh` run, the Core Service logs are automatically captured -to core-service-for-testing.log in the repository root. +After a `make probe-locally` run, the Core Service logs can be examined in the +Core Service container (usually `dss_sandbox_local-dss-core-service_1`) or +dumped to core-service-for-testing.log using `make collect-local-logs`. -## Lint checks -One of the continuous integration presubmit checks on this repository checks Go -style with a linter. To run this check yourself, run the following command in -the root folder of this repo: -```shell script -make lint -``` +## Continuous integration +The other tests involved in continuous integration presubmit checks are +described in [the continuous integration folder](../.github/workflows/CI.md). diff --git a/test/docker_e2e.sh b/test/docker_e2e.sh deleted file mode 100755 index e5da48428f..0000000000 --- a/test/docker_e2e.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -echo "Re/Create e2e_test_result file" -RESULTFILE="$(pwd)/e2e_test_result" -touch "${RESULTFILE}" -cat /dev/null > "${RESULTFILE}" - -OS=$(uname) -if [[ $OS == "Darwin" ]]; then - # OSX uses BSD readlink - BASEDIR="$(dirname "$0")/.." -else - BASEDIR=$(readlink -e "$(dirname "$0")/..") -fi - -echo "e2e base directory is ${BASEDIR}" -cd "${BASEDIR}" - -function gather_logs() { - docker logs core-service-for-testing 2> core-service-for-testing.log -} - -function cleanup() { - # ----------- clean up ----------- - echo "Stopping dummy oauth container" - docker rm -f dummy-oauth-for-testing &> /dev/null || true - - echo "Stopping core-service container" - docker kill -f core-service-for-testing &> /dev/null || true - - echo "Stopping crdb docker" - docker rm -f dss-crdb-for-debugging &> /dev/null || true -} - -function on_exit() { - gather_logs || true - cleanup -} - -function on_sigint() { - cleanup - exit -} - -trap on_exit EXIT -trap on_sigint SIGINT - - -echo " -------------- BOOTSTRAP ----------------- " -echo "Building local container for testing (see core-service-build.log for details)" -docker build --rm . -t local-interuss-dss-image > core-service-build.log - -echo " ---------------- CRDB -------------------- " -echo "cleaning up any crdb pre-existing containers" -docker rm -f dss-crdb-for-debugging &> /dev/null || echo "No CRDB to clean up" - -echo "Starting cockroachdb with admin port on :8080" -docker run -d --rm --name dss-crdb-for-debugging \ - -p 26257:26257 \ - -p 8080:8080 \ - cockroachdb/cockroach:v21.2.7 start-single-node \ - --insecure > /dev/null - -sleep 1 -echo "Bootstrapping RID Database tables" -docker run --rm --name rid-db-manager \ - --link dss-crdb-for-debugging:crdb \ - local-interuss-dss-image \ - /usr/bin/db-manager \ - --schemas_dir db-schemas/rid \ - --db_version "latest" \ - --cockroach_host crdb - -sleep 1 -echo "Bootstrapping SCD Database tables" -docker run --rm --name scd-db-manager \ - --link dss-crdb-for-debugging:crdb \ - local-interuss-dss-image \ - /usr/bin/db-manager \ - --schemas_dir db-schemas/scd \ - --db_version "latest" \ - --cockroach_host crdb - -sleep 1 -echo " ------------ CORE SERVICE ---------------- " -echo "Cleaning up any pre-existing core-service container" -docker rm -f core-service-for-testing &> /dev/null || echo "No core service to clean up" - -echo "Starting core service on :8082" -docker run -d --name core-service-for-testing \ - --link dss-crdb-for-debugging:crdb \ - -v "$(pwd)/build/test-certs/auth2.pem:/app/test.crt" \ - local-interuss-dss-image \ - core-service \ - -addr :8082 \ - --cockroach_host crdb \ - -public_key_files /app/test.crt \ - -log_format console \ - -dump_requests \ - -accepted_jwt_audiences core-service \ - -enable_scd \ - -enable_http - -echo " -------------- DUMMY OAUTH -------------- " -echo "Building dummy-oauth server container" -docker build --rm -f cmds/dummy-oauth/Dockerfile . -t local-dummy-oauth > dummy-oauth-build.log - -echo "Cleaning up any pre-existing dummy-oauth container" -docker rm -f dummy-oauth-for-testing &> /dev/null || echo "No dummy oauth to clean up" - -echo "Starting mock oauth server on :8085" -docker run -d --name dummy-oauth-for-testing -p 8085:8085 \ - -v "$(pwd)/build/test-certs/auth2.key:/app/test.key" \ - local-dummy-oauth \ - -private_key_file /app/test.key - -sleep 1 -echo " -------------- PYTEST -------------- " -echo "Building monitoring (Integration Test) image" -docker build -q --rm -f monitoring/Dockerfile monitoring -t interuss/monitoring - -echo "Finally Begin Testing" -if ! docker run --link dummy-oauth-for-testing:oauth \ - --link core-service-for-testing:core-service \ - -v "${RESULTFILE}:/app/test_result" \ - -w /app/monitoring/prober \ - interuss/monitoring \ - pytest \ - "${1:-.}" \ - -rsx \ - --junitxml=/app/test_result \ - --dss-endpoint http://core-service:8082 \ - --rid-auth "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \ - --rid-v2-auth "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \ - --scd-auth1 "DummyOAuth(http://oauth:8085/token,sub=fake_uss)" \ - --scd-auth2 "DummyOAuth(http://oauth:8085/token,sub=fake_uss2)" \ - --scd-api-version 1.0.0; then - - if [ "$CI" == "true" ]; then - echo "=== END OF TEST RESULTS ===" - echo "Dumping core-service logs" - docker logs core-service-for-testing - fi -fi - -echo "Cleaning up core-service container" -docker stop core-service-for-testing > /dev/null -test "$(docker inspect core-service-for-testing --format='{{.State.ExitCode}}')" = 0