diff --git a/tests/README.md b/tests/README.md index 277e7e752..ba72b5c51 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,33 +1,24 @@ -## Run the tests - -### Ubuntu -Run automated tests: +## Testing +Build scitran-core and run automated tests in a docker container: ``` -# Follow installation instructions in README first -. /runtime/bin/activate # Or wherever your scitran virtualenv is -./test/bin/setup-integration-tests-ubuntu.sh -./test/bin/run-tests-ubuntu.sh +./tests/bin/docker-tests.sh ``` -All tests are executed by default. Subsets can be run using the filtering options: -* To run linting, use `--lint` (`-l`) -* To run unit tests, use `--unit` (`-u`) -* To run integration tests, use `--integ` (`-i`) +* To skip building the image, use `--no-build` (`-B`) +* All tests (unit, integration and linting) are executed by default * To pass any arguments to `py.test`, use `-- PYTEST_ARGS` + * To run only a subset of test, use the [keyword expression filter](https://docs.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests) `-k` + * To see `print` output during tests, increase verbosity with `-vvv` + * To get a debugger session on failures, use [`--pdb`](https://docs.pytest.org/en/latest/usage.html#dropping-to-pdb-python-debugger-on-failures) See [py.test usage](https://docs.pytest.org/en/latest/usage.html) for more. -### Docker -Build scitran-core image and run automated tests in a docker container: -``` -./tests/bin/run-tests-docker.sh -``` -* To skip building the image, use `--no-build` (`-B`) -* To pass any arguments to `run-tests-ubuntu.sh`, use `-- TEST_ARGS` - +### Example +The most common use case is adding a new (still failing) test, and wanting to +* (re-)run it as fast as possible (`-B` and `-k foo`) +* see output from quick and dirty `print`s in the test (`-vvv`) +* get into an interactive pdb session to inspect what went wrong (`--pdb`) -#### Example -Without rebuilding the image, run only integration tests matching `foo`, use the highest verbosity level for test output and jump into a python debugger session in case an assertion fails: ``` -./tests/bin/run-tests-docker.sh -B -- -i -- -k foo -vvv --pdb +./tests/bin/docker-tests.sh -B -- -k foo -vvv --pdb ``` diff --git a/tests/bin/docker-tests.sh b/tests/bin/docker-tests.sh index eff259574..0ea8412fa 100755 --- a/tests/bin/docker-tests.sh +++ b/tests/bin/docker-tests.sh @@ -7,7 +7,7 @@ cd "$( dirname "$0" )/../.." USAGE=" Usage: - $0 [OPTION...] [-- TEST_ARGS...] + $0 [OPTION...] [-- PYTEST_ARGS...] Build scitran/core image and run tests in a Docker container. Also displays coverage report and saves HTML in htmlcov dir. @@ -17,17 +17,21 @@ Options: -B, --no-build Skip rebuilding default Docker image --image IMAGE Use custom Docker image - -- TEST_ARGS Arguments passed to tests/bin/tests.sh + -- PYTEST_ARGS Arguments passed to py.test " main() { local DOCKER_IMAGE= - local TEST_ARGS= + local PYTEST_ARGS= while [ $# -gt 0 ]; do case "$1" in + -h|--help) + log "$USAGE" + exit 0 + ;; -B|--no-build) DOCKER_IMAGE="scitran/core:testing" ;; @@ -37,16 +41,12 @@ main() { ;; --) shift - TEST_ARGS="$@" + PYTEST_ARGS="$@" break ;; - -h|--help) - printf "$USAGE" >&2 - exit 0 - ;; *) - printf "Invalid argument: $1\n" >&2 - printf "$USAGE" >&2 + log "Invalid argument: $1" + log "$USAGE" exit 1 ;; esac @@ -61,23 +61,20 @@ main() { docker tag "$DOCKER_IMAGE" "scitran/core:testing" fi - trap clean_up EXIT - - docker network create core-test - - local SCITRAN_CORE_DRONE_SECRET="secret" - - # Clean dev/test artifacts like pyc files and coverage reports + log "Cleaning pyc and previous coverage results ..." # Run within container to avoid permission problems docker run --rm \ --name core-test-cleanup \ --volume $(pwd):/src/core \ scitran/core:testing \ - sh -c ' - find . -type d -name __pycache__ -exec rm -rf {}\;; - find . -type f -name "*.pyc" -delete; + sh -c " + find . -type d -name __pycache__ -exec rm -rf {} \;; + find . -type f -name '*.pyc' -delete; rm -rf .coverage htmlcov; - ' + " + + trap clean_up EXIT + docker network create core-test # Launch core + mongo docker run -d \ @@ -85,7 +82,7 @@ main() { --network core-test \ --volume $(pwd)/api:/src/core/api \ --volume $(pwd)/tests:/src/core/tests \ - --env SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET \ + --env SCITRAN_CORE_DRONE_SECRET=secret \ --env SCITRAN_RUNTIME_COVERAGE=true \ --env SCITRAN_CORE_ACCESS_LOG_ENABLED=true \ scitran/core:testing @@ -97,11 +94,11 @@ main() { --volume $(pwd)/api:/src/core/api \ --volume $(pwd)/tests:/src/core/tests \ --env SCITRAN_SITE_API_URL=http://core-test-service/api \ - --env SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET \ + --env SCITRAN_CORE_DRONE_SECRET=secret \ --env SCITRAN_PERSISTENT_DB_URI=mongodb://core-test-service:27017/scitran \ --env SCITRAN_PERSISTENT_DB_LOG_URI=mongodb://core-test-service:27017/logs \ scitran/core:testing \ - tests/bin/tests.sh $TEST_ARGS + tests/bin/tests.sh -- $PYTEST_ARGS } diff --git a/tests/bin/tests.sh b/tests/bin/tests.sh index 7c7607562..5592c688b 100755 --- a/tests/bin/tests.sh +++ b/tests/bin/tests.sh @@ -7,20 +7,15 @@ cd "$( dirname "$0" )/../.." USAGE=" Usage: - $0 [OPTION...] [-- PYTEST_ARGS...] + $0 [-- PYTEST_ARGS...] -Runs linting and all tests if no options are provided. -Runs subset of tests when using the filtering options. +Runs all tests (unit, integ and linting) if no options are provided. Assumes running in a scitran/core:testing container or that core and all of its dependencies are installed the same way as in the Dockerfile. Options: -h, --help Print this help and exit - - -l, --lint Run linting - -u, --unit Run unit tests - -i, --integ Run integration tests -- PYTEST_ARGS Arguments passed to py.test Envvars (required for integration tests): @@ -34,74 +29,41 @@ Envvars (required for integration tests): main() { export PYTHONDONTWRITEBYTECODE=1 - export RUN_ALL=true - local RUN_LINT=false - local RUN_UNIT=false - local RUN_INTEG=false local PYTEST_ARGS= while [ $# -gt 0 ]; do case "$1" in - -l|--lint) - RUN_ALL=false - RUN_LINT=true - ;; - -u|--unit) - RUN_ALL=false - RUN_UNIT=true - ;; - -i|--integ) - RUN_ALL=false - RUN_INTEG=true + -h|--help) + log "$USAGE" + exit 0 ;; --) shift PYTEST_ARGS="$@" break ;; - -h|--help) - printf "$USAGE" >&2 - exit 0 - ;; *) - printf "Invalid argument: $1\n" >&2 - printf "$USAGE" >&2 + log "Invalid argument: $1" + log "$USAGE" >&2 exit 1 ;; esac shift done - if ${RUN_ALL}; then - # No filtering options used, run everything by default - RUN_LINT=true - RUN_UNIT=true - RUN_INTEG=true - elif ${RUN_LINT} && ${RUN_UNIT} && ${RUN_INTEG}; then - # All filtering options were used, the same as none - RUN_ALL=true - fi - - if ${RUN_LINT}; then - log "Running pylint ..." - # TODO Enable Refactor and Convention reports - # TODO Move --disable into rc - pylint --jobs=4 --reports=no --disable=C,R,W0312,W0141,W0110 api - - # log "Running pep8 ..." - # pep8 --max-line-length=150 --ignore=E402 api - fi - - if ${RUN_UNIT}; then - log "Running unit tests ..." - rm -f .coverage - py.test --cov=api --cov-report= tests/unit_tests/python $PYTEST_ARGS - fi - - if ${RUN_INTEG}; then - log "Running integration tests ..." - py.test tests/integration_tests/python $PYTEST_ARGS - fi + log "Running unit tests ..." + py.test --cov=api --cov-report= tests/unit_tests/python $PYTEST_ARGS + + log "Running integration tests ..." + py.test tests/integration_tests/python $PYTEST_ARGS + + log "Running pylint ..." + # TODO Enable Refactor and Convention reports + # TODO Move --disable into rc + pylint --jobs=4 --reports=no --disable=C,R,W0312,W0141,W0110 api + + # log "Running pep8 ..." + # pep8 --max-line-length=150 --ignore=E402 api }