Simplify testing and update test docs

scitran · Jan 25, 2018 · 5c4756d · 5c4756d
1 parent ba51d52
commit 5c4756d
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 105 deletions.
diff --git a/tests/README.md b/tests/README.md
@@ -1,33 +1,24 @@
-## Run the tests
-
-### Ubuntu
-Run automated tests:
+## Testing
+Build scitran-core and run automated tests in a docker container:
 ```
-# Follow installation instructions in README first
-. /runtime/bin/activate # Or wherever your scitran virtualenv is
-./test/bin/setup-integration-tests-ubuntu.sh
-./test/bin/run-tests-ubuntu.sh
+./tests/bin/docker-tests.sh
 ```
-All tests are executed by default. Subsets can be run using the filtering options:
 
-* To run linting, use `--lint` (`-l`)
-* To run unit tests, use `--unit` (`-u`)
-* To run integration tests, use `--integ` (`-i`)
+* To skip building the image, use `--no-build` (`-B`)
+* All tests (unit, integration and linting) are executed by default
 * To pass any arguments to `py.test`, use `-- PYTEST_ARGS`
+    * To run only a subset of test, use the [keyword expression filter](https://docs.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests) `-k`
+    * To see `print` output during tests, increase verbosity with `-vvv`
+    * To get a debugger session on failures, use [`--pdb`](https://docs.pytest.org/en/latest/usage.html#dropping-to-pdb-python-debugger-on-failures)
 
 See [py.test usage](https://docs.pytest.org/en/latest/usage.html) for more.
 
-### Docker
-Build scitran-core image and run automated tests in a docker container:
-```
-./tests/bin/run-tests-docker.sh
-```
-* To skip building the image, use `--no-build` (`-B`)
-* To pass any arguments to `run-tests-ubuntu.sh`, use `-- TEST_ARGS`
-
+### Example
+The most common use case is adding a new (still failing) test, and wanting to
+* (re-)run it as fast as possible (`-B` and `-k foo`)
+* see output from quick and dirty `print`s in the test (`-vvv`)
+* get into an interactive pdb session to inspect what went wrong (`--pdb`)
 
-#### Example
-Without rebuilding the image, run only integration tests matching `foo`, use the highest verbosity level for test output and jump into a python debugger session in case an assertion fails:
 ```
-./tests/bin/run-tests-docker.sh -B -- -i -- -k foo -vvv --pdb
+./tests/bin/docker-tests.sh -B -- -k foo -vvv --pdb
 ```
diff --git a/tests/bin/docker-tests.sh b/tests/bin/docker-tests.sh
@@ -7,7 +7,7 @@ cd "$( dirname "$0" )/../.."
 
 USAGE="
 Usage:
-    $0 [OPTION...] [-- TEST_ARGS...]
+    $0 [OPTION...] [-- PYTEST_ARGS...]
 
 Build scitran/core image and run tests in a Docker container.
 Also displays coverage report and saves HTML in htmlcov dir.
@@ -17,17 +17,21 @@ Options:
 
     -B, --no-build      Skip rebuilding default Docker image
     --image IMAGE       Use custom Docker image
-    -- TEST_ARGS        Arguments passed to tests/bin/tests.sh
+    -- PYTEST_ARGS      Arguments passed to py.test
 
 "
 
 
 main() {
     local DOCKER_IMAGE=
-    local TEST_ARGS=
+    local PYTEST_ARGS=
 
     while [ $# -gt 0 ]; do
         case "$1" in
+            -h|--help)
+                log "$USAGE"
+                exit 0
+                ;;
             -B|--no-build)
                 DOCKER_IMAGE="scitran/core:testing"
                 ;;
@@ -37,16 +41,12 @@ main() {
                 ;;
             --)
                 shift
-                TEST_ARGS="$@"
+                PYTEST_ARGS="$@"
                 break
                 ;;
-            -h|--help)
-                printf "$USAGE" >&2
-                exit 0
-                ;;
             *)
-                printf "Invalid argument: $1\n" >&2
-                printf "$USAGE" >&2
+                log "Invalid argument: $1"
+                log "$USAGE"
                 exit 1
                 ;;
         esac
@@ -61,31 +61,28 @@ main() {
         docker tag "$DOCKER_IMAGE" "scitran/core:testing"
     fi
 
-    trap clean_up EXIT
-
-    docker network create core-test
-
-    local SCITRAN_CORE_DRONE_SECRET="secret"
-
-    # Clean dev/test artifacts like pyc files and coverage reports
+    log "Cleaning pyc and previous coverage results ..."
     # Run within container to avoid permission problems
     docker run --rm \
         --name core-test-cleanup \
         --volume $(pwd):/src/core \
         scitran/core:testing \
-        sh -c '
-            find . -type d -name __pycache__ -exec rm -rf {}\;;
-            find . -type f -name "*.pyc" -delete;
+        sh -c "
+            find . -type d -name __pycache__ -exec rm -rf {} \;;
+            find . -type f -name '*.pyc' -delete;
             rm -rf .coverage htmlcov;
-        '
+        "
+
+    trap clean_up EXIT
+    docker network create core-test
 
     # Launch core + mongo
     docker run -d \
         --name core-test-service \
         --network core-test \
         --volume $(pwd)/api:/src/core/api \
         --volume $(pwd)/tests:/src/core/tests \
-        --env SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET \
+        --env SCITRAN_CORE_DRONE_SECRET=secret \
         --env SCITRAN_RUNTIME_COVERAGE=true \
         --env SCITRAN_CORE_ACCESS_LOG_ENABLED=true \
         scitran/core:testing
@@ -97,11 +94,11 @@ main() {
         --volume $(pwd)/api:/src/core/api \
         --volume $(pwd)/tests:/src/core/tests \
         --env SCITRAN_SITE_API_URL=http://core-test-service/api \
-        --env SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET \
+        --env SCITRAN_CORE_DRONE_SECRET=secret \
         --env SCITRAN_PERSISTENT_DB_URI=mongodb://core-test-service:27017/scitran \
         --env SCITRAN_PERSISTENT_DB_LOG_URI=mongodb://core-test-service:27017/logs \
         scitran/core:testing \
-        tests/bin/tests.sh $TEST_ARGS
+        tests/bin/tests.sh -- $PYTEST_ARGS
 }
 
 

diff --git a/tests/bin/tests.sh b/tests/bin/tests.sh
@@ -7,20 +7,15 @@ cd "$( dirname "$0" )/../.."
 
 USAGE="
 Usage:
-    $0 [OPTION...] [-- PYTEST_ARGS...]
+    $0 [-- PYTEST_ARGS...]
 
-Runs linting and all tests if no options are provided.
-Runs subset of tests when using the filtering options.
+Runs all tests (unit, integ and linting) if no options are provided.
 
 Assumes running in a scitran/core:testing container or that core and all
 of its dependencies are installed the same way as in the Dockerfile.
 
 Options:
     -h, --help           Print this help and exit
-
-    -l, --lint           Run linting
-    -u, --unit           Run unit tests
-    -i, --integ          Run integration tests
     -- PYTEST_ARGS       Arguments passed to py.test
 
 Envvars (required for integration tests):
@@ -34,74 +29,41 @@ Envvars (required for integration tests):
 
 main() {
     export PYTHONDONTWRITEBYTECODE=1
-    export RUN_ALL=true
-    local RUN_LINT=false
-    local RUN_UNIT=false
-    local RUN_INTEG=false
     local PYTEST_ARGS=
 
     while [ $# -gt 0 ]; do
         case "$1" in
-            -l|--lint)
-                RUN_ALL=false
-                RUN_LINT=true
-                ;;
-            -u|--unit)
-                RUN_ALL=false
-                RUN_UNIT=true
-                ;;
-            -i|--integ)
-                RUN_ALL=false
-                RUN_INTEG=true
+            -h|--help)
+                log "$USAGE"
+                exit 0
                 ;;
             --)
                 shift
                 PYTEST_ARGS="$@"
                 break
                 ;;
-            -h|--help)
-                printf "$USAGE" >&2
-                exit 0
-                ;;
             *)
-                printf "Invalid argument: $1\n" >&2
-                printf "$USAGE" >&2
+                log "Invalid argument: $1"
+                log "$USAGE" >&2
                 exit 1
                 ;;
         esac
         shift
     done
 
-    if ${RUN_ALL}; then
-        # No filtering options used, run everything by default
-        RUN_LINT=true
-        RUN_UNIT=true
-        RUN_INTEG=true
-    elif ${RUN_LINT} && ${RUN_UNIT} && ${RUN_INTEG}; then
-        # All filtering options were used, the same as none
-        RUN_ALL=true
-    fi
-
-    if ${RUN_LINT}; then
-        log "Running pylint ..."
-        # TODO Enable Refactor and Convention reports
-        # TODO Move --disable into rc
-        pylint --jobs=4 --reports=no --disable=C,R,W0312,W0141,W0110 api
-
-        # log "Running pep8 ..."
-        # pep8 --max-line-length=150 --ignore=E402 api
-    fi
-
-    if ${RUN_UNIT}; then
-        log "Running unit tests ..."
-        rm -f .coverage
-        py.test --cov=api --cov-report= tests/unit_tests/python $PYTEST_ARGS
-    fi
-
-    if ${RUN_INTEG}; then
-        log "Running integration tests ..."
-        py.test tests/integration_tests/python $PYTEST_ARGS
-    fi
+    log "Running unit tests ..."
+    py.test --cov=api --cov-report= tests/unit_tests/python $PYTEST_ARGS
+
+    log "Running integration tests ..."
+    py.test tests/integration_tests/python $PYTEST_ARGS
+
+    log "Running pylint ..."
+    # TODO Enable Refactor and Convention reports
+    # TODO Move --disable into rc
+    pylint --jobs=4 --reports=no --disable=C,R,W0312,W0141,W0110 api
+
+    # log "Running pep8 ..."
+    # pep8 --max-line-length=150 --ignore=E402 api
 }