Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Prevent certain tests from being run twice in CI #4529

Merged
merged 13 commits into from
Jul 24, 2023
Merged

Prevent certain tests from being run twice in CI #4529

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
0ff57ce
Add currently running conformance tests to CI
stxue1 Jun 29, 2023
d28f6e9
add 1.1 and fix ci integration
stxue1 Jun 30, 2023
af44fa2
Run conformance properly
stxue1 Jun 30, 2023
0f131f8
Fix conformance
stxue1 Jun 30, 2023
6ecbdd1
improve error printing
stxue1 Jul 3, 2023
c2ea962
switch from return code to parsing output
stxue1 Jul 3, 2023
18d41cf
make pytest take in markers
stxue1 Jul 5, 2023
5dc757d
only include markers for batchsystems and wdl
stxue1 Jul 11, 2023
5b1c0c2
Remove accidental junk from other branch
stxue1 Jul 12, 2023
9b2cb3c
Update .gitlab suggestions
stxue1 Jul 17, 2023
9ce59c9
Merge branch 'master' into issues/4521-duplicate-ci-tests
adamnovak Jul 20, 2023
1da3228
Merge branch 'master' into issues/4521-duplicate-ci-tests
adamnovak Jul 20, 2023
21c195e
Merge branch 'master' into issues/4521-duplicate-ci-tests
adamnovak Jul 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 10 additions & 13 deletions .gitlab-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ variables:
DEBIAN_FRONTEND: "noninteractive"
TOIL_OWNER_TAG: "shared"
MAIN_PYTHON_PKG: "python3.9"

# Used to tell pytest which tests to be run by specifying markers,
# Allows partitioning of tests to prevent duplicate running of tests in different jobs.
# Currently specifies special tests that are not run by quick_test_offline.
MARKER: "tes or integrative or encryption or kubernetes or mesos or server_mode or fetchable_appliance or appliance or slow or docker or cwl or singularity or rsync3"
before_script:
# Log where we are running, in case some Kubernetes hosts are busted. IPs are assigned per host.
- ip addr
Expand Down Expand Up @@ -65,15 +68,15 @@ cwl_dependency_is_stand_alone:
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[cwl]
- make test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testCwlexample
- make marker="${MARKER}" test tests=src/toil/test/docs/scriptsTest.py::ToilDocumentationTest::testCwlexample


wdl_dependency_is_stand_alone:
stage: linting_and_dependencies
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[wdl]
- make test tests=src/toil/test/wdl/toilwdlTest.py::ToilWdlTest::testMD5sum
- make marker="${MARKER}" test tests=src/toil/test/wdl/toilwdlTest.py::ToilWdlTest::testMD5sum

quick_test_offline:
stage: basic_tests
Expand Down Expand Up @@ -158,7 +161,7 @@ batch_systems:
- export TOIL_TES_ENDPOINT="http://127.0.0.1:8000"
- export TOIL_TES_USER="${FUNNEL_SERVER_USER}"
- export TOIL_TES_PASSWORD="${FUNNEL_SERVER_PASSWORD}"
- make test tests="src/toil/test/batchSystems/batchSystemTest.py src/toil/test/mesos/MesosDataStructuresTest.py"
- make marker="${MARKER}" test tests="src/toil/test/batchSystems/batchSystemTest.py src/toil/test/mesos/MesosDataStructuresTest.py"
- kill $(jobs -p) || true

slurm_test:
Expand Down Expand Up @@ -270,22 +273,16 @@ wdl:
- pwd
- apt update && apt install -y default-jre
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
- make test tests=src/toil/test/wdl/wdltoil_test.py
- make marker="${MARKER}" test tests=src/toil/test/wdl/wdltoil_test.py
- which java &> /dev/null || { echo >&2 "Java is not installed. Install java to run these tests."; exit 1; }
- make test tests="src/toil/test/wdl/toilwdlTest.py src/toil/test/wdl/builtinTest.py" # needs java (default-jre) to run "GATK.jar"
# Run some WDL conformance tests.
# These are not set up as Toil PyTest tests (yet?)
- git clone https://github.com/adamnovak/wdl-conformance-tests.git
- cd wdl-conformance-tests
- git checkout 0ca00774331630d8eb06990caa8a1ba178cabecb
- python run.py --runner toil-wdl-runner --versions 1.0 --threads 8
Comment on lines -275 to -281
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't cutting this be part of #4530 and not this PR?

I guess it's fine to do it here though.

- make marker="${MARKER}" test tests="src/toil/test/wdl/toilwdlTest.py src/toil/test/wdl/builtinTest.py" # needs java (default-jre) to run "GATK.jar"

jobstore_and_provisioning:
stage: main_tests
script:
- pwd
- ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all] packages='htcondor==10.2.0.post1'
- make test tests="src/toil/test/lib/aws/ src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"
- make marker="${MARKER}" test tests="src/toil/test/lib/aws/ src/toil/test/jobStores/jobStoreTest.py src/toil/test/sort/sortTest.py src/toil/test/provisioners/aws/awsProvisionerTest.py src/toil/test/provisioners/clusterScalerTest.py"
# - make test tests=src/toil/test/provisioners/gceProvisionerTest.py
# https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
# guessing decorators are masking class as function? ^ also, abstract class is run as normal test? should hide.
Expand Down
10 changes: 7 additions & 3 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ normal=\033[0m
red=\033[0;31m
cyan=\033[0;36m

# expression to tell pytest what tests to run based on their marks
# the empty string means all tests will be run
marker=""

develop: check_venv
pip install -e .$(extras) $(packages)

Expand Down Expand Up @@ -127,20 +131,20 @@ clean_sdist:
# Setting SET_OWNER_TAG will tag cloud resources so that UCSC's cloud murder bot won't kill them.
test: check_venv check_build_reqs
TOIL_OWNER_TAG="shared" \
python -m pytest --durations=0 --strict-markers --log-level DEBUG --log-cli-level INFO -r s $(cov) -n auto --dist loadscope $(tests)
python -m pytest --durations=0 --strict-markers --log-level DEBUG --log-cli-level INFO -r s $(cov) -n auto --dist loadscope $(tests) -m "$(marker)"


# This target will skip building docker and all docker based tests
# these are our travis tests; rename?
test_offline: check_venv check_build_reqs
@printf "$(cyan)All docker related tests will be skipped.$(normal)\n"
TOIL_SKIP_DOCKER=True \
python -m pytest -vv --timeout=600 --strict-markers --log-level DEBUG --log-cli-level INFO $(cov) -n auto --dist loadscope $(tests)
python -m pytest -vv --timeout=600 --strict-markers --log-level DEBUG --log-cli-level INFO $(cov) -n auto --dist loadscope $(tests) -m "$(marker)"

# This target will run about 1 minute of tests, and stop at the first failure
test_1min: check_venv check_build_reqs
TOIL_SKIP_DOCKER=True \
python -m pytest -vv --timeout=10 --strict-markers --log-level DEBUG --log-cli-level INFO --maxfail=1 src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::CWLWorkflowTest::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior
python -m pytest -vv --timeout=10 --strict-markers --log-level DEBUG --log-cli-level INFO --maxfail=1 src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::CWLWorkflowTest::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior -m "$(marker)"

ifdef TOIL_DOCKER_REGISTRY

Expand Down