From 406b6325799ae5311c48a8fcdd79fe17f6641282 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 22 Nov 2021 22:42:51 +0000 Subject: [PATCH 1/7] Add to exception which `tool_format` is unknown --- lib/galaxy/managers/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/managers/tools.py b/lib/galaxy/managers/tools.py index 8cb347f3dacc..bca41bd11519 100644 --- a/lib/galaxy/managers/tools.py +++ b/lib/galaxy/managers/tools.py @@ -77,7 +77,7 @@ def create_tool(self, trans, tool_payload, allow_load=True): tool_version = representation.get("version", None) value = representation else: - raise Exception("Unknown tool type encountered.") + raise Exception(f"Unknown tool format [{tool_format}] encountered.") # TODO: enforce via DB constraint and catch appropriate # exception. existing_tool = self.get_tool_by_uuid(uuid) From 553d8e85106a362ae970f2c15d9ad31150149c7a Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 22 Nov 2021 23:42:14 +0000 Subject: [PATCH 2/7] Allow to specify `object_id` in `BaseWorkflowPopulator.import_workflow_from_path()` --- lib/galaxy_test/base/populators.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 35862e1d09ef..edb8b6c32f17 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -903,15 +903,16 @@ def simple_workflow(self, name: str, **create_kwds) -> str: workflow = self.load_workflow(name) return self.create_workflow(workflow, **create_kwds) - def import_workflow_from_path_raw(self, from_path: str) -> Response: + def import_workflow_from_path_raw(self, from_path: str, object_id: str = None) -> Response: data = dict( - from_path=from_path + from_path=from_path, + object_id=object_id, ) import_response = self._post("workflows", data=data) return import_response - def import_workflow_from_path(self, from_path: str) -> str: - import_response = self.import_workflow_from_path_raw(from_path) + def import_workflow_from_path(self, from_path: str, object_id: str = None) -> str: + import_response = self.import_workflow_from_path_raw(from_path, object_id) api_asserts.assert_status_code_is(import_response, 200) return import_response.json()["id"] From 8464f678db353257b7eb9654e7b87b68d38aef73 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 22 Nov 2021 23:43:18 +0000 Subject: [PATCH 3/7] Allow to specify `inputs_by` in `BaseWorkflowPopulator.invoke_workflow()` Also: - Replace deprecated `workflows/{workflow_id}/usage` API endpoint with `workflows/{workflow_id}/invocations` --- lib/galaxy_test/base/populators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index edb8b6c32f17..da39bcb7879a 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -996,13 +996,13 @@ def validate_biocompute_object(self, bco, expected_schema_version='https://w3id. api_asserts.assert_has_keys(bco['io_domain'], "input_subdomain", "output_subdomain") def invoke_workflow_raw(self, workflow_id: str, request: dict, assert_ok: bool = False) -> Response: - url = f"workflows/{workflow_id}/usage" + url = f"workflows/{workflow_id}/invocations" invocation_response = self._post(url, data=request) if assert_ok: invocation_response.raise_for_status() return invocation_response - def invoke_workflow(self, workflow_id: str, history_id: Optional[str] = None, inputs: Optional[dict] = None, request: Optional[dict] = None, assert_ok: bool = True): + def invoke_workflow(self, workflow_id: str, history_id: Optional[str] = None, inputs: Optional[dict] = None, request: Optional[dict] = None, assert_ok: bool = True, inputs_by: str = 'step_index'): if inputs is None: inputs = {} @@ -1015,7 +1015,7 @@ def invoke_workflow(self, workflow_id: str, history_id: Optional[str] = None, in if inputs: request["inputs"] = json.dumps(inputs) - request["inputs_by"] = 'step_index' + request["inputs_by"] = inputs_by invocation_response = self.invoke_workflow_raw(workflow_id, request) if assert_ok: api_asserts.assert_status_code_is(invocation_response, 200) From 11c2cb51a9fcb587cffb97d06ff909fba3546557 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 22 Nov 2021 23:48:15 +0000 Subject: [PATCH 4/7] Allow to specify `tool_or_workflow` and `job_dir` in `stage_inputs()` --- lib/galaxy/tool_util/client/staging.py | 5 ++--- lib/galaxy_test/base/populators.py | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tool_util/client/staging.py b/lib/galaxy/tool_util/client/staging.py index 51e0df336dbe..be679c08e06a 100644 --- a/lib/galaxy/tool_util/client/staging.py +++ b/lib/galaxy/tool_util/client/staging.py @@ -57,7 +57,7 @@ def _fetch_post(self, payload, files_attached=False): def _handle_job(self, job_response): """Implementer can decide if to wait for job(s) individually or not here.""" - def stage(self, tool_or_workflow, history_id, job=None, job_path=None, use_path_paste=LOAD_TOOLS_FROM_PATH, to_posix_lines=True): + def stage(self, tool_or_workflow, history_id, job=None, job_path=None, use_path_paste=LOAD_TOOLS_FROM_PATH, to_posix_lines=True, job_dir="."): files_attached = [False] def upload_func_fetch(upload_target): @@ -223,8 +223,7 @@ def create_collection_func(element_identifiers, collection_type): job_dir = os.path.dirname(os.path.abspath(job_path)) else: assert job is not None - # Figure out what "." should be here instead. - job_dir = "." + assert job_dir is not None if self.use_fetch_api: upload = upload_func_fetch diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index da39bcb7879a..f34e4f510274 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -1868,10 +1868,27 @@ def read_test_data(test_dict): return inputs, label_map, has_uploads -def stage_inputs(galaxy_interactor, history_id, job, use_path_paste=True, use_fetch_api=True, to_posix_lines=True): +def stage_inputs( + galaxy_interactor, + history_id, + job, + use_path_paste=True, + use_fetch_api=True, + to_posix_lines=True, + tool_or_workflow="workflow", + job_dir=None +): """Alternative to load_data_dict that uses production-style workflow inputs.""" + kwds = dict( + history_id=history_id, + job=job, + use_path_paste=use_path_paste, + to_posix_lines=to_posix_lines, + ) + if job_dir is not None: + kwds["job_dir"] = job_dir inputs, datasets = InteractorStaging(galaxy_interactor, use_fetch_api=use_fetch_api).stage( - "workflow", history_id=history_id, job=job, use_path_paste=use_path_paste, to_posix_lines=to_posix_lines + tool_or_workflow, **kwds ) return inputs, datasets From 03e20362206fac16a9605da611f4fc6fe1ef2e8e Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 22 Nov 2021 23:52:51 +0000 Subject: [PATCH 5/7] Allow specifying `allow_path_paste` in Galaxy test classes --- lib/galaxy_test/driver/driver_util.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index b9780102f3c5..461ecb39c1ad 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -157,6 +157,7 @@ def setup_galaxy_config( conda_auto_install=False, use_shared_connection_for_amqp=False, allow_tool_conf_override: bool = True, + allow_path_paste=False, ): """Setup environment and build config for test Galaxy instance.""" # For certain docker operations this needs to be evaluated out - e.g. for cwltool. @@ -222,6 +223,7 @@ def setup_galaxy_config( config = dict( admin_users='test@bx.psu.edu', allow_library_path_paste=True, + allow_path_paste=allow_path_paste, allow_user_creation=True, allow_user_deletion=True, api_allow_run_as='test@bx.psu.edu', @@ -1063,6 +1065,7 @@ def _register_and_run_servers(self, config_object=None, handle_config=None): galaxy_config = galaxy_config() if galaxy_config is None: setup_galaxy_config_kwds = dict( + allow_path_paste=getattr(config_object, "allow_path_paste", False), use_test_file_dir=not self.testing_shed_tools, default_install_db_merged=True, default_tool_conf=self.default_tool_conf, From 9a586a58b9da21157a87dd0e3a295ea8fd6c2917 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 30 Jul 2017 07:24:05 -0400 Subject: [PATCH 6/7] Add CWL conformance API tests 1) Download conformance tests to `test/functional/tools/cwl_tools` where they can be used also by framework and unit tests in the future. 2) Add populators to enable executing CWL conformance tests 3) Generate CWL conformance API tests only when needed for tests --- .ci/flake8_ignorelist.txt | 1 + .github/workflows/cwl_conformance.yaml | 64 +++ .gitignore | 7 +- Makefile | 24 ++ lib/galaxy/dependencies/dev-requirements.txt | 4 +- lib/galaxy_test/api/cwl/__init__.py | 0 lib/galaxy_test/api/test_workflows_cwl.py | 14 + lib/galaxy_test/base/populators.py | 211 +++++++++ pyproject.toml | 1 + pytest.ini | 34 +- run_tests.sh | 22 + scripts/cwl_conformance_to_test_cases.py | 431 +++++++++++++++++++ scripts/update_cwl_conformance_tests.sh | 29 ++ 13 files changed, 839 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/cwl_conformance.yaml create mode 100644 lib/galaxy_test/api/cwl/__init__.py create mode 100644 lib/galaxy_test/api/test_workflows_cwl.py create mode 100644 scripts/cwl_conformance_to_test_cases.py create mode 100755 scripts/update_cwl_conformance_tests.sh diff --git a/.ci/flake8_ignorelist.txt b/.ci/flake8_ignorelist.txt index f5276b3fc470..298dace888f5 100644 --- a/.ci/flake8_ignorelist.txt +++ b/.ci/flake8_ignorelist.txt @@ -15,5 +15,6 @@ lib/galaxy/util/jstree.py lib/galaxy/web/proxy/js/node_modules static/maps static/scripts +test/functional/tools/cwl_tools/v1.?/ build dist diff --git a/.github/workflows/cwl_conformance.yaml b/.github/workflows/cwl_conformance.yaml new file mode 100644 index 000000000000..66bafe046965 --- /dev/null +++ b/.github/workflows/cwl_conformance.yaml @@ -0,0 +1,64 @@ +name: CWL conformance +on: + push: + paths-ignore: + - 'client/**' + - 'doc/**' + pull_request: + paths-ignore: + - 'client/**' + - 'doc/**' +env: + GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8' +concurrency: + group: cwl-conformance-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + name: Test + runs-on: ubuntu-latest + continue-on-error: ${{ startsWith(matrix.marker, 'red') }} + strategy: + fail-fast: false + matrix: + python-version: ['3.7'] + marker: ['green', 'red and required', 'red and not required'] + conformance-version: ['cwl_conformance_v1_0'] #, 'cwl_conformance_v1_1', 'cwl_conformance_v1_2'] + services: + postgres: + image: postgres:13 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v2 + with: + path: 'galaxy root' + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Get full Python version + id: full-python-version + shell: bash + run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") + - name: Cache pip dir + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Cache galaxy venv + uses: actions/cache@v2 + with: + path: .venv + key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Run tests + run: ./run_tests.sh --skip_flakey_fails -cwl lib/galaxy_test/api/cwl -- -m "${{ matrix.marker }} and ${{ matrix.conformance-version }}" + working-directory: 'galaxy root' + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: CWL conformance test results (${{ matrix.python-version }}, ${{ matrix.marker }}, ${{ matrix.conformance-version }}) + path: 'galaxy root/run_cwl_tests.html' diff --git a/.gitignore b/.gitignore index 46d33f76c580..34e09958e335 100644 --- a/.gitignore +++ b/.gitignore @@ -125,12 +125,13 @@ tool-data/**/*.fa .pytest_cache/ assets/ test-data-cache +run_api_tests.html +run_cwl_tests.html run_framework_tests.html run_functional_tests.html run_integration_tests.html run_selenium_tests.html run_toolshed_tests.html -run_api_tests.html test/tool_shed/tmp/* .coverage htmlcov @@ -192,3 +193,7 @@ config/plugins/**/static/*.map # viz-specific build artifacts to ignore (until these are removed from codebase) config/plugins/visualizations/annotate_image/static/jquery.contextMenu.css config/plugins/visualizations/nvd3/nvd3_bar/static/nvd3.js + +# CWL conformance tests +lib/galaxy_test/api/cwl/test_cwl_conformance_v1_?.py +test/functional/tools/cwl_tools/v1.?/ diff --git a/Makefile b/Makefile index 7007169a9089..04da9cb3cd72 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,12 @@ OPEN_RESOURCE=bash -c 'open $$0 || xdg-open $$0' SLIDESHOW_TO_PDF?=bash -c 'docker run --rm -v `pwd`:/cwd astefanutti/decktape /cwd/$$0 /cwd/`dirname $$0`/`basename -s .html $$0`.pdf' YARN := $(shell command -v yarn 2> /dev/null) YARN_INSTALL_OPTS=--network-timeout 300000 --check-files +CWL_TARGETS := test/functional/tools/cwl_tools/v1.0/conformance_tests.yaml \ + test/functional/tools/cwl_tools/v1.1/conformance_tests.yaml \ + test/functional/tools/cwl_tools/v1.2/conformance_tests.yaml \ + lib/galaxy_test/api/cwl/test_cwl_conformance_v1_0.py \ + lib/galaxy_test/api/cwl/test_cwl_conformance_v1_1.py \ + lib/galaxy_test/api/cwl/test_cwl_conformance_v1_2.py all: help @echo "This makefile is used for building Galaxy's JS client, documentation, and drive the release process. A sensible all target is not implemented." @@ -130,6 +136,24 @@ update-lint-requirements: update-dependencies: update-lint-requirements ## update pinned and dev dependencies $(IN_VENV) ./lib/galaxy/dependencies/update.sh +$(CWL_TARGETS): + ./scripts/update_cwl_conformance_tests.sh + +generate-cwl-conformance-tests: $(CWL_TARGETS) ## Initialise CWL conformance tests + +clean-cwl-conformance-tests: # Clean CWL conformance tests + for f in $(CWL_TARGETS); do \ + if [ $$(basename "$$f") = conformance_tests.yaml ]; then \ + rm -rf $$(dirname "$$f"); \ + else \ + rm -f "$$f"; \ + fi \ + done + +update-cwl-conformance-tests: ## update CWL conformance tests + $(MAKE) clean-cwl-conformance-tests + $(MAKE) generate-cwl-conformance-tests + node-deps: ## Install NodeJS dependencies. ifndef YARN @echo "Could not find yarn, which is required to build the Galaxy client.\nTo install yarn, please visit \033[0;34mhttps://yarnpkg.com/en/docs/install\033[0m for instructions, and package information for all platforms.\n" diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 6cc44f832d85..3f3234dd6127 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -50,11 +50,12 @@ commonmark==0.9.1; python_version >= "3.6" and python_version < "4.0" contextvars==2.4; python_version < "3.7" and python_version >= "3.6" coverage==6.1.1; python_version >= "3.6" cryptography==35.0.0; python_version >= "3.6" +cwltest==2.2.20210901154959; python_version >= "3.6" and python_version < "4" cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4" dataclasses==0.8; python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.6.1" debtcollector==2.3.0; python_version >= "3.6" decorator==5.1.0; python_version >= "3.6" -defusedxml==0.7.1; python_version >= "3.0" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.0" +defusedxml==0.7.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4" deprecated==1.2.13; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" deprecation==2.1.0 dictobj==0.4 @@ -99,6 +100,7 @@ jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or p jsonpatch==1.32; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" jsonpointer==2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" jsonschema==4.0.0 +junit-xml==1.9; python_version >= "3.6" and python_version < "4" keystoneauth1==4.4.0; python_version >= "3.6" kombu==5.1.0; python_version >= "3.6" lagom==1.4.1; python_version >= "3.6" and python_version < "4.0" diff --git a/lib/galaxy_test/api/cwl/__init__.py b/lib/galaxy_test/api/cwl/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lib/galaxy_test/api/test_workflows_cwl.py b/lib/galaxy_test/api/test_workflows_cwl.py new file mode 100644 index 000000000000..d5da915eb294 --- /dev/null +++ b/lib/galaxy_test/api/test_workflows_cwl.py @@ -0,0 +1,14 @@ +"""Test CWL workflow functionality.""" +from galaxy_test.api.test_workflows import BaseWorkflowsApiTestCase +from galaxy_test.base.populators import CwlPopulator + + +class BaseCwlWorkflowTestCase(BaseWorkflowsApiTestCase): + allow_path_paste = True + require_admin_user = True + + def setUp(self): + super().setUp() + self.cwl_populator = CwlPopulator( + self.dataset_populator, self.workflow_populator + ) diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index f34e4f510274..d1fce9d5d7b8 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -59,6 +59,7 @@ Tuple, ) +import cwltest import requests import yaml from bioblend.galaxy import GalaxyClient @@ -71,6 +72,7 @@ from requests.models import Response from galaxy.tool_util.client.staging import InteractorStaging +from galaxy.tool_util.cwl.util import guess_artifact_type from galaxy.tool_util.verify.test_data import TestDataResolver from galaxy.tool_util.verify.wait import ( timeout_type, @@ -79,12 +81,15 @@ ) from galaxy.util import ( DEFAULT_SOCKET_TIMEOUT, + galaxy_root_path, unicodify, ) from . import api_asserts from .api import ApiTestInteractor +CWL_TOOL_DIRECTORY = os.path.join(galaxy_root_path, "test", "functional", "tools", "cwl_tools") + # Simple workflow that takes an input and call cat wrapper on it. workflow_str = unicodify(resource_string(__name__, "data/test_workflow_1.ga")) # Simple workflow that takes an input and filters with random lines twice in a @@ -221,6 +226,212 @@ def _raise_skip_if(check, *args): raise SkipTest(*args) +def load_conformance_tests(directory, path="conformance_tests.yaml"): + conformance_tests_path = os.path.join(directory, path) + with open(conformance_tests_path) as f: + conformance_tests = yaml.safe_load(f) + + expanded_conformance_tests = [] + for conformance_test in conformance_tests: + if "$import" in conformance_test: + import_path = conformance_test["$import"] + expanded_conformance_tests.extend(load_conformance_tests(directory, import_path)) + else: + subdirectory = os.path.dirname(path) + if subdirectory: + conformance_test["relative_path"] = os.path.join(directory, subdirectory) + expanded_conformance_tests.append(conformance_test) + return expanded_conformance_tests + + +class CwlRun: + + def __init__(self, dataset_populator, history_id): + self.dataset_populator = dataset_populator + self.history_id = history_id + + +class CwlToolRun(CwlRun): + + def __init__(self, dataset_populator, history_id, run_response): + super().__init__(dataset_populator, history_id) + self.run_response = run_response + + @property + def job_id(self): + return self.run_response["jobs"][0]["id"] + + def wait(self): + final_state = self.dataset_populator.wait_for_job(self.job_id) + assert final_state == "ok" + + +class CwlWorkflowRun(CwlRun): + + def __init__(self, dataset_populator, workflow_populator, history_id, workflow_id, invocation_id): + super().__init__(dataset_populator, history_id) + self.workflow_populator = workflow_populator + self.workflow_id = workflow_id + self.invocation_id = invocation_id + + def wait(self): + self.workflow_populator.wait_for_invocation_and_jobs( + self.history_id, self.workflow_id, self.invocation_id + ) + + +class CwlPopulator: + + def __init__(self, dataset_populator, workflow_populator): + self.dataset_populator = dataset_populator + self.workflow_populator = workflow_populator + + def get_conformance_test(self, version, doc): + conformance_tests = load_conformance_tests(os.path.join(CWL_TOOL_DIRECTORY, str(version))) + for test in conformance_tests: + if test.get("doc") == doc: + return test + raise Exception(f"doc [{doc}] not found") + + def _run_cwl_tool_job( + self, + tool_id: str, + job: dict, + history_id: str, + assert_ok: bool = True, + ): + galaxy_tool_id: Optional[str] = tool_id + tool_uuid = None + + if os.path.exists(tool_id): + raw_tool_id = os.path.basename(tool_id) + index = self.dataset_populator._get("tools", data=dict(in_panel=False)) + tools = index.json() + # In panels by default, so flatten out sections... + tool_ids = [itemgetter("id")(_) for _ in tools] + if raw_tool_id in tool_ids: + galaxy_tool_id = raw_tool_id + else: + dynamic_tool = self.dataset_populator.create_tool_from_path(tool_id) + galaxy_tool_id = None + tool_uuid = dynamic_tool["uuid"] + + run_response = self.dataset_populator.run_tool(galaxy_tool_id, job, history_id, assert_ok=assert_ok, tool_uuid=tool_uuid) + run_object = CwlToolRun(self.dataset_populator, history_id, run_response) + if assert_ok: + try: + final_state = self.dataset_populator.wait_for_job(run_object.job_id) + assert final_state == "ok" + except Exception: + self.dataset_populator._summarize_history(history_id) + raise + + return run_object + + def _run_cwl_workflow_job( + self, + workflow_path: str, + job: dict, + history_id: str, + assert_ok: bool = True, + ): + object_id = None + if "#" in workflow_path: + workflow_path, object_id = workflow_path.split("#", 1) + workflow_id = self.workflow_populator.import_workflow_from_path(workflow_path, object_id) + + request = { + # TODO: rework tool state corrections so more of these are valid in Galaxy + # workflows as well, and then make it the default. Or decide they are safe. + "allow_tool_state_corrections": True, + } + invocation_id = self.workflow_populator.invoke_workflow(workflow_id, history_id=history_id, inputs=job, request=request, inputs_by="name") + return CwlWorkflowRun(self.dataset_populator, self.workflow_populator, history_id, workflow_id, invocation_id) + + def run_cwl_job( + self, + artifact_path: str, + job_path: Optional[str] = None, + job: Optional[Dict] = None, + test_data_directory: Optional[str] = None, + history_id: Optional[str] = None, + assert_ok=True, + ): + if history_id is None: + history_id = self.dataset_populator.new_history() + if not os.path.isabs(artifact_path): + artifact_path = os.path.join(CWL_TOOL_DIRECTORY, artifact_path) + tool_or_workflow = guess_artifact_type(artifact_path) + if job_path and not os.path.isabs(job_path): + job_path = os.path.join(CWL_TOOL_DIRECTORY, job_path) + if test_data_directory is None and job_path is not None: + test_data_directory = os.path.dirname(job_path) + if job_path is not None: + assert job is None + with open(job_path) as f: + if job_path.endswith(".yml") or job_path.endswith(".yaml"): + job = yaml.safe_load(f) + else: + job = json.load(f) + elif job is None: + job = {} + _, datasets_uploaded = stage_inputs( + self.dataset_populator.galaxy_interactor, + history_id, + job, + use_fetch_api=False, + tool_or_workflow=tool_or_workflow, + job_dir=test_data_directory, + ) + if datasets_uploaded: + self.dataset_populator.wait_for_history(history_id=history_id, assert_ok=True) + if tool_or_workflow == "tool": + run_object = self._run_cwl_tool_job( + artifact_path, + job, + history_id, + assert_ok=assert_ok, + ) + else: + run_object = self._run_cwl_workflow_job( + artifact_path, + job, + history_id, + assert_ok=assert_ok, + ) + run_object.wait() + return run_object + + def run_conformance_test(self, version, doc): + test = self.get_conformance_test(version, doc) + directory = os.path.join(CWL_TOOL_DIRECTORY, version) + artifact = os.path.join(directory, test["tool"]) + job_path = test.get("job") + if job_path is not None: + job_path = os.path.join(directory, job_path) + should_fail = test.get("should_fail", False) + try: + run = self.run_cwl_job(artifact, job_path=job_path) + except Exception: + # Should fail so this is good! + if should_fail: + return True + raise + + if should_fail: + self.dataset_populator._summarize_history(run.history_id) + raise Exception("Expected run to fail but it didn't.") + + expected_outputs = test["output"] + try: + for key, value in expected_outputs.items(): + actual_output = run.get_output_as_object(key) + cwltest.compare(value, actual_output) + except Exception: + self.dataset_populator._summarize_history(run.history_id) + raise + + class BasePopulator(metaclass=ABCMeta): @abstractmethod diff --git a/pyproject.toml b/pyproject.toml index 8323d933c1a0..055f086bc10e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,7 @@ Whoosh = "*" zipstream-new = "*" [tool.poetry.dev-dependencies] +cwltest = "2.2.20210901154959" fluent-logger = "*" httpx = "*" gunicorn = "*" diff --git a/pytest.ini b/pytest.ini index 913bc72676b6..2b5fb07fe04e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,5 +6,37 @@ markers = data_manager: marks test as a data_manager test tool: marks test as a tool test gtn_screenshot: marks test as a screenshot producer for galaxy training network - local: mark indicates, that it's sufficient to run test locally to get relevant artifacts (e.g. screenshots) + local: mark indicates, that it is sufficient to run test locally to get relevant artifacts (e.g. screenshots) external: mark indicates, that test has to be run against external production server to get relevant artifacts (e.g. screenshots) + green: test expected to pass + red: test expected to fail + command_line_tool: CWL conformance test tagged command_line_tool + conditional: CWL conformance test tagged conditional + docker: CWL conformance test tagged docker + env_var: CWL conformance test tagged env_var + expression_tool: CWL conformance test tagged expression_tool + format_checking: CWL conformance test tagged format_checking + initial_work_dir: CWL conformance test tagged initial_work_dir + inline_javascript: CWL conformance test tagged inline_javascript + inplace_update: CWL conformance test tagged inplace_update + input_object_requirements: CWL conformance test tagged input_object_requirements + load_listing: CWL conformance test tagged load_listing + multiple: CWL conformance test tagged multiple + multiple_input: CWL conformance test tagged multiple_input + networkaccess: CWL conformance test tagged networkaccess + required: required for CWL conformance + resource: CWL conformance test tagged resource + scatter: CWL conformance test tagged scatter + schema_def: CWL conformance test tagged schema_def + secondary_files: CWL conformance test tagged secondary_files + shell_command: CWL conformance test tagged shell_command + step_input: CWL conformance test tagged step_input + step_input_expression: CWL conformance test tagged step_input_expression + subworkflow: CWL conformance test tagged subworkflow + timelimit: CWL conformance test tagged timelimit + workflow: CWL conformance test tagged workflow + work_reuse: CWL conformance test tagged work_reuse + cwl_conformance: all CWL conformance tests + cwl_conformance_v1_0: CWL v1.0 conformance tests + cwl_conformance_v1_1: CWL v1.1 conformance tests + cwl_conformance_v1_2: CWL v1.2 conformance tests diff --git a/run_tests.sh b/run_tests.sh index e1023af1d56c..fb801eb6874d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -11,6 +11,8 @@ cat < Date: Tue, 23 Nov 2021 00:45:42 +0000 Subject: [PATCH 7/7] Temporarily skip CWL conformance GH workflow --- .github/workflows/cwl_conformance.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cwl_conformance.yaml b/.github/workflows/cwl_conformance.yaml index 66bafe046965..9e5cf7bdbb7c 100644 --- a/.github/workflows/cwl_conformance.yaml +++ b/.github/workflows/cwl_conformance.yaml @@ -16,6 +16,7 @@ concurrency: jobs: test: name: Test + if: ${{ false }} runs-on: ubuntu-latest continue-on-error: ${{ startsWith(matrix.marker, 'red') }} strategy: