Add CWL conformance API tests

.ci/flake8_ignorelist.txt

@@ -15,5 +15,6 @@ lib/galaxy/util/jstree.py
 lib/galaxy/web/proxy/js/node_modules
 static/maps
 static/scripts
+test/functional/tools/cwl_tools/v1.?/
 build
 dist

.github/workflows/cwl_conformance.yaml

@@ -0,0 +1,65 @@
+name: CWL conformance
+on:
+  push:
+    paths-ignore:
+      - 'client/**'
+      - 'doc/**'
+  pull_request:
+    paths-ignore:
+      - 'client/**'
+      - 'doc/**'
+env:
+  GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
+concurrency:
+  group: cwl-conformance-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  test:
+    name: Test
+    if: ${{ false }}
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ startsWith(matrix.marker, 'red') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7']
+        marker: ['green', 'red and required', 'red and not required']
+        conformance-version: ['cwl_conformance_v1_0'] #, 'cwl_conformance_v1_1', 'cwl_conformance_v1_2']
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: 'galaxy root'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
+      - name: Cache pip dir
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Run tests
+        run: ./run_tests.sh --skip_flakey_fails -cwl lib/galaxy_test/api/cwl -- -m "${{ matrix.marker }} and ${{ matrix.conformance-version }}"
+        working-directory: 'galaxy root'
+      - uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: CWL conformance test results (${{ matrix.python-version }}, ${{ matrix.marker }}, ${{ matrix.conformance-version }})
+          path: 'galaxy root/run_cwl_tests.html'

.gitignore

@@ -125,12 +125,13 @@ tool-data/**/*.fa
 .pytest_cache/
 assets/
 test-data-cache
+run_api_tests.html
+run_cwl_tests.html
 run_framework_tests.html
 run_functional_tests.html
 run_integration_tests.html
 run_selenium_tests.html
 run_toolshed_tests.html
-run_api_tests.html
 test/tool_shed/tmp/*
 .coverage
 htmlcov
@@ -192,3 +193,7 @@ config/plugins/**/static/*.map
 # viz-specific build artifacts to ignore (until these are removed from codebase)
 config/plugins/visualizations/annotate_image/static/jquery.contextMenu.css
 config/plugins/visualizations/nvd3/nvd3_bar/static/nvd3.js
+
+# CWL conformance tests
+lib/galaxy_test/api/cwl/test_cwl_conformance_v1_?.py
+test/functional/tools/cwl_tools/v1.?/

Makefile

@@ -13,6 +13,12 @@ OPEN_RESOURCE=bash -c 'open $$0 || xdg-open $$0'
 SLIDESHOW_TO_PDF?=bash -c 'docker run --rm -v `pwd`:/cwd astefanutti/decktape /cwd/$$0 /cwd/`dirname $$0`/`basename -s .html $$0`.pdf'
 YARN := $(shell command -v yarn 2> /dev/null)
 YARN_INSTALL_OPTS=--network-timeout 300000 --check-files
+CWL_TARGETS := test/functional/tools/cwl_tools/v1.0/conformance_tests.yaml \
+	test/functional/tools/cwl_tools/v1.1/conformance_tests.yaml \
+	test/functional/tools/cwl_tools/v1.2/conformance_tests.yaml \
+	lib/galaxy_test/api/cwl/test_cwl_conformance_v1_0.py \
+	lib/galaxy_test/api/cwl/test_cwl_conformance_v1_1.py \
+	lib/galaxy_test/api/cwl/test_cwl_conformance_v1_2.py
 
 all: help
 	@echo "This makefile is used for building Galaxy's JS client, documentation, and drive the release process. A sensible all target is not implemented."
@@ -130,6 +136,24 @@ update-lint-requirements:
 update-dependencies: update-lint-requirements ## update pinned and dev dependencies
 	$(IN_VENV) ./lib/galaxy/dependencies/update.sh
 
+$(CWL_TARGETS):
+	./scripts/update_cwl_conformance_tests.sh
+
+generate-cwl-conformance-tests: $(CWL_TARGETS)  ## Initialise CWL conformance tests
+
+clean-cwl-conformance-tests:  # Clean CWL conformance tests
+	for f in $(CWL_TARGETS); do \
+		if [ $$(basename "$$f") = conformance_tests.yaml ]; then \
+			rm -rf $$(dirname "$$f"); \
+		else \
+			rm -f "$$f"; \
+		fi \
+	done
+
+update-cwl-conformance-tests: ## update CWL conformance tests
+	$(MAKE) clean-cwl-conformance-tests
+	$(MAKE) generate-cwl-conformance-tests
+
 node-deps: ## Install NodeJS dependencies.
 ifndef YARN
 	@echo "Could not find yarn, which is required to build the Galaxy client.\nTo install yarn, please visit \033[0;34mhttps://yarnpkg.com/en/docs/install\033[0m for instructions, and package information for all platforms.\n"

lib/galaxy/dependencies/dev-requirements.txt

@@ -50,11 +50,12 @@ commonmark==0.9.1; python_version >= "3.6" and python_version < "4.0"
 contextvars==2.4; python_version < "3.7" and python_version >= "3.6"
 coverage==6.1.1; python_version >= "3.6"
 cryptography==35.0.0; python_version >= "3.6"
+cwltest==2.2.20210901154959; python_version >= "3.6" and python_version < "4"
 cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4"
 dataclasses==0.8; python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.6.1"
 debtcollector==2.3.0; python_version >= "3.6"
 decorator==5.1.0; python_version >= "3.6"
-defusedxml==0.7.1; python_version >= "3.0" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.0"
+defusedxml==0.7.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4"
 deprecated==1.2.13; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 deprecation==2.1.0
 dictobj==0.4
@@ -99,6 +100,7 @@ jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or p
 jsonpatch==1.32; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 jsonpointer==2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 jsonschema==4.0.0
+junit-xml==1.9; python_version >= "3.6" and python_version < "4"
 keystoneauth1==4.4.0; python_version >= "3.6"
 kombu==5.1.0; python_version >= "3.6"
 lagom==1.4.1; python_version >= "3.6" and python_version < "4.0"

lib/galaxy/managers/tools.py

@@ -77,7 +77,7 @@ def create_tool(self, trans, tool_payload, allow_load=True):
             tool_version = representation.get("version", None)
             value = representation
         else:
-            raise Exception("Unknown tool type encountered.")
+            raise Exception(f"Unknown tool format [{tool_format}] encountered.")
         # TODO: enforce via DB constraint and catch appropriate
         # exception.
         existing_tool = self.get_tool_by_uuid(uuid)

lib/galaxy/tool_util/client/staging.py

@@ -57,7 +57,7 @@ def _fetch_post(self, payload, files_attached=False):
     def _handle_job(self, job_response):
         """Implementer can decide if to wait for job(s) individually or not here."""
 
-    def stage(self, tool_or_workflow, history_id, job=None, job_path=None, use_path_paste=LOAD_TOOLS_FROM_PATH, to_posix_lines=True):
+    def stage(self, tool_or_workflow, history_id, job=None, job_path=None, use_path_paste=LOAD_TOOLS_FROM_PATH, to_posix_lines=True, job_dir="."):
         files_attached = [False]
 
         def upload_func_fetch(upload_target):
@@ -223,8 +223,7 @@ def create_collection_func(element_identifiers, collection_type):
             job_dir = os.path.dirname(os.path.abspath(job_path))
         else:
             assert job is not None
-            # Figure out what "." should be here instead.
-            job_dir = "."
+            assert job_dir is not None
 
         if self.use_fetch_api:
             upload = upload_func_fetch

lib/galaxy_test/api/test_workflows_cwl.py

@@ -0,0 +1,14 @@
+"""Test CWL workflow functionality."""
+from galaxy_test.api.test_workflows import BaseWorkflowsApiTestCase
+from galaxy_test.base.populators import CwlPopulator
+
+
+class BaseCwlWorkflowTestCase(BaseWorkflowsApiTestCase):
+    allow_path_paste = True
+    require_admin_user = True
+
+    def setUp(self):
+        super().setUp()
+        self.cwl_populator = CwlPopulator(
+            self.dataset_populator, self.workflow_populator
+        )