Add CWL conformance API tests

1) Download conformance tests to `test/functional/tools/cwl_tools` where
  they can be used also by framework and unit tests in the future.
2) Add populators to enable executing CWL conformance tests
3) Generate CWL conformance API tests only when needed for tests

.ci/flake8_ignorelist.txt

@@ -15,5 +15,6 @@ lib/galaxy/util/jstree.py
 lib/galaxy/web/proxy/js/node_modules
 static/maps
 static/scripts
+test/functional/tools/cwl_tools/v1.?/
 build
 dist

.github/workflows/cwl_conformance.yaml

@@ -0,0 +1,64 @@
+name: CWL conformance
+on:
+  push:
+    paths-ignore:
+      - 'client/**'
+      - 'doc/**'
+  pull_request:
+    paths-ignore:
+      - 'client/**'
+      - 'doc/**'
+env:
+  GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
+concurrency:
+  group: cwl-conformance-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ startsWith(matrix.marker, 'red') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7']
+        marker: ['green', 'red and required', 'red and not required']
+        conformance-version: ['cwl_conformance_v1_0'] #, 'cwl_conformance_v1_1', 'cwl_conformance_v1_2']
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: 'galaxy root'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
+      - name: Cache pip dir
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Run tests
+        run: ./run_tests.sh --skip_flakey_fails -cwl lib/galaxy_test/api/cwl -- -m "${{ matrix.marker }} and ${{ matrix.conformance-version }}"
+        working-directory: 'galaxy root'
+      - uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: CWL conformance test results (${{ matrix.python-version }}, ${{ matrix.marker }}, ${{ matrix.conformance-version }})
+          path: 'galaxy root/run_cwl_tests.html'

.gitignore

@@ -125,12 +125,13 @@ tool-data/**/*.fa
 .pytest_cache/
 assets/
 test-data-cache
+run_api_tests.html
+run_cwl_tests.html
 run_framework_tests.html
 run_functional_tests.html
 run_integration_tests.html
 run_selenium_tests.html
 run_toolshed_tests.html
-run_api_tests.html
 test/tool_shed/tmp/*
 .coverage
 htmlcov
@@ -192,3 +193,7 @@ config/plugins/**/static/*.map
 # viz-specific build artifacts to ignore (until these are removed from codebase)
 config/plugins/visualizations/annotate_image/static/jquery.contextMenu.css
 config/plugins/visualizations/nvd3/nvd3_bar/static/nvd3.js
+
+# CWL conformance tests
+lib/galaxy_test/api/cwl/test_cwl_conformance_v1_?.py
+test/functional/tools/cwl_tools/v1.?/

Makefile

@@ -13,6 +13,12 @@ OPEN_RESOURCE=bash -c 'open $$0 || xdg-open $$0'
 SLIDESHOW_TO_PDF?=bash -c 'docker run --rm -v `pwd`:/cwd astefanutti/decktape /cwd/$$0 /cwd/`dirname $$0`/`basename -s .html $$0`.pdf'
 YARN := $(shell command -v yarn 2> /dev/null)
 YARN_INSTALL_OPTS=--network-timeout 300000 --check-files
+CWL_TARGETS := test/functional/tools/cwl_tools/v1.0/conformance_tests.yaml \
+	test/functional/tools/cwl_tools/v1.1/conformance_tests.yaml \
+	test/functional/tools/cwl_tools/v1.2/conformance_tests.yaml \
+	lib/galaxy_test/api/cwl/test_cwl_conformance_v1_0.py \
+	lib/galaxy_test/api/cwl/test_cwl_conformance_v1_1.py \
+	lib/galaxy_test/api/cwl/test_cwl_conformance_v1_2.py
 
 all: help
 	@echo "This makefile is used for building Galaxy's JS client, documentation, and drive the release process. A sensible all target is not implemented."
@@ -130,6 +136,24 @@ update-lint-requirements:
 update-dependencies: update-lint-requirements ## update pinned and dev dependencies
 	$(IN_VENV) ./lib/galaxy/dependencies/update.sh
 
+$(CWL_TARGETS):
+	./scripts/update_cwl_conformance_tests.sh
+
+generate-cwl-conformance-tests: $(CWL_TARGETS)  ## Initialise CWL conformance tests
+
+clean-cwl-conformance-tests:  # Clean CWL conformance tests
+	for f in $(CWL_TARGETS); do \
+		if [ $$(basename "$$f") = conformance_tests.yaml ]; then \
+			rm -rf $$(dirname "$$f"); \
+		else \
+			rm -f "$$f"; \
+		fi \
+	done
+
+update-cwl-conformance-tests: ## update CWL conformance tests
+	$(MAKE) clean-cwl-conformance-tests
+	$(MAKE) generate-cwl-conformance-tests
+
 node-deps: ## Install NodeJS dependencies.
 ifndef YARN
 	@echo "Could not find yarn, which is required to build the Galaxy client.\nTo install yarn, please visit \033[0;34mhttps://yarnpkg.com/en/docs/install\033[0m for instructions, and package information for all platforms.\n"

lib/galaxy/dependencies/dev-requirements.txt

@@ -50,11 +50,12 @@ commonmark==0.9.1; python_version >= "3.6" and python_version < "4.0"
 contextvars==2.4; python_version < "3.7" and python_version >= "3.6"
 coverage==6.1.1; python_version >= "3.6"
 cryptography==35.0.0; python_version >= "3.6"
+cwltest==2.2.20210901154959; python_version >= "3.6" and python_version < "4"
 cwltool==3.1.20211107152837; python_version >= "3.6" and python_version < "4"
 dataclasses==0.8; python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.6.1"
 debtcollector==2.3.0; python_version >= "3.6"
 decorator==5.1.0; python_version >= "3.6"
-defusedxml==0.7.1; python_version >= "3.0" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.0"
+defusedxml==0.7.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4"
 deprecated==1.2.13; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
 deprecation==2.1.0
 dictobj==0.4
@@ -99,6 +100,7 @@ jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or p
 jsonpatch==1.32; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 jsonpointer==2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
 jsonschema==4.0.0
+junit-xml==1.9; python_version >= "3.6" and python_version < "4"
 keystoneauth1==4.4.0; python_version >= "3.6"
 kombu==5.1.0; python_version >= "3.6"
 lagom==1.4.1; python_version >= "3.6" and python_version < "4.0"

lib/galaxy_test/api/test_workflows_cwl.py

@@ -0,0 +1,14 @@
+"""Test CWL workflow functionality."""
+from galaxy_test.api.test_workflows import BaseWorkflowsApiTestCase
+from galaxy_test.base.populators import CwlPopulator
+
+
+class BaseCwlWorkflowTestCase(BaseWorkflowsApiTestCase):
+    allow_path_paste = True
+    require_admin_user = True
+
+    def setUp(self):
+        super().setUp()
+        self.cwl_populator = CwlPopulator(
+            self.dataset_populator, self.workflow_populator
+        )