Implement a subset of the Common Workflow Language.

.ci/flake8_ignorelist.txt

@@ -15,5 +15,6 @@ lib/galaxy/util/jstree.py
 lib/galaxy/web/proxy/js/node_modules
 static/maps
 static/scripts
+test/functional/tools/cwl_tools/v1.?/
 build
 dist

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ cwl-1.0 ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ cwl-1.0 ]
+  schedule:
+    - cron: '16 6 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ['javascript', 'python']
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/cwl_conformance.yaml

@@ -0,0 +1,65 @@
+name: CWL conformance
+on:
+  push:
+    paths-ignore:
+      - 'client/**'
+      - 'doc/**'
+  pull_request:
+    paths-ignore:
+      - 'client/**'
+      - 'doc/**'
+env:
+  GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
+concurrency:
+  group: cwl-conformance-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.marker == 'red' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7']
+        marker: ['green', 'red']
+        required: ['required', 'not required']
+        conformance-version: ['cwl_conformance_v1_0', 'cwl_conformance_v1_1', 'cwl_conformance_v1_2']
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: 'galaxy root'
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Get full Python version
+        id: full-python-version
+        shell: bash
+        run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
+      - name: Cache pip dir
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Cache galaxy venv
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Run tests
+        run: ./run_tests.sh --skip_flakey_fails -cwl lib/galaxy_test/api/cwl -- -m "${{ matrix.marker }} and ${{ matrix.required }} and ${{ matrix.conformance-version }}"
+        working-directory: 'galaxy root'
+      - uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: CWL conformance test results (${{ matrix.python-version }}, ${{ matrix.marker }}, ${{ matrix.required }}, ${{ matrix.conformance-version }})
+          path: 'galaxy root/run_cwl_tests.html'

.gitignore

@@ -125,12 +125,13 @@ tool-data/**/*.fa
 .pytest_cache/
 assets/
 test-data-cache
+run_api_tests.html
+run_cwl_tests.html
 run_framework_tests.html
 run_functional_tests.html
 run_integration_tests.html
 run_selenium_tests.html
 run_toolshed_tests.html
-run_api_tests.html
 test/tool_shed/tmp/*
 .coverage
 htmlcov
@@ -187,3 +188,7 @@ config/plugins/**/static/dist
 config/plugins/**/static/script.js
 config/plugins/**/static/main.css
 config/plugins/**/static/plugin_build_hash.txt
+
+# CWL conformance tests
+lib/galaxy_test/api/cwl/test_cwl_conformance_v1_?.py
+test/functional/tools/cwl_tools/v1.?/

Makefile

@@ -13,6 +13,12 @@ OPEN_RESOURCE=bash -c 'open $$0 || xdg-open $$0'
 SLIDESHOW_TO_PDF?=bash -c 'docker run --rm -v `pwd`:/cwd astefanutti/decktape /cwd/$$0 /cwd/`dirname $$0`/`basename -s .html $$0`.pdf'
 YARN := $(shell command -v yarn 2> /dev/null)
 YARN_INSTALL_OPTS=--network-timeout 300000 --check-files
+CWL_TARGETS := test/functional/tools/cwl_tools/v1.0/conformance_tests.yaml \
+	test/functional/tools/cwl_tools/v1.1/conformance_tests.yaml \
+	test/functional/tools/cwl_tools/v1.2/conformance_tests.yaml \
+	lib/galaxy_test/api/cwl/test_cwl_conformance_v1_0.py \
+	lib/galaxy_test/api/cwl/test_cwl_conformance_v1_1.py \
+	lib/galaxy_test/api/cwl/test_cwl_conformance_v1_2.py
 
 all: help
 	@echo "This makefile is used for building Galaxy's JS client, documentation, and drive the release process. A sensible all target is not implemented."
@@ -130,6 +136,14 @@ update-lint-requirements:
 update-dependencies: update-lint-requirements ## update pinned and dev dependencies
 	$(IN_VENV) ./lib/galaxy/dependencies/update.sh
 
+update-cwl-conformance-tests: ## update CWL conformance tests
+	./scripts/update_cwl_conformance_tests.sh
+
+generate-cwl-conformance-tests: $(CWL_TARGETS)  ## Initialise CWL conformance tests
+
+$(CWL_TARGETS):
+	./scripts/update_cwl_conformance_tests.sh
+
 node-deps: ## Install NodeJS dependencies.
 ifndef YARN
 	@echo "Could not find yarn, which is required to build the Galaxy client.\nTo install yarn, please visit \033[0;34mhttps://yarnpkg.com/en/docs/install\033[0m for instructions, and package information for all platforms.\n"

client/src/components/Form/Elements/parameters.js

@@ -5,6 +5,7 @@ import Backbone from "backbone";
 import { getGalaxyInstance } from "app";
 import Utils from "utils/utils";
 import Ui from "mvc/ui/ui-misc";
+import UiField from "mvc/ui/ui-field";
 import SelectContent from "mvc/ui/ui-select-content";
 import SelectLibrary from "mvc/ui/ui-select-library";
 import SelectFtp from "mvc/ui/ui-select-ftp";
@@ -35,6 +36,7 @@ export default Backbone.View.extend({
         rules: "_fieldRulesEdit",
         directory_uri: "_fieldDirectoryUri",
         data_dialog: "_fieldDialog",
+        field: "_fieldField",
     },
 
     /** Returns an input field for a given field type */
@@ -245,4 +247,11 @@ export default Backbone.View.extend({
             onchange: input_def.onchange,
         });
     },
+
+    _fieldField: function (input_def) {
+        return new UiField({
+            id: `field-${input_def.id}`,
+            onchange: input_def.onchange,
+        });
+    },
 });

client/src/mvc/ui/ui-field.js

@@ -0,0 +1,97 @@
+/** Renders the color picker used e.g. in the tool form **/
+import Utils from "utils/utils";
+import Ui from "mvc/ui/ui-misc";
+
+/** Renders an input element used e.g. in the tool form */
+export default Backbone.View.extend({
+    initialize: function (options) {
+        this.model =
+            (options && options.model) ||
+            new Backbone.Model({
+                value: { src: "json", value: null, representation: "null" },
+            }).set(options);
+        this.$el = $("<div><p>moo cow</p></div>").addClass("ui-field");
+        console.log(this.model.get("value"));
+        var menuButton = new Ui.ButtonMenu({
+            id: "options",
+            icon: "fa-caret-down",
+            title: "Input Type",
+            tooltip: "View available input type options",
+        });
+        menuButton.addMenu({
+            title: "Integer",
+            onclick: () => {
+                this._changeType("integer");
+            },
+        });
+        menuButton.addMenu({
+            title: "Leave Unselected",
+            onclick: () => {
+                this._changeType("null");
+            },
+        });
+        this.$menuButton = menuButton;
+        this.$inputDiv = $("<div/>").addClass("select-input");
+
+        this.$el.append(menuButton.$el);
+        this.$el.append(this.$inputDiv);
+        this.setElement(this.$el);
+        this.listenTo(this.model, "change", this.render, this);
+        this.render();
+    },
+    value: function (new_val) {
+        var options = this.model.attributes;
+        if (new_val) {
+            this.model.set("value", new_val);
+            this.model.trigger("change");
+            options.onchange(new_val);
+        }
+        return this.model.get("value");
+    },
+    render: function () {
+        const value = this.model.get("value");
+        const rep = value.representation;
+        if (rep == "null") {
+            this.$inputDiv.html($("<p>No value selected (null)</p>"));
+        } else if (rep == "integer") {
+            const tagName = this.model.get("area") ? "textarea" : "input";
+            this.$inputDiv.html($(`<${tagName} value="${value.value}"/>`));
+            console.log(this.$inputDiv.find("input"));
+            this.$inputDiv.find("input").on("change", () => {
+                this._onchange();
+            });
+        }
+        return this;
+    },
+    _changeType: function (representation) {
+        const previousValue = this.model.get("value");
+        const previousRawValue = previousValue.value;
+        if (representation == "null") {
+            this.model.set("value", { src: "json", value: null, representation: "null" });
+        } else if (representation == "integer") {
+            var value = parseInt(previousRawValue);
+            if (isNaN(value)) {
+                value = 0;
+            }
+            this.model.set("value", { src: "json", value: 0, representation: "integer" });
+        }
+    },
+    _rawValue: function (previousValue) {
+        const rep = previousValue.representation;
+        let rawVal;
+        if (rep == "null") {
+            rawVal = null;
+        } else if (rep == "integer") {
+            rawVal = parseInt(this.$inputDiv.find("input").val());
+        }
+        console.log("_rawValue returning " + rawVal);
+        return rawVal;
+    },
+    _onchange: function () {
+        const previousValue = this.model.get("value");
+        const newValue = this._rawValue(previousValue);
+        previousValue["value"] = newValue;
+        this.value(previousValue);
+        this.model.get("onchange") && this.model.get("onchange")(this.model.get("value"));
+    },
+});

lib/galaxy/app_unittest_utils/tools_support.py

@@ -83,14 +83,18 @@ def _init_tool(
         tool_id="test_tool",
         extra_file_contents=None,
         extra_file_path=None,
+        tool_path=None,
     ):
+        if tool_path is None:
+            self.tool_file = os.path.join(self.test_directory, filename)
+            contents_template = string.Template(tool_contents or SIMPLE_TOOL_CONTENTS)
+            tool_contents = contents_template.safe_substitute(dict(version=version, profile=profile, tool_id=tool_id))
+            self.__write_tool(tool_contents)
+            if extra_file_contents and extra_file_path:
+                self.__write_tool(extra_file_contents, path=os.path.join(self.test_directory, extra_file_path))
+        else:
+            self.tool_file = tool_path
         self._init_app_for_tools()
-        self.tool_file = os.path.join(self.test_directory, filename)
-        contents_template = string.Template(tool_contents)
-        tool_contents = contents_template.safe_substitute(dict(version=version, profile=profile, tool_id=tool_id))
-        self.__write_tool(tool_contents)
-        if extra_file_contents and extra_file_path:
-            self.__write_tool(extra_file_contents, path=os.path.join(self.test_directory, extra_file_path))
         return self.__setup_tool()
 
     def _init_tool_for_path(self, tool_file):

lib/galaxy/config/__init__.py

@@ -685,6 +685,9 @@ def _process_config(self, kwargs):
         self.email_domain_blocklist_content = self._load_list_from_file(self._in_config_dir(self.email_domain_blocklist_file)) if self.email_domain_blocklist_file else None
         self.email_domain_allowlist_content = self._load_list_from_file(self._in_config_dir(self.email_domain_allowlist_file)) if self.email_domain_allowlist_file else None
 
+        # TODO: migrate to schema.
+        # Should CWL artifacts be loaded with strict validation enabled.
+        self.strict_cwl_validation = string_as_bool(kwargs.get('strict_cwl_validation', 'True'))
         # These are not even beta - just experiments - don't use them unless
         # you want yours tools to be broken in the future.
         self.enable_beta_tool_formats = string_as_bool(kwargs.get('enable_beta_tool_formats', 'False'))

lib/galaxy/config/sample/datatypes_conf.xml.sample

@@ -246,7 +246,7 @@
     <datatype extension="tar" type="galaxy.datatypes.binary:CompressedArchive" subclass="true" display_in_upload="true">
       <converter file="tar_to_directory.xml" target_datatype="directory"/>
     </datatype>
-    <datatype extension="directory" type="galaxy.datatypes.data:Directory">
+    <datatype extension="directory" type="galaxy.datatypes.data:Directory" display_in_upload="true">
     </datatype>
     <datatype extension="yaml" type="galaxy.datatypes.text:Yaml" display_in_upload="true" />
     <!-- Proteomics Datatypes -->

lib/galaxy/datatypes/converters/tar_to_directory.xml

@@ -4,15 +4,21 @@
         <requirement type="package" version="19.9">galaxy-util</requirement>
     </requirements>
     <command>
+        cp '$provided_metadata' 'galaxy.json';
         mkdir '$output1.files_path';
         cd '$output1.files_path';
         python -c "from galaxy.util.compression_utils import CompressedFile; CompressedFile('$input1').extract('.');"
+        ls '$output1.files_path'
     </command>
+    <configfiles>
+        <configfile name="provided_metadata">{"output1": {"created_from_basename": "${input1.created_from_basename}"}}
+        </configfile>
+    </configfiles>
     <inputs>
         <param format="tar" name="input1" type="data"/>
     </inputs>
     <outputs>
-        <data format="directory" name="output1"/>
+        <data format="directory" name="output1" metadata_source="input1" />
     </outputs>
     <tests>
         <test>