Add a test flag to force tests not to skip on required tools.

I will occasionally see tests that skip instead of failing because the tool required to run the test stopped loading - maybe a parsing error or a misconfiguration around sample data tables. I don't think this should be the default but we should make sure all our CI tests are running properly.

lib/galaxy_test/api/conftest.py

@@ -22,7 +22,7 @@
 )
 from galaxy_test.base.env import setup_keep_outdir
 from galaxy_test.base.populators import (
-    _raise_skip_if,
+    check_missing_tool,
     DatasetCollectionPopulator,
     DatasetPopulator,
     get_tool_ids,
@@ -148,7 +148,7 @@ def check_required_tools(anonymous_galaxy_interactor, request):
     for marker in request.node.iter_markers():
         if marker.name == "requires_tool_id":
             tool_id = marker.args[0]
-            _raise_skip_if(tool_id not in get_tool_ids(anonymous_galaxy_interactor))
+            check_missing_tool(tool_id not in get_tool_ids(anonymous_galaxy_interactor))
 
 
 @pytest.fixture

lib/galaxy_test/base/env.py

@@ -10,7 +10,10 @@
     Tuple,
 )
 
+from galaxy.util import asbool
+
 DEFAULT_WEB_HOST = socket.gethostbyname("localhost")
+REQUIRE_ALL_NEEDED_TOOLS = asbool(os.environ.get("GALAXY_TEST_REQUIRE_ALL_NEEDED_TOOLS", "0"))
 
 GalaxyTarget = Tuple[str, Optional[str], str]
 

lib/galaxy_test/base/populators.py

@@ -128,6 +128,7 @@
     HasAnonymousGalaxyInteractor,
 )
 from .api_util import random_name
+from .env import REQUIRE_ALL_NEEDED_TOOLS
 
 FILE_URL = "https://raw.githubusercontent.com/galaxyproject/galaxy/dev/test-data/4.bed"
 FILE_MD5 = "37b59762b59fff860460522d271bc111"
@@ -180,7 +181,7 @@ def method_wrapper(method):
 
         @wraps(method)
         def wrapped_method(api_test_case, *args, **kwargs):
-            _raise_skip_if(tool_id not in get_tool_ids(api_test_case.anonymous_galaxy_interactor))
+            check_missing_tool(tool_id not in get_tool_ids(api_test_case.anonymous_galaxy_interactor))
             return method(api_test_case, *args, **kwargs)
 
         return wrapped_method
@@ -268,6 +269,16 @@ def _raise_skip_if(check, *args):
         raise unittest.SkipTest(*args)
 
 
+def check_missing_tool(check):
+    if check:
+        if REQUIRE_ALL_NEEDED_TOOLS:
+            raise AssertionError("Test requires a missing tool and GALAXY_TEST_REQUIRE_ALL_NEEDED_TOOLS is enabled")
+        else:
+            raise unittest.SkipTest(
+                "Missing tool required to run test, skipping. If this is not intended, ensure GALAXY_TEST_TOOL_CONF if set contains the required tool_conf.xml target and the tool properly parses and loads in Galaxy's test configuration"
+            )
+
+
 def conformance_tests_gen(directory, filename="conformance_tests.yaml"):
     conformance_tests_path = os.path.join(directory, filename)
     with open(conformance_tests_path) as f: