Merge remote-tracking branch 'upstream/release_24.0' into release_24.1

lib/galaxy/datatypes/dataproviders/dataset.py

@@ -15,6 +15,7 @@
 )
 
 from galaxy.util import sqlite
+from galaxy.util.compression_utils import get_fileobj
 from . import (
     base,
     column,
@@ -54,7 +55,7 @@ def __init__(self, dataset, **kwargs):
         # this dataset file is obviously the source
         # TODO: this might be a good place to interface with the object_store...
         mode = "rb" if dataset.datatype.is_binary else "r"
-        super().__init__(open(dataset.get_file_name(), mode))
+        super().__init__(get_fileobj(dataset.get_file_name(), mode))
 
     # TODO: this is a bit of a mess
     @classmethod

lib/galaxy/files/sources/_rdm.py

@@ -8,7 +8,6 @@
 
 from typing_extensions import Unpack
 
-from galaxy.exceptions import AuthenticationRequired
 from galaxy.files import OptionalUserContext
 from galaxy.files.sources import (
     BaseFilesSource,
@@ -201,15 +200,11 @@ def _serialization_props(self, user_context: OptionalUserContext = None):
             effective_props[key] = self._evaluate_prop(val, user_context=user_context)
         return effective_props
 
-    def get_authorization_token(self, user_context: OptionalUserContext) -> str:
+    def get_authorization_token(self, user_context: OptionalUserContext) -> Optional[str]:
         token = None
         if user_context:
             effective_props = self._serialization_props(user_context)
             token = effective_props.get("token")
-        if not token:
-            raise AuthenticationRequired(
-                f"Please provide a personal access token in your user's preferences for '{self.label}'"
-            )
         return token
 
     def get_public_name(self, user_context: OptionalUserContext) -> Optional[str]:

lib/galaxy/files/sources/invenio.py

@@ -275,12 +275,7 @@ def create_draft_record(
             },
         }
 
-        headers = self._get_request_headers(user_context)
-        if "Authorization" not in headers:
-            raise Exception(
-                "Cannot create record without authentication token. Please set your personal access token in your Galaxy preferences."
-            )
-
+        headers = self._get_request_headers(user_context, auth_required=True)
         response = requests.post(self.records_url, json=create_record_request, headers=headers)
         self._ensure_response_has_expected_status_code(response, 201)
         record = response.json()
@@ -296,7 +291,7 @@ def upload_file_to_draft_record(
     ):
         record = self._get_draft_record(record_id, user_context=user_context)
         upload_file_url = record["links"]["files"]
-        headers = self._get_request_headers(user_context)
+        headers = self._get_request_headers(user_context, auth_required=True)
 
         # Add file metadata entry
         response = requests.post(upload_file_url, json=[{"key": filename}], headers=headers)
@@ -452,28 +447,38 @@ def _get_creator_from_public_name(self, public_name: Optional[str] = None) -> Cr
         }
 
     def _get_response(
-        self, user_context: OptionalUserContext, request_url: str, params: Optional[Dict[str, Any]] = None
+        self,
+        user_context: OptionalUserContext,
+        request_url: str,
+        params: Optional[Dict[str, Any]] = None,
+        auth_required: bool = False,
     ) -> dict:
-        headers = self._get_request_headers(user_context)
+        headers = self._get_request_headers(user_context, auth_required)
         response = requests.get(request_url, params=params, headers=headers)
         self._ensure_response_has_expected_status_code(response, 200)
         return response.json()
 
-    def _get_request_headers(self, user_context: OptionalUserContext):
+    def _get_request_headers(self, user_context: OptionalUserContext, auth_required: bool = False):
         token = self.plugin.get_authorization_token(user_context)
         headers = {"Authorization": f"Bearer {token}"} if token else {}
+        if auth_required and token is None:
+            self._raise_auth_required()
         return headers
 
     def _ensure_response_has_expected_status_code(self, response, expected_status_code: int):
-        if response.status_code == 403:
-            record_url = response.url.replace("/api", "").replace("/files", "")
-            raise AuthenticationRequired(f"Please make sure you have the necessary permissions to access: {record_url}")
         if response.status_code != expected_status_code:
+            if response.status_code == 403:
+                self._raise_auth_required()
             error_message = self._get_response_error_message(response)
             raise Exception(
                 f"Request to {response.url} failed with status code {response.status_code}: {error_message}"
             )
 
+    def _raise_auth_required(self):
+        raise AuthenticationRequired(
+            f"Please provide a personal access token in your user's preferences for '{self.plugin.label}'"
+        )
+
     def _get_response_error_message(self, response):
         response_json = response.json()
         error_message = response_json.get("message") if response.status_code == 400 else response.text

lib/galaxy/managers/hdcas.py

@@ -45,7 +45,7 @@ def write_dataset_collection(dataset_collection_instance, archive):
         raise RequestParameterInvalidException("Attempt to write dataset collection that has not been populated yet")
     names, hdas = get_hda_and_element_identifiers(dataset_collection_instance)
     for name, hda in zip(names, hdas):
-        if hda.state != hda.states.OK:
+        if hda.state != hda.states.OK or hda.purged or hda.dataset.purged:
             continue
         for file_path, relpath in hda.datatype.to_archive(dataset=hda, name=name):
             archive.write(file_path, relpath)

lib/galaxy/model/__init__.py

@@ -6683,7 +6683,7 @@ def dataset_elements_and_identifiers(self, identifiers=None):
         return elements
 
     @property
-    def first_dataset_element(self):
+    def first_dataset_element(self) -> Optional["DatasetCollectionElement"]:
         for element in self.elements:
             if element.is_collection:
                 first_element = element.child_collection.first_dataset_element
@@ -10309,6 +10309,10 @@ class Page(Base, HasTags, Dictifiable, RepresentById, UsesCreateAndUpdateTime):
 
     def to_dict(self, view="element"):
         rval = super().to_dict(view=view)
+        if "importable" in rval and rval["importable"] is None:
+            # pages created prior to 2011 might not have importable field
+            # probably not worth creating a migration to fix that
+            rval["importable"] = False
         rev = []
         for a in self.revisions:
             rev.append(a.id)

lib/galaxy/objectstore/__init__.py

@@ -882,7 +882,12 @@ def _delete(self, obj, entire_dir=False, **kwargs):
         except OSError as ex:
             # Likely a race condition in which we delete the job working directory
             # and another process writes files into that directory.
-            log.critical(f"{self.__get_filename(obj, **kwargs)} delete error {ex}", exc_info=True)
+            # If the path doesn't exist anymore, another rmtree call was successful.
+            path = self.__get_filename(obj, **kwargs)
+            if path is None:
+                return True
+            else:
+                log.critical(f"{path} delete error {ex}", exc_info=True)
         return False
 
     def _get_data(self, obj, start=0, count=-1, **kwargs):

lib/galaxy/tools/__init__.py

@@ -3444,18 +3444,20 @@ def produce_outputs(self, trans, out_data, output_collections, incoming, history
         how = incoming["which"]["which_dataset"]
         if how == "first":
             extracted_element = collection.first_dataset_element
+            if not extracted_element:
+                raise exceptions.RequestParameterInvalidException("Input collection has no dataset elements.")
         elif how == "by_identifier":
             try:
                 extracted_element = collection[incoming["which"]["identifier"]]
             except KeyError as e:
-                raise exceptions.MessageException(e.args[0])
+                raise exceptions.RequestParameterInvalidException(e.args[0])
         elif how == "by_index":
             try:
                 extracted_element = collection[int(incoming["which"]["index"])]
             except KeyError as e:
-                raise exceptions.MessageException(e.args[0])
+                raise exceptions.RequestParameterInvalidException(e.args[0])
         else:
-            raise exceptions.MessageException("Invalid tool parameters.")
+            raise exceptions.RequestParameterInvalidException("Invalid tool parameters.")
         extracted = extracted_element.element_object
         extracted_o = extracted.copy(
             copy_tags=extracted.tags, new_name=extracted_element.element_identifier, flush=False

lib/tool_shed/webapp/model/migrations/alembic/versions/1b5bf427db25_add_non_nullable_column_deleted_to_api_.py

@@ -5,6 +5,7 @@
 Create Date: 2024-05-29 21:53:53.516506
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy import (

test/unit/tool_shed/_util.py

@@ -46,6 +46,7 @@ class TestToolShedConfig:
     file_path: str
     id_secret: str = "thisistheshedunittestsecret"
     smtp_server: Optional[str] = None
+    hgweb_repo_prefix = "repos/"
     config_hg_for_dev = False
 
     def __init__(self, temp_directory):