From 97371f1486bda858dcca08cdd2be6178942da903 Mon Sep 17 00:00:00 2001
From: Martin Cech <marten@bx.psu.edu>
Date: Fri, 19 Apr 2024 12:37:25 -0700
Subject: [PATCH 01/23] add relocation docs to the object store samples

---
 .../config/sample/object_store_conf.sample.yml       | 12 +++++++++---
 .../config/sample/object_store_conf.xml.sample       | 11 ++++++++---
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/lib/galaxy/config/sample/object_store_conf.sample.yml b/lib/galaxy/config/sample/object_store_conf.sample.yml
index 5db8d6ffa08a..b1b2cb34afec 100644
--- a/lib/galaxy/config/sample/object_store_conf.sample.yml
+++ b/lib/galaxy/config/sample/object_store_conf.sample.yml
@@ -61,7 +61,7 @@ backends:
 # Sample Distributed Object Store with disk backends configuration
 #
 
-# In the distributed object store, existing dataests will be located by the `object_store_id` column in the `dataset`
+# In the distributed object store, existing datasets will be located by the `object_store_id` column in the `dataset`
 # table of the Galaxy database, which corresponds to the `id` option on the backend. New datasets are created based on
 # the `weight` option: a backend with weight "2" has twice the chance of being (randomly) selected for new datasets as a
 # backend with weight "1". A weight of "0" will still allow datasets in that backend to be read, but no new datasets
@@ -299,20 +299,25 @@ extra_dirs:
 # The admin is responsible for routinely cleaning that storage using Galaxy's admin scripts - this object store
 # configuration just allows the user selection and communicates expectations to the user. Training related to Galaxy
 # cleanup scripts can be found in the Galaxy Training Network:
-# 
+#
 # Slides: https://gxy.io/GTN:S00103
 # Tutorial: https://gxy.io/GTN:T00324
 #
 # In this example, the scratch storage is marked as user-private by setting the `private` option to "true" on the
-# backend definition. This means it cannot be used in public datasets, shared between users, etc.. This is more example
+# backend definition. This means it cannot be used in public datasets, shared between users, etc.. This is for example
 # purposes - you may very well not want scratch storage to be defined as private as it prevents a lot of regular
 # functionality and Galaxy handles regularly cleaned datasets fairly gracefully when the appropriate admin scripts are
 # used.
+#
+# It is safe to just relabel the object store that a dataset belongs to if the underlying paths mapped to by the object
+# stores are the same and the dataset has not been copied. To enable users to relocate datasets this way set the
+# backends' `device` property to the same value.
 
 type: distributed
 backends:
   - id: default
     type: disk
+    device: device1
     weight: 1
     allow_selection: true
     name: Default Galaxy Storage
@@ -329,6 +334,7 @@ backends:
           found on our [Archive Tier Storage](https://www.msi.umn.edu/content/archive-tier-storage) page.
   - id: scratch
     type: disk
+    device: device2
     weight: 0
     allow_selection: true
     private: true
diff --git a/lib/galaxy/config/sample/object_store_conf.xml.sample b/lib/galaxy/config/sample/object_store_conf.xml.sample
index 2d726077a164..5b81dc3d4cd5 100644
--- a/lib/galaxy/config/sample/object_store_conf.xml.sample
+++ b/lib/galaxy/config/sample/object_store_conf.xml.sample
@@ -62,7 +62,7 @@
 <!--
     Sample Distributed Object Store with disk backends
 
-    In the distributed object store, existing dataests will be located by the
+    In the distributed object store, existing datasets will be located by the
     `object_store_id` column in the `dataset` table of the Galaxy database,
     which corresponds to the `id` attribute on the backend tag. New datasets are
     created based on the "weight" attribute: a backend with weight "2" has a
@@ -272,11 +272,16 @@
     of regular functionality and Galaxy handles regularly cleaned
     datasets fairly gracefully when the appropriate admin scripts
     are used.
+
+    It is safe to just relabel the object store that a dataset belongs
+    to if the underlying paths mapped to by the object stores are the
+    same and the dataset has not been copied. To enable users to relocate
+    datasets this way set the backends' `device` property to the same value.
 -->
 <!--
 <object_store type="distributed">
     <backends>
-        <backend id="default" allow_selection="true" type="disk" weight="1" name="Default Galaxy Storage">
+        <backend id="default" allow_selection="true" type="disk" device="device1" weight="1" name="Default Galaxy Storage">
             <description>This is Galaxy's default object store - this disk is regularly backed up and all user's have a default quota of 200 GB.
             </description>
             <files_dir path="database/objects/deafult"/>
@@ -286,7 +291,7 @@
                 <backed_up>Backed up to Galaxy's institutional long term tape drive nightly. More information about our tape drive can be found on our [Archive Tier Storage](https://www.msi.umn.edu/content/archive-tier-storage) page.</backed_up>
             </badges>
         </backend>
-        <backend id="scratch" allow_selection="true" type="disk" weight="0" name="Scratch Storage" private="true">
+        <backend id="scratch" allow_selection="true" type="disk" device="device2" weight="0" name="Scratch Storage" private="true">
             <quota source="second_tier" />
             <description>This object store is connected to institutional scratch storage. This disk is not backed up and private to your user and datasets belonging to this storage will be automatically deleted after one month.
 </description>

From 03111fad43c5baef2a742683f667a96b3691fbad Mon Sep 17 00:00:00 2001
From: Martin Cech <marten@bx.psu.edu>
Date: Fri, 19 Apr 2024 13:23:10 -0700
Subject: [PATCH 02/23] move the link from heading to the body and change to a
 button

- add minor text adjustments
- close the modal when clicking on the currently selected storage
---
 .../Dataset/DatasetStorage/DatasetStorage.vue     | 14 ++++++--------
 .../Dataset/DatasetStorage/RelocateDialog.vue     | 15 ++++++++++-----
 .../Dataset/DatasetStorage/RelocateLink.vue       | 12 +++++++++---
 3 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue b/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
index ec020cf1e174..b36bdfad2296 100644
--- a/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
+++ b/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
@@ -59,14 +59,7 @@ watch(props, fetch, { immediate: true });
 
 <template>
     <div>
-        <h2 v-if="includeTitle" class="h-md">
-            Dataset Storage
-            <RelocateLink
-                v-if="storageInfo"
-                :dataset-id="datasetId"
-                :dataset-storage-details="storageInfo"
-                @relocated="fetch" />
-        </h2>
+        <h2 v-if="includeTitle" class="h-md">Dataset Storage</h2>
         <div v-if="errorMessage" class="error">{{ errorMessage }}</div>
         <LoadingSpan v-else-if="storageInfo == null"> </LoadingSpan>
         <div v-else-if="discarded">
@@ -84,5 +77,10 @@ watch(props, fetch, { immediate: true });
         <div v-else>
             <DescribeObjectStore what="This dataset is stored in" :storage-info="storageInfo" />
         </div>
+        <RelocateLink
+            v-if="storageInfo"
+            :dataset-id="datasetId"
+            :dataset-storage-details="storageInfo"
+            @relocated="fetch" />
     </div>
 </template>
diff --git a/client/src/components/Dataset/DatasetStorage/RelocateDialog.vue b/client/src/components/Dataset/DatasetStorage/RelocateDialog.vue
index e0823d4b9bd5..90d385896018 100644
--- a/client/src/components/Dataset/DatasetStorage/RelocateDialog.vue
+++ b/client/src/components/Dataset/DatasetStorage/RelocateDialog.vue
@@ -13,24 +13,26 @@ defineProps<RelocateProps>();
 
 const emit = defineEmits<{
     (e: "relocate", value: string): void;
+    (e: "closeModal"): void;
 }>();
 
-const fromWhat = "This dataset location in a";
-const toWhat = "This dataset will be relocated to";
+const fromWhat = "This dataset location is";
+const toWhat = "This dataset will be moved to";
 </script>
 
 <template>
     <div>
-        <p>Relocate the dataset's current object store of:</p>
+        <p>Currently the dataset is located in:</p>
         <b-button-group vertical size="lg" class="select-button-group">
             <ObjectStoreSelectButton
                 :key="fromObjectStore.object_store_id"
                 id-prefix="swap-target"
                 class="swap-target-object-store-select-button"
                 variant="info"
-                :object-store="fromObjectStore" />
+                :object-store="fromObjectStore"
+                @click="emit('closeModal')" />
         </b-button-group>
-        <p>Select a new object store below to relocate the dataset</p>
+        <p class="move-to">Select new storage location for the dataset:</p>
         <b-button-group vertical size="lg" class="select-button-group">
             <ObjectStoreSelectButton
                 v-for="objectStore in targetObjectStores"
@@ -60,4 +62,7 @@ const toWhat = "This dataset will be relocated to";
     margin: auto;
     width: 400px;
 }
+.move-to {
+    margin-top: 2em;
+}
 </style>
diff --git a/client/src/components/Dataset/DatasetStorage/RelocateLink.vue b/client/src/components/Dataset/DatasetStorage/RelocateLink.vue
index 8af6f8aaacee..5f4d17eac19b 100644
--- a/client/src/components/Dataset/DatasetStorage/RelocateLink.vue
+++ b/client/src/components/Dataset/DatasetStorage/RelocateLink.vue
@@ -1,4 +1,5 @@
 <script setup lang="ts">
+import { BButton } from "bootstrap-vue";
 import { storeToRefs } from "pinia";
 import { computed, ref } from "vue";
 
@@ -70,6 +71,10 @@ const emit = defineEmits<{
     (e: "relocated"): void;
 }>();
 
+function closeModal() {
+    showModal.value = false;
+}
+
 async function relocate(objectStoreId: string) {
     try {
         await updateObjectStore(props.datasetId, objectStoreId);
@@ -84,12 +89,13 @@ async function relocate(objectStoreId: string) {
 
 <template>
     <span class="storage-relocate-link">
-        <SelectModal v-if="currentObjectStore" v-model="showModal" title="Relocate Dataset Storage">
+        <SelectModal v-if="currentObjectStore" v-model="showModal" title="Move Dataset">
             <RelocateDialog
                 :from-object-store="currentObjectStore"
                 :target-object-stores="validTargets"
-                @relocate="relocate" />
+                @relocate="relocate"
+                @closeModal="closeModal" />
         </SelectModal>
-        <b-link v-if="relocatable" href="#" @click="showModal = true">(relocate)</b-link>
+        <BButton v-if="relocatable" @click="showModal = true">Move Dataset</BButton>
     </span>
 </template>

From cff711f7f7824509397d77d6e880ad8d7f0ec6df Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Thu, 18 Apr 2024 20:51:33 +0200
Subject: [PATCH 03/23] Raise appropriate exception if user forces non-ready
 collection as input

---
 lib/galaxy/tools/parameters/dataset_matcher.py | 2 +-
 lib/galaxy/tools/parameters/meta.py            | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/tools/parameters/dataset_matcher.py b/lib/galaxy/tools/parameters/dataset_matcher.py
index e810abf538ec..4a9d2a552561 100644
--- a/lib/galaxy/tools/parameters/dataset_matcher.py
+++ b/lib/galaxy/tools/parameters/dataset_matcher.py
@@ -262,7 +262,7 @@ def hdca_match(self, history_dataset_collection_association):
     def dataset_collection_match(self, dataset_collection):
         # If dataset collection not yet populated, cannot determine if it
         # would be a valid match for this parameter.
-        if not dataset_collection.populated:
+        if not dataset_collection.populated_optimized:
             return False
 
         valid = True
diff --git a/lib/galaxy/tools/parameters/meta.py b/lib/galaxy/tools/parameters/meta.py
index f4d3a3ea26cc..fee9f6d6079e 100644
--- a/lib/galaxy/tools/parameters/meta.py
+++ b/lib/galaxy/tools/parameters/meta.py
@@ -242,6 +242,8 @@ def __expand_collection_parameter(trans, input_key, incoming_val, collections_to
             subcollection_type = None
     hdc_id = trans.app.security.decode_id(encoded_hdc_id)
     hdc = trans.sa_session.get(HistoryDatasetCollectionAssociation, hdc_id)
+    if not hdc.collection.populated_optimized:
+        raise exceptions.ToolInputsNotReadyException("An input collection is not populated.")
     collections_to_match.add(input_key, hdc, subcollection_type=subcollection_type, linked=linked)
     if subcollection_type is not None:
         subcollection_elements = subcollections.split_dataset_collection_instance(hdc, subcollection_type)

From a671090c8db49b022a5fcbe181f40427d6422bcf Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Fri, 19 Apr 2024 10:12:18 +0200
Subject: [PATCH 04/23] Fix test_map_over_empty_collection test

It was using the wrong id type, and since we don't prepopulate the
collection we need to wait until discovery happens. It actually nicely
illustrates the bug.
---
 lib/galaxy_test/api/test_tools.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index 7befe7f221b1..3eacb456a0ec 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -1659,9 +1659,8 @@ def test_map_over_collection(self, history_id):
 
     @skip_without_tool("cat1")
     def test_map_over_empty_collection(self, history_id):
-        hdca_id = self.dataset_collection_populator.create_list_in_history(history_id, contents=[]).json()["outputs"][
-            0
-        ]["id"]
+        response = self.dataset_collection_populator.create_list_in_history(history_id, contents=[], wait=True).json()
+        hdca_id = response["output_collections"][0]["id"]
         inputs = {
             "input1": {"batch": True, "values": [{"src": "hdca", "id": hdca_id}]},
         }

From b1333e01cf4c64539192a72e3c4d98cf56db42e3 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 21 Apr 2024 21:09:21 +0200
Subject: [PATCH 05/23] Also perform strict checks for new database objects

I think that's the missing piece to find all the instances where we
might still be committing HDAs without HIDs, and I think we might have
a similar bug for DatasetCollectionElements.
---
 lib/galaxy/model/base.py | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/lib/galaxy/model/base.py b/lib/galaxy/model/base.py
index 8936ca1cf48c..77bc38218f3b 100644
--- a/lib/galaxy/model/base.py
+++ b/lib/galaxy/model/base.py
@@ -174,18 +174,32 @@ def versioned_objects_strict(iter):
             yield obj
 
 
-if os.environ.get("GALAXY_TEST_RAISE_EXCEPTION_ON_HISTORYLESS_HDA"):
-    log.debug("Using strict flush checks")
-    versioned_objects = versioned_objects_strict  # noqa: F811
+def get_before_flush_handler():
+    if os.environ.get("GALAXY_TEST_RAISE_EXCEPTION_ON_HISTORYLESS_HDA"):
+        log.debug("Using strict flush checks")
+
+        def before_flush(session, flush_context, instances):
+            for obj in session.new:
+                if hasattr(obj, "__strict_check_before_flush__"):
+                    obj.__strict_check_before_flush__()
+            for obj in versioned_objects_strict(session.dirty):
+                obj.__create_version__(session)
+            for obj in versioned_objects_strict(session.deleted):
+                obj.__create_version__(session, deleted=True)
+
+    else:
+
+        def before_flush(session, flush_context, instances):
+            for obj in versioned_objects(session.dirty):
+                obj.__create_version__(session)
+            for obj in versioned_objects(session.deleted):
+                obj.__create_version__(session, deleted=True)
+
+    return before_flush
 
 
 def versioned_session(session):
-    @event.listens_for(session, "before_flush")
-    def before_flush(session, flush_context, instances):
-        for obj in versioned_objects(session.dirty):
-            obj.__create_version__(session)
-        for obj in versioned_objects(session.deleted):
-            obj.__create_version__(session, deleted=True)
+    event.listens_for(session, "before_flush")(get_before_flush_handler())
 
 
 def ensure_object_added_to_session(object_to_add, *, object_in_session=None, session=None) -> bool:

From 1b1094be215437c272e241bd4e351ba019ef3e59 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 21 Apr 2024 11:48:46 +0200
Subject: [PATCH 06/23] Set (copied) output hid before queuing job

Otherwise we may end up with a history item that either temporarily or
permanently lacks a hid.

This could currently be encountered via implicit conversion that set
`visible=False` in the when calling `datatype.convert_dataset`.
---
 lib/galaxy/datatypes/data.py                   |  6 +++++-
 .../display_applications/parameters.py         | 18 +-----------------
 lib/galaxy/datatypes/protocols.py              |  2 ++
 lib/galaxy/model/__init__.py                   |  9 +++------
 4 files changed, 11 insertions(+), 24 deletions(-)

diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py
index cad936f06b16..1c8c53827839 100644
--- a/lib/galaxy/datatypes/data.py
+++ b/lib/galaxy/datatypes/data.py
@@ -842,7 +842,11 @@ def convert_dataset(
         # Make the target datatype available to the converter
         params["__target_datatype__"] = target_type
         # Run converter, job is dispatched through Queue
-        job, converted_datasets, *_ = converter.execute(trans, incoming=params, set_output_hid=visible, history=history)
+        job, converted_datasets, *_ = converter.execute(
+            trans, incoming=params, set_output_hid=visible, history=history, flush_job=False
+        )
+        for converted_dataset in converted_datasets.values():
+            original_dataset.attach_implicitly_converted_dataset(trans.sa_session, converted_dataset, target_type)
         trans.app.job_manager.enqueue(job, tool=converter)
         if len(params) > 0:
             trans.log_event(f"Converter params: {str(params)}", tool_id=converter.id)
diff --git a/lib/galaxy/datatypes/display_applications/parameters.py b/lib/galaxy/datatypes/display_applications/parameters.py
index 25e90032f248..7d022d38d1fb 100644
--- a/lib/galaxy/datatypes/display_applications/parameters.py
+++ b/lib/galaxy/datatypes/display_applications/parameters.py
@@ -11,7 +11,6 @@
 
 from galaxy.datatypes.data import Data
 from galaxy.model import DatasetInstance
-from galaxy.model.base import transaction
 from galaxy.schema.schema import DatasetState
 from galaxy.util import string_as_bool
 from galaxy.util.template import fill_template
@@ -182,22 +181,7 @@ def prepare(self, other_values, dataset_hash, user_hash, trans):
                 if target_ext and not converted_dataset:
                     if isinstance(data, DisplayDataValueWrapper):
                         data = data.value
-                    new_data = next(
-                        iter(
-                            data.datatype.convert_dataset(
-                                trans, data, target_ext, return_output=True, visible=False
-                            ).values()
-                        )
-                    )
-                    new_data.hid = data.hid
-                    new_data.name = data.name
-                    trans.sa_session.add(new_data)
-                    assoc = trans.app.model.ImplicitlyConvertedDatasetAssociation(
-                        parent=data, file_type=target_ext, dataset=new_data, metadata_safe=False
-                    )
-                    trans.sa_session.add(assoc)
-                    with transaction(trans.sa_session):
-                        trans.sa_session.commit()
+                    data.datatype.convert_dataset(trans, data, target_ext, return_output=True, visible=False)
                 elif converted_dataset and converted_dataset.state == DatasetState.ERROR:
                     raise Exception(f"Dataset conversion failed for data parameter: {self.name}")
         return self.get_value(other_values, dataset_hash, user_hash, trans)
diff --git a/lib/galaxy/datatypes/protocols.py b/lib/galaxy/datatypes/protocols.py
index bbd702538b74..5bc07c20c9f2 100644
--- a/lib/galaxy/datatypes/protocols.py
+++ b/lib/galaxy/datatypes/protocols.py
@@ -85,5 +85,7 @@ def has_data(self) -> bool: ...
 
     def set_peek(self) -> None: ...
 
+    def attach_implicitly_converted_dataset(self, session, new_dataset, target_ext: str) -> None: ...
+
 
 class DatasetHasHidProtocol(DatasetProtocol, HasHid, Protocol): ...
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 3e78689a2a39..ade3ed17d68c 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -4736,7 +4736,7 @@ def get_converted_dataset(self, trans, target_ext, target_context=None, history=
             raise NoConverterException(f"A dependency ({dependency}) is missing a converter.")
         except KeyError:
             pass  # No deps
-        new_dataset = next(
+        return next(
             iter(
                 self.datatype.convert_dataset(
                     trans,
@@ -4750,7 +4750,6 @@ def get_converted_dataset(self, trans, target_ext, target_context=None, history=
                 ).values()
             )
         )
-        return self.attach_implicitly_converted_dataset(trans.sa_session, new_dataset, target_ext)
 
     def attach_implicitly_converted_dataset(self, session, new_dataset, target_ext: str):
         new_dataset.name = self.name
@@ -4760,9 +4759,6 @@ def attach_implicitly_converted_dataset(self, session, new_dataset, target_ext:
         )
         session.add(new_dataset)
         session.add(assoc)
-        with transaction(session):
-            session.commit()
-        return new_dataset
 
     def copy_attributes(self, new_dataset):
         """
@@ -5134,7 +5130,8 @@ def copy_tags_to(self, copy_tags=None):
                 self.tags.append(copied_tag)
 
     def copy_attributes(self, new_dataset):
-        new_dataset.hid = self.hid
+        if new_dataset.hid is None:
+            new_dataset.hid = self.hid
 
     def to_library_dataset_dataset_association(
         self,

From e3aba2d2b0b2f325459ad3b86487146646c46e1c Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 21 Apr 2024 22:51:00 +0200
Subject: [PATCH 07/23] Relax requirement for DCEs of collections that are not
 populated

---
 lib/galaxy/model/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index ade3ed17d68c..b5297e23656b 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -7146,7 +7146,8 @@ def __init__(
         self.element_identifier = element_identifier or str(element_index)
 
     def __strict_check_before_flush__(self):
-        assert self.element_object, "Dataset Collection Element without child entity detected, this is not valid"
+        if self.collection.populated_optimized:
+            assert self.element_object, "Dataset Collection Element without child entity detected, this is not valid"
 
     @property
     def element_type(self):

From 5648478e6d413e1fa606e005f06fe0361bd18ab3 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 21 Apr 2024 22:51:36 +0200
Subject: [PATCH 08/23] Adjust schema for collection elements of collections
 that aren't populated

---
 client/src/api/schema/schema.ts | 7 ++++---
 lib/galaxy/schema/schema.py     | 8 ++++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts
index 817248b6b433..fa5d31f0fa3e 100644
--- a/client/src/api/schema/schema.ts
+++ b/client/src/api/schema/schema.ts
@@ -3751,7 +3751,7 @@ export interface components {
              * Element Type
              * @description The type of the element. Used to interpret the `object` field.
              */
-            element_type: components["schemas"]["DCEType"];
+            element_type?: components["schemas"]["DCEType"] | null;
             /**
              * Dataset Collection Element ID
              * @example 0123456789ABCDEF
@@ -3767,10 +3767,11 @@ export interface components {
              * Object
              * @description The element's specific data depending on the value of `element_type`.
              */
-            object:
+            object?:
                 | components["schemas"]["HDAObject"]
                 | components["schemas"]["HDADetailed"]
-                | components["schemas"]["DCObject"];
+                | components["schemas"]["DCObject"]
+                | null;
         };
         /**
          * DCEType
diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py
index 277bb82f481e..5badff543962 100644
--- a/lib/galaxy/schema/schema.py
+++ b/lib/galaxy/schema/schema.py
@@ -971,13 +971,13 @@ class DCESummary(Model, WithModelClass):
         title="Element Identifier",
         description="The actual name of this element.",
     )
-    element_type: DCEType = Field(
-        ...,
+    element_type: Optional[DCEType] = Field(
+        None,
         title="Element Type",
         description="The type of the element. Used to interpret the `object` field.",
     )
-    object: Union[HDAObject, HDADetailed, DCObject] = Field(
-        ...,
+    object: Optional[Union[HDAObject, HDADetailed, DCObject]] = Field(
+        None,
         title="Object",
         description="The element's specific data depending on the value of `element_type`.",
     )

From bf3499734a123423a6d77aad2709d4181afbf2dc Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 21 Apr 2024 23:44:40 +0200
Subject: [PATCH 09/23] Fix unitialize typo

---
 lib/galaxy/managers/collections.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/managers/collections.py b/lib/galaxy/managers/collections.py
index 8db9037b55a5..75993f74403b 100644
--- a/lib/galaxy/managers/collections.py
+++ b/lib/galaxy/managers/collections.py
@@ -95,7 +95,7 @@ def precreate_dataset_collection_instance(
         # TODO: prebuild all required HIDs and send them in so no need to flush in between.
         dataset_collection = self.precreate_dataset_collection(
             structure,
-            allow_unitialized_element=implicit_output_name is not None,
+            allow_uninitialized_element=implicit_output_name is not None,
             completed_collection=completed_collection,
             implicit_output_name=implicit_output_name,
         )
@@ -112,10 +112,10 @@ def precreate_dataset_collection_instance(
         return instance
 
     def precreate_dataset_collection(
-        self, structure, allow_unitialized_element=True, completed_collection=None, implicit_output_name=None
+        self, structure, allow_uninitialized_element=True, completed_collection=None, implicit_output_name=None
     ):
         has_structure = not structure.is_leaf and structure.children_known
-        if not has_structure and allow_unitialized_element:
+        if not has_structure and allow_uninitialized_element:
             dataset_collection = model.DatasetCollectionElement.UNINITIALIZED_ELEMENT
         elif not has_structure:
             collection_type_description = structure.collection_type_description
@@ -143,7 +143,7 @@ def precreate_dataset_collection(
                         element = model.DatasetCollectionElement.UNINITIALIZED_ELEMENT
                     else:
                         element = self.precreate_dataset_collection(
-                            substructure, allow_unitialized_element=allow_unitialized_element
+                            substructure, allow_uninitialized_element=allow_uninitialized_element
                         )
 
                 element = model.DatasetCollectionElement(

From 5aecc0828222bb70f137273f30bfd3016c012339 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 21 Apr 2024 23:45:05 +0200
Subject: [PATCH 10/23] Fall back to populated method if collection not
 committed yet

---
 lib/galaxy/model/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index b5297e23656b..4153c004fe44 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -6359,6 +6359,8 @@ def has_deferred_data(self):
     @property
     def populated_optimized(self):
         if not hasattr(self, "_populated_optimized"):
+            if not self.id:
+                return self.populated
             _populated_optimized = True
             if ":" not in self.collection_type:
                 _populated_optimized = self.populated_state == DatasetCollection.populated_states.OK

From d88900210ac7421eaa730e4575ab0fd1558f767b Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 21 Apr 2024 23:55:41 +0200
Subject: [PATCH 11/23] Commit only after copying library item to history

---
 lib/galaxy/model/__init__.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 4153c004fe44..54305c2cc116 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -5818,11 +5818,9 @@ def to_history_dataset_association(self, target_history, parent_id=None, add_to_
 
         tag_manager = galaxy.model.tags.GalaxyTagHandler(sa_session)
         src_ldda_tags = tag_manager.get_tags_str(self.tags)
-        tag_manager.apply_item_tags(user=self.user, item=hda, tags_str=src_ldda_tags)
+        tag_manager.apply_item_tags(user=self.user, item=hda, tags_str=src_ldda_tags, flush=False)
         sa_session.add(hda)
-        with transaction(sa_session):
-            sa_session.commit()
-        hda.metadata = self.metadata  # need to set after flushed, as MetadataFiles require dataset.id
+        hda.metadata = self.metadata
         if add_to_history and target_history:
             target_history.add_dataset(hda)
         with transaction(sa_session):

From e1a45a226e2e3e4511833c32de1ab264f41e7c30 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Sun, 21 Apr 2024 23:56:43 +0200
Subject: [PATCH 12/23] Drop unused copy_from_ldda method

---
 test/unit/app/managers/test_HDAManager.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/test/unit/app/managers/test_HDAManager.py b/test/unit/app/managers/test_HDAManager.py
index 6c7300a44271..226b1e8d70d9 100644
--- a/test/unit/app/managers/test_HDAManager.py
+++ b/test/unit/app/managers/test_HDAManager.py
@@ -133,13 +133,6 @@ def test_copy_from_hda(self):
         hda3_annotation = self.hda_manager.annotation(hda3)
         assert annotation == hda3_annotation
 
-    # def test_copy_from_ldda( self ):
-    #    owner = self.user_manager.create( self.trans, **user2_data )
-    #    history1 = self.history_mgr.create( self.trans, name='history1', user=owner )
-    #
-    #    self.log( "should be able to copy an HDA" )
-    #    hda2 = self.hda_manager.copy_ldda( history1, hda1 )
-
     def test_delete(self):
         owner = self.user_manager.create(**user2_data)
         history1 = self.history_manager.create(name="history1", user=owner)

From a56f642631e778b0ba77474cd1ed31ffdc7002bd Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 22 Apr 2024 00:04:50 +0200
Subject: [PATCH 13/23] Drop unused method

---
 lib/galaxy/managers/hdas.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py
index 3be812fcf0e8..5fc02313c509 100644
--- a/lib/galaxy/managers/hdas.py
+++ b/lib/galaxy/managers/hdas.py
@@ -221,12 +221,6 @@ def copy(
 
         return copy
 
-    def copy_ldda(self, history, ldda, **kwargs):
-        """
-        Copy this HDA as a LDDA and return.
-        """
-        return ldda.to_history_dataset_association(history, add_to_history=True)
-
     # .... deletion and purging
     def purge(self, hda, flush=True, **kwargs):
         if self.app.config.enable_celery_tasks:

From badbd3ae78cb7cb51d5365639a99e836d1a3bd53 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 22 Apr 2024 11:06:10 +0200
Subject: [PATCH 14/23] Fix premature flush in hda_manager.copy

---
 lib/galaxy/managers/hdas.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py
index 5fc02313c509..9432e54b7805 100644
--- a/lib/galaxy/managers/hdas.py
+++ b/lib/galaxy/managers/hdas.py
@@ -202,7 +202,7 @@ def copy(
         if not isinstance(item, model.HistoryDatasetAssociation):
             raise TypeError()
         hda = item
-        copy = hda.copy(parent_id=kwargs.get("parent_id"), copy_hid=False, copy_tags=hda.tags, flush=flush)
+        copy = hda.copy(parent_id=kwargs.get("parent_id"), copy_hid=False, copy_tags=hda.tags, flush=False)
         if hide_copy:
             copy.visible = False
         if history:

From c2140bd00c23cf8c02c69c7f09c237689126e301 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 22 Apr 2024 13:32:09 +0200
Subject: [PATCH 15/23] Avoid flush in set_meta by using MetadataFile.uuid

---
 lib/galaxy/model/deferred.py | 2 +-
 lib/galaxy/model/metadata.py | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/lib/galaxy/model/deferred.py b/lib/galaxy/model/deferred.py
index e72d4e126aca..cea7a0eba17a 100644
--- a/lib/galaxy/model/deferred.py
+++ b/lib/galaxy/model/deferred.py
@@ -165,7 +165,7 @@ def ensure_materialized(
             if transient_paths:
                 metadata_tmp_files_dir = transient_paths.metadata_files_dir
             else:
-                # If metadata_tmp_files_dir is set we generate a MetdataTempFile,
+                # If metadata_tmp_files_dir is set we generate a MetadataTempFile,
                 # which we don't want when we're generating an attached materialized dataset instance
                 metadata_tmp_files_dir = None
             materialized_dataset_instance.set_meta(metadata_tmp_files_dir=metadata_tmp_files_dir)
diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py
index b9fb7839cdbb..5fdfd8a49678 100644
--- a/lib/galaxy/model/metadata.py
+++ b/lib/galaxy/model/metadata.py
@@ -670,7 +670,7 @@ def from_external_value(self, value, parent, path_rewriter=None):
                 # directory. Correct.
                 file_name = path_rewriter(file_name)
             mf.update_from_file(file_name)
-            value = mf.id
+            value = str(mf.uuid)
         return value
 
     def to_external_value(self, value):
@@ -692,11 +692,9 @@ def new_file(self, dataset=None, metadata_tmp_files_dir=None, **kwds):
             sa_session = object_session(dataset)
             if sa_session:
                 sa_session.add(mf)
-                with transaction(sa_session):
-                    sa_session.commit()  # commit to assign id
             return mf
         else:
-            # we need to make a tmp file that is accessable to the head node,
+            # we need to make a tmp file that is accessible to the head node,
             # we will be copying its contents into the MetadataFile objects filename after restoring from JSON
             # we do not include 'dataset' in the kwds passed, as from_JSON_value() will handle this for us
             return MetadataTempFile(metadata_tmp_files_dir=metadata_tmp_files_dir, **kwds)

From 121f27e2f4b9b90b14d64a103d963bd5fa3a753c Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 22 Apr 2024 16:09:22 +0200
Subject: [PATCH 16/23] Default HID to -1 if for whatever reason we didn't
 record a HID

---
 lib/galaxy/managers/hdas.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py
index 9432e54b7805..f92a75b56f24 100644
--- a/lib/galaxy/managers/hdas.py
+++ b/lib/galaxy/managers/hdas.py
@@ -555,6 +555,7 @@ def add_serializers(self):
         annotatable.AnnotatableSerializerMixin.add_serializers(self)
 
         serializers: Dict[str, base.Serializer] = {
+            "hid": lambda item, key, **context: item.hid if item.hid is not None else -1,
             "model_class": lambda item, key, **context: "HistoryDatasetAssociation",
             "history_content_type": lambda item, key, **context: "dataset",
             "hda_ldda": lambda item, key, **context: "hda",

From b8c39a57df431b3b60697ebc5d7cba968ea29f62 Mon Sep 17 00:00:00 2001
From: Martin Cech <marten@bx.psu.edu>
Date: Mon, 22 Apr 2024 09:58:02 -0700
Subject: [PATCH 17/23] change language to relocate

---
 .../components/Dataset/DatasetStorage/RelocateDialog.vue    | 6 +++---
 .../src/components/Dataset/DatasetStorage/RelocateLink.vue  | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/client/src/components/Dataset/DatasetStorage/RelocateDialog.vue b/client/src/components/Dataset/DatasetStorage/RelocateDialog.vue
index 90d385896018..1caf060b53b1 100644
--- a/client/src/components/Dataset/DatasetStorage/RelocateDialog.vue
+++ b/client/src/components/Dataset/DatasetStorage/RelocateDialog.vue
@@ -17,7 +17,7 @@ const emit = defineEmits<{
 }>();
 
 const fromWhat = "This dataset location is";
-const toWhat = "This dataset will be moved to";
+const toWhat = "This dataset will be relocated to";
 </script>
 
 <template>
@@ -32,7 +32,7 @@ const toWhat = "This dataset will be moved to";
                 :object-store="fromObjectStore"
                 @click="emit('closeModal')" />
         </b-button-group>
-        <p class="move-to">Select new storage location for the dataset:</p>
+        <p class="relocate-to">Select new storage location for the dataset:</p>
         <b-button-group vertical size="lg" class="select-button-group">
             <ObjectStoreSelectButton
                 v-for="objectStore in targetObjectStores"
@@ -62,7 +62,7 @@ const toWhat = "This dataset will be moved to";
     margin: auto;
     width: 400px;
 }
-.move-to {
+.relocate-to {
     margin-top: 2em;
 }
 </style>
diff --git a/client/src/components/Dataset/DatasetStorage/RelocateLink.vue b/client/src/components/Dataset/DatasetStorage/RelocateLink.vue
index 5f4d17eac19b..fb71dacc1b04 100644
--- a/client/src/components/Dataset/DatasetStorage/RelocateLink.vue
+++ b/client/src/components/Dataset/DatasetStorage/RelocateLink.vue
@@ -89,13 +89,13 @@ async function relocate(objectStoreId: string) {
 
 <template>
     <span class="storage-relocate-link">
-        <SelectModal v-if="currentObjectStore" v-model="showModal" title="Move Dataset">
+        <SelectModal v-if="currentObjectStore" v-model="showModal" title="Relocate Dataset">
             <RelocateDialog
                 :from-object-store="currentObjectStore"
                 :target-object-stores="validTargets"
                 @relocate="relocate"
                 @closeModal="closeModal" />
         </SelectModal>
-        <BButton v-if="relocatable" @click="showModal = true">Move Dataset</BButton>
+        <BButton v-if="relocatable" @click="showModal = true">Relocate Dataset</BButton>
     </span>
 </template>

From 857a14e2924c1b60f7968b082f44ef05a5867244 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Mon, 22 Apr 2024 19:29:43 +0200
Subject: [PATCH 18/23] Change wrong quota_source value from KeyError to
 ValueError

Which is caught in https://github.com/galaxyproject/galaxy/blob/44782a44c14885df2d9d1bcb91698245bfb3aed8/lib/galaxy/managers/base.py#L1121-L1124

Fixes https://sentry.galaxyproject.org/share/issue/16a3342631224d8292ef792298175dea/:
```
KeyError: "Could not find key nul in object store keys [None, 'scratch']"
  File "starlette/applications.py", line 123, in __call__
    await self.middleware_stack(scope, receive, send)
  File "starlette/middleware/errors.py", line 186, in __call__
    raise exc
  File "starlette/middleware/errors.py", line 164, in __call__
    await self.app(scope, receive, _send)
  File "starlette_context/middleware/raw_middleware.py", line 92, in __call__
    await self.app(scope, receive, send_wrapper)
  File "starlette/middleware/base.py", line 189, in __call__
    with collapse_excgroups():
  File "contextlib.py", line 155, in __exit__
    self.gen.throw(typ, value, traceback)
  File "starlette/_utils.py", line 93, in collapse_excgroups
    raise exc
  File "starlette/middleware/base.py", line 191, in __call__
    response = await self.dispatch_func(request, call_next)
  File "galaxy/webapps/galaxy/fast_app.py", line 108, in add_x_frame_options
    response = await call_next(request)
  File "starlette/middleware/base.py", line 165, in call_next
    raise app_exc
  File "starlette/middleware/base.py", line 151, in coro
    await self.app(scope, receive_or_disconnect, send_no_error)
  File "starlette/middleware/exceptions.py", line 62, in __call__
    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
  File "starlette/_exception_handler.py", line 64, in wrapped_app
    raise exc
  File "starlette/_exception_handler.py", line 53, in wrapped_app
    await app(scope, receive, sender)
  File "starlette/routing.py", line 758, in __call__
    await self.middleware_stack(scope, receive, send)
  File "starlette/routing.py", line 778, in app
    await route.handle(scope, receive, send)
  File "starlette/routing.py", line 299, in handle
    await self.app(scope, receive, send)
  File "starlette/routing.py", line 79, in app
    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
  File "starlette/_exception_handler.py", line 64, in wrapped_app
    raise exc
  File "starlette/_exception_handler.py", line 53, in wrapped_app
    await app(scope, receive, sender)
  File "starlette/routing.py", line 74, in app
    response = await func(request)
  File "fastapi/routing.py", line 278, in app
    raw_response = await run_endpoint_function(
  File "fastapi/routing.py", line 193, in run_endpoint_function
    return await run_in_threadpool(dependant.call, **values)
  File "starlette/concurrency.py", line 42, in run_in_threadpool
    return await anyio.to_thread.run_sync(func, *args)
  File "anyio/to_thread.py", line 56, in run_sync
    return await get_async_backend().run_sync_in_worker_thread(
  File "anyio/_backends/_asyncio.py", line 2144, in run_sync_in_worker_thread
    return await future
  File "anyio/_backends/_asyncio.py", line 851, in run
    result = context.run(func, *args)
  File "galaxy/webapps/galaxy/api/history_contents.py", line 465, in index
    items = self.service.index(
  File "galaxy/webapps/galaxy/services/history_contents.py", line 312, in index
    return self.__index_v2(trans, history_id, params, serialization_params, filter_query_params, accept)
  File "galaxy/webapps/galaxy/services/history_contents.py", line 972, in __index_v2
    filters = self.history_contents_filters.parse_query_filters_with_relations(filter_query_params, history_id)
  File "galaxy/managers/history_contents.py", line 525, in parse_query_filters_with_relations
    return super().parse_query_filters(query_filters)
  File "galaxy/managers/base.py", line 1087, in parse_query_filters
    return self.parse_filters(filter_params)
  File "galaxy/managers/base.py", line 1096, in parse_filters
    filter_ = self.parse_filter(attr, op, val)
  File "galaxy/managers/base.py", line 1116, in parse_filter
    orm_filter = self._parse_orm_filter(attr, op, val)
  File "galaxy/managers/history_contents.py", line 601, in _parse_orm_filter
    if (column_filter := get_filter(attr, op, val)) is not None:
  File "galaxy/managers/history_contents.py", line 595, in get_filter
    raise KeyError(f"Could not find key {val} in object store keys {list(ids.keys())}")
```
---
 lib/galaxy/managers/history_contents.py | 2 +-
 test/integration/test_quota.py          | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py
index a07953918738..adf680ade8c3 100644
--- a/lib/galaxy/managers/history_contents.py
+++ b/lib/galaxy/managers/history_contents.py
@@ -592,7 +592,7 @@ def get_filter(attr, op, val):
                     if val == "__null__":
                         val = None
                     if val not in ids:
-                        raise KeyError(f"Could not find key {val} in object store keys {list(ids.keys())}")
+                        raise ValueError(f"Could not find key {val} in object store keys {list(ids.keys())}")
                     object_store_ids = ids[val]
                     return sql.column("object_store_id").in_(object_store_ids)
 
diff --git a/test/integration/test_quota.py b/test/integration/test_quota.py
index 24928ef0e007..07137786a353 100644
--- a/test/integration/test_quota.py
+++ b/test/integration/test_quota.py
@@ -196,6 +196,13 @@ def test_quota_source_label_basics(self):
         labels = [q["quota_source_label"] for q in quotas]
         assert "mylabel" in labels
 
+        with self.dataset_populator.test_history() as history_id:
+            response = self.dataset_populator._get_contents_request(
+                history_id, data={"q": "quota_source_label-eq", "qv": "invalid", "v": "dev"}
+            )
+        assert response.status_code == 400
+        assert "unparsable value for filter" in response.json()["err_msg"]
+
     def _create_quota_with_name(self, quota_name: str, is_default: bool = False):
         payload = self._build_quota_payload_with_name(quota_name, is_default)
         create_response = self._post("quotas", data=payload, json=True)

From 4bc09c00145fb9681803ffb0dea2ab4bf0bc71a0 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 23 Apr 2024 11:55:44 +0200
Subject: [PATCH 19/23] Ensure that offset and limit are never negative

And set minimum limit consistently to 1 ... 0 doesn't seem to have much
utility but would also be an option.

Fixes part of https://github.com/galaxyproject/galaxy/issues/18043.
---
 lib/galaxy/webapps/galaxy/api/folder_contents.py | 3 ++-
 lib/galaxy/webapps/galaxy/api/jobs.py            | 3 ++-
 lib/galaxy/webapps/galaxy/api/pages.py           | 3 ++-
 lib/galaxy/webapps/galaxy/api/workflows.py       | 5 ++++-
 4 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/galaxy/webapps/galaxy/api/folder_contents.py b/lib/galaxy/webapps/galaxy/api/folder_contents.py
index 6b6fbffdd0a9..0c8f1f5e165c 100644
--- a/lib/galaxy/webapps/galaxy/api/folder_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/folder_contents.py
@@ -36,10 +36,11 @@
     Path(..., title="Folder ID", description="The encoded identifier of the library folder."),
 ]
 
-LimitQueryParam: int = Query(default=10, title="Limit", description="Maximum number of contents to return.")
+LimitQueryParam: int = Query(default=10, ge=1, title="Limit", description="Maximum number of contents to return.")
 
 OffsetQueryParam: int = Query(
     default=0,
+    ge=0,
     title="Offset",
     description="Return contents from this specified position. For example, if ``limit`` is set to 100 and ``offset`` to 200, contents between position 200-299 will be returned.",
 )
diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py
index 7d0d2757b333..089a226058c7 100644
--- a/lib/galaxy/webapps/galaxy/api/jobs.py
+++ b/lib/galaxy/webapps/galaxy/api/jobs.py
@@ -161,10 +161,11 @@
     description="Sort results by specified field.",
 )
 
-LimitQueryParam: int = Query(default=500, title="Limit", description="Maximum number of jobs to return.")
+LimitQueryParam: int = Query(default=500, ge=1, title="Limit", description="Maximum number of jobs to return.")
 
 OffsetQueryParam: int = Query(
     default=0,
+    ge=0,
     title="Offset",
     description="Return jobs starting from this specified position. For example, if ``limit`` is set to 100 and ``offset`` to 200, jobs 200-299 will be returned.",
 )
diff --git a/lib/galaxy/webapps/galaxy/api/pages.py b/lib/galaxy/webapps/galaxy/api/pages.py
index e098e9870e48..61aaea66ead0 100644
--- a/lib/galaxy/webapps/galaxy/api/pages.py
+++ b/lib/galaxy/webapps/galaxy/api/pages.py
@@ -72,10 +72,11 @@
     description="Sort in descending order?",
 )
 
-LimitQueryParam: int = Query(default=100, lt=1000, title="Limit number of queries.")
+LimitQueryParam: int = Query(default=100, ge=1, lt=1000, title="Limit number of queries.")
 
 OffsetQueryParam: int = Query(
     default=0,
+    ge=0,
     title="Number of pages to skip in sorted query (to enable pagination).",
 )
 
diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py
index 54386e7596c6..2fef4d09e3fe 100644
--- a/lib/galaxy/webapps/galaxy/api/workflows.py
+++ b/lib/galaxy/webapps/galaxy/api/workflows.py
@@ -815,10 +815,11 @@ def __get_stored_workflow(self, trans, workflow_id, **kwd):
     description="Sort in descending order?",
 )
 
-LimitQueryParam: Optional[int] = Query(default=None, title="Limit number of queries.")
+LimitQueryParam: Optional[int] = Query(default=None, ge=1, title="Limit number of queries.")
 
 OffsetQueryParam: Optional[int] = Query(
     default=0,
+    ge=0,
     title="Number of workflows to skip in sorted query (to enable pagination).",
 )
 
@@ -1229,6 +1230,7 @@ def show_workflow(
 InvocationsLimitQueryParam = Annotated[
     Optional[int],
     Query(
+        ge=1,
         title="Limit",
         description="Limit the number of invocations to return.",
     ),
@@ -1237,6 +1239,7 @@ def show_workflow(
 InvocationsOffsetQueryParam = Annotated[
     Optional[int],
     Query(
+        ge=0,
         title="Offset",
         description="Number of invocations to skip.",
     ),

From 5545dcd7bcf74d70b86810fc9d2accad10563733 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 23 Apr 2024 14:00:13 +0200
Subject: [PATCH 20/23] Adjust and enhance job filtering test

---
 lib/galaxy_test/api/test_jobs.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py
index 69a5bb520ccd..09193ea2065e 100644
--- a/lib/galaxy_test/api/test_jobs.py
+++ b/lib/galaxy_test/api/test_jobs.py
@@ -182,14 +182,19 @@ def test_index_workflow_filter_implicit_jobs(self, history_id):
 
     @pytest.mark.require_new_history
     def test_index_limit_and_offset_filter(self, history_id):
+        # create 2 datasets
+        self.__history_with_new_dataset(history_id)
         self.__history_with_new_dataset(history_id)
         jobs = self.__jobs_index(data={"history_id": history_id})
         assert len(jobs) > 0
         length = len(jobs)
         jobs = self.__jobs_index(data={"history_id": history_id, "offset": 1})
         assert len(jobs) == length - 1
-        jobs = self.__jobs_index(data={"history_id": history_id, "limit": 0})
-        assert len(jobs) == 0
+        jobs = self.__jobs_index(data={"history_id": history_id, "limit": 1})
+        assert len(jobs) == 1
+        response = self._get("jobs", data={"history_id": history_id, "limit": -1})
+        assert response.status_code == 400
+        assert response.json()["err_msg"] == "Input should be greater than or equal to 1 in ('query', 'limit')"
 
     @pytest.mark.require_new_history
     def test_index_search_filter_tool_id(self, history_id):

From f26e90f7128652905a7c9f21fdea3ce148e0cfc2 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Tue, 23 Apr 2024 15:50:13 +0200
Subject: [PATCH 21/23] Log start and end of request

Configurable via `use_access_logging_middleware`.
Looks like this:

```
galaxy.webapps.base.api DEBUG 2024-04-23 17:14:43,776 [pN:main,p:73562,tN:MainThread] GET /history/current_history_json?since=2024-04-22T17:24:13.936493 f8c75e2c-30eb-4fcb-9b96-25bd4b2df435
galaxy.webapps.base.api DEBUG 2024-04-23 17:14:48,719 [pN:main,p:73562,tN:MainThread] GET /history/current_history_json?since=2024-04-22T17:24:13.936493 f8c75e2c-30eb-4fcb-9b96-25bd4b2df435 200
galaxy.webapps.base.api DEBUG 2024-04-23 17:14:51,767 [pN:main,p:73562,tN:MainThread] GET /history/current_history_json?since=2024-04-22T17:24:13.936493 2fe6094c-67bb-4144-a19f-cbb4a1b18fb4
galaxy.webapps.base.api DEBUG 2024-04-23 17:14:51,786 [pN:main,p:73562,tN:MainThread] GET /history/current_history_json?since=2024-04-22T17:24:13.936493 2fe6094c-67bb-4144-a19f-cbb4a1b18fb4 200
galaxy.webapps.base.api DEBUG 2024-04-23 17:14:53,139 [pN:main,p:73562,tN:MainThread] GET /api/entry_points?running=true 763693e9-4fd3-4a97-a100-a3c259b4b4c8
```
---
 doc/source/admin/galaxy_options.rst         | 14 ++++++--
 lib/galaxy/config/sample/galaxy.yml.sample  |  4 +++
 lib/galaxy/config/schemas/config_schema.yml |  7 ++++
 lib/galaxy/webapps/base/api.py              | 36 ++++++++++++++++++++-
 lib/galaxy/webapps/galaxy/fast_app.py       |  6 +++-
 5 files changed, 62 insertions(+), 5 deletions(-)

diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst
index f3021dc25d48..4e2fca9c11d0 100644
--- a/doc/source/admin/galaxy_options.rst
+++ b/doc/source/admin/galaxy_options.rst
@@ -3030,6 +3030,17 @@
 :Type: bool
 
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+``use_access_logging_middleware``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Log request start as well as request end. Disables uvicorn access
+    log handler.
+:Default: ``false``
+:Type: bool
+
+
 ~~~~~~~~~~~~
 ``use_lint``
 ~~~~~~~~~~~~
@@ -5477,6 +5488,3 @@
     This requires the help_forum_api_url to be set.
 :Default: ``false``
 :Type: bool
-
-
-
diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample
index 21b325bdb490..fef52325c762 100644
--- a/lib/galaxy/config/sample/galaxy.yml.sample
+++ b/lib/galaxy/config/sample/galaxy.yml.sample
@@ -1743,6 +1743,10 @@ galaxy:
   # job is complete.
   #debug: false
 
+  # Log request start as well as request end. Disables uvicorn access
+  # log handler.
+  #use_access_logging_middleware: false
+
   # Check for WSGI compliance.
   #use_lint: false
 
diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml
index ea25af4fc4c4..81078d8aef6f 100644
--- a/lib/galaxy/config/schemas/config_schema.yml
+++ b/lib/galaxy/config/schemas/config_schema.yml
@@ -2193,6 +2193,13 @@ mapping:
           causes the files used by PBS/SGE (submission script, output, and error)
           to remain on disk after the job is complete.
 
+      use_access_logging_middleware:
+        type: bool
+        default: false
+        required: false
+        desc: |
+          Log request start as well as request end. Disables uvicorn access log handler.
+
       use_lint:
         type: bool
         default: false
diff --git a/lib/galaxy/webapps/base/api.py b/lib/galaxy/webapps/base/api.py
index 7923fc297331..df779127f0c3 100644
--- a/lib/galaxy/webapps/base/api.py
+++ b/lib/galaxy/webapps/base/api.py
@@ -1,6 +1,8 @@
 import os
 import stat
 import typing
+import uuid
+from logging import getLogger
 
 import anyio
 from fastapi import (
@@ -15,8 +17,12 @@
     FileResponse,
     Response,
 )
+from starlette_context import context
 from starlette_context.middleware import RawContextMiddleware
-from starlette_context.plugins import RequestIdPlugin
+from starlette_context.plugins import (
+    Plugin,
+    RequestIdPlugin,
+)
 
 from galaxy.exceptions import MessageException
 from galaxy.exceptions.utils import api_error_to_dict
@@ -32,6 +38,9 @@
     )
 
 
+log = getLogger(__name__)
+
+
 # Copied from https://github.com/tiangolo/fastapi/issues/1240#issuecomment-1055396884
 def _get_range_header(range_header: str, file_size: int) -> typing.Tuple[int, int]:
     def _invalid_range():
@@ -192,6 +201,31 @@ async def message_exception_middleware(request: Request, exc: MessageException)
         return get_error_response_for_request(request, exc)
 
 
+class AccessLoggingMiddleware(Plugin):
+
+    key = "access_line"
+
+    async def process_request(self, request):
+        scope = request.scope
+        path = scope["root_path"] + scope["path"]
+        if scope["query_string"]:
+            path = f"{path}?{scope['query_string'].decode('ascii')}"
+        access_line = f"{scope['method']} {path} {uuid.uuid4()}"
+        log.debug(access_line)
+        return access_line
+
+    async def enrich_response(self, response) -> None:
+        access_line = context.get("access_line")
+        if status := response.get("status"):
+            log.debug(f"{access_line} {status}")
+
+
+def add_raw_context_middlewares(app: FastAPI):
+    getLogger("uvicorn.access").handlers = []
+    plugins = (RequestIdPlugin(force_new_uuid=True), AccessLoggingMiddleware())
+    app.add_middleware(RawContextMiddleware, plugins=plugins)
+
+
 def add_request_id_middleware(app: FastAPI):
     app.add_middleware(RawContextMiddleware, plugins=(RequestIdPlugin(force_new_uuid=True),))
 
diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py
index 0a2e0e26ec14..4409708ed8dc 100644
--- a/lib/galaxy/webapps/galaxy/fast_app.py
+++ b/lib/galaxy/webapps/galaxy/fast_app.py
@@ -16,6 +16,7 @@
 from galaxy.version import VERSION
 from galaxy.webapps.base.api import (
     add_exception_handler,
+    add_raw_context_middlewares,
     add_request_id_middleware,
     GalaxyFileResponse,
     include_all_package_routers,
@@ -179,7 +180,10 @@ def initialize_fast_app(gx_wsgi_webapp, gx_app):
     app = get_fastapi_instance(root_path=root_path)
     add_exception_handler(app)
     add_galaxy_middleware(app, gx_app)
-    add_request_id_middleware(app)
+    if gx_app.config.use_access_logging_middleware:
+        add_raw_context_middlewares(app)
+    else:
+        add_request_id_middleware(app)
     include_all_package_routers(app, "galaxy.webapps.galaxy.api")
     include_legacy_openapi(app, gx_app)
     wsgi_handler = WSGIMiddleware(gx_wsgi_webapp)

From 64be33ebc9c0e6244f32f9aa587d3a8b1a976651 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 24 Apr 2024 17:11:53 +0200
Subject: [PATCH 22/23] Update k8s-version

---
 .github/workflows/integration.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index 58aa81dd63c3..e90a4ea836b2 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -62,7 +62,7 @@ jobs:
         id: minikube
         uses: CodingNagger/minikube-setup-action@v1.0.6
         with:
-          k8s-version: '1.19.16'
+          k8s-version: '1.23.0'
       - name: Launch Minikube
         run: eval ${{ steps.minikube.outputs.launcher }}
       - name: Check pods

From ab087383c8a3f9fa59df101afa35f3978cf74257 Mon Sep 17 00:00:00 2001
From: mvdbeek <m.vandenbeek@gmail.com>
Date: Wed, 24 Apr 2024 17:38:58 +0200
Subject: [PATCH 23/23] Update setup-miniconda@v3 action

---
 .github/workflows/osx_startup.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/osx_startup.yaml b/.github/workflows/osx_startup.yaml
index ccc17526f870..5c0ffbe2b534 100644
--- a/.github/workflows/osx_startup.yaml
+++ b/.github/workflows/osx_startup.yaml
@@ -50,7 +50,7 @@ jobs:
           path: .tox
           key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-osx
       - name: Install miniconda  # use this job to test using Python from a conda environment
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           activate-environment: ''
       - name: Restore client cache