diff --git a/client/src/api/datasets.ts b/client/src/api/datasets.ts
index 63b8b5714d9c..a82b2022a63e 100644
--- a/client/src/api/datasets.ts
+++ b/client/src/api/datasets.ts
@@ -47,11 +47,10 @@ export async function fetchDatasetDetails(params: { id: string }): Promise
@@ -60,10 +72,19 @@ function goToHistoriesOverview() {
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue b/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
index 9d63b587c5e0..770c274e4740 100644
--- a/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
+++ b/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
@@ -1,5 +1,5 @@
@@ -157,36 +107,28 @@ async function onPermanentlyDeleteDataset(datasetId: string) {
Histories Storage Overview page to see
the storage taken by all your histories.
-
- Note: these graphs include deleted datasets. Remember that, even if you delete datasets, they still
- take up storage space. However, you can free up the storage space by permanently deleting them from the
- Discarded Items section of the
- Storage Manager page or by selecting them
- individually in the graph and clicking the Permanently Delete button.
-
-
+
+ v-bind="byteFormattingForChart">
{{ localize(`Top ${numberOfDatasetsToDisplay} Datasets by Size`) }}
@@ -204,8 +146,8 @@ async function onPermanentlyDeleteDataset(datasetId: string) {
item-type="dataset"
:is-recoverable="isRecoverableDataPoint(data)"
@view-item="onViewDataset"
- @undelete-item="onUndeleteDataset"
- @permanently-delete-item="onPermanentlyDeleteDataset" />
+ @undelete-item="onUndelete"
+ @permanently-delete-item="onPermDelete" />
@@ -218,8 +160,7 @@ async function onPermanentlyDeleteDataset(datasetId: string) {
)
"
:data="activeVsDeletedTotalSizeData"
- :label-formatter="bytesLabelFormatter"
- :value-formatter="bytesValueFormatter">
+ v-bind="byteFormattingForChart">
+import { library } from "@fortawesome/fontawesome-svg-core";
+import { faChartBar } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { BButton } from "bootstrap-vue";
+import { computed } from "vue";
+
+import localize from "@/utils/localization";
+
+import type { DataValuePoint } from "./Charts";
+
+interface Props {
+ data: DataValuePoint;
+}
+
+const props = defineProps();
+
+library.add(faChartBar);
+
+const label = computed(() => props.data.id);
+const viewDetailsIcon = computed(() => "chart-bar");
+
+const emit = defineEmits<{
+ (e: "view-item", itemId: string): void;
+}>();
+
+function onViewItem() {
+ emit("view-item", props.data.id);
+}
+
+
+
+
+ {{ label }}
+
+
+
+
+
+
+
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue b/client/src/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue
new file mode 100644
index 000000000000..06237e694f62
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue
@@ -0,0 +1,123 @@
+
+
+
+
+
+ Here you will find some Graphs displaying the storage taken by datasets in the object store:
+ {{ objectStoreId }}. You can use these graphs to identify the datasets that take the most space in this object store.
+
+
+
+
+
+
+
+
+ {{ localize(`Top ${numberOfDatasetsToDisplay} Datasets by Size`) }}
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue b/client/src/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue
new file mode 100644
index 000000000000..79af90cf7bb3
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue
@@ -0,0 +1,71 @@
+
+
+
+
+ Here you can find various graphs displaying the storage size taken by all your histories grouped by object
+ store.
+
+
+
+
+
+
+
+
+ {{ localize(`Object Stores by Usage`) }}
+
+
+
+
+
+
+
+
+
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/ShowObjectStore.vue b/client/src/components/User/DiskUsage/Visualizations/ShowObjectStore.vue
new file mode 100644
index 000000000000..313f2f6c0142
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/ShowObjectStore.vue
@@ -0,0 +1,50 @@
+
+
+
+
+
+
+
+ {{ error }}
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/WarnDeletedDatasets.vue b/client/src/components/User/DiskUsage/Visualizations/WarnDeletedDatasets.vue
new file mode 100644
index 000000000000..65ac06f9a170
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/WarnDeletedDatasets.vue
@@ -0,0 +1,9 @@
+
+
+ Note: these graphs include deleted datasets. Remember that, even if you delete datasets, they still take
+ up storage space. However, you can free up the storage space by permanently deleting them from the
+ Discarded Items section of the
+ Storage Manager page or by selecting them
+ individually in the graph and clicking the Permanently Delete button.
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/WarnDeletedHistories.vue b/client/src/components/User/DiskUsage/Visualizations/WarnDeletedHistories.vue
new file mode 100644
index 000000000000..aa11326c2e07
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/WarnDeletedHistories.vue
@@ -0,0 +1,9 @@
+
+
+ Note: these graphs include deleted histories. Remember that, even if you delete histories, they still
+ take up storage space. However, you can free up the storage space by permanently deleting them from the
+ Discarded Items section of the
+ Storage Manager page or by selecting them
+ individually in the graph and clicking the Permanently Delete button.
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/service.ts b/client/src/components/User/DiskUsage/Visualizations/service.ts
index 2a83827c8cd8..e88481eca64f 100644
--- a/client/src/components/User/DiskUsage/Visualizations/service.ts
+++ b/client/src/components/User/DiskUsage/Visualizations/service.ts
@@ -45,6 +45,17 @@ export async function fetchHistoryContentsSizeSummary(historyId: string, limit =
return response.data as unknown as ItemSizeSummary[];
}
+export async function fetchObjectStoreContentsSizeSummary(objectStoreId: string, limit = 5000) {
+ const response = await datasetsFetcher({
+ keys: itemSizeSummaryFields,
+ limit,
+ order: "size-dsc",
+ q: ["purged", "history_content_type", "object_store_id"],
+ qv: ["false", "dataset", objectStoreId],
+ });
+ return response.data as unknown as ItemSizeSummary[];
+}
+
export async function undeleteHistoryById(historyId: string): Promise {
const response = await undeleteHistory({ history_id: historyId });
return response.data as unknown as ItemSizeSummary;
@@ -55,12 +66,12 @@ export async function purgeHistoryById(historyId: string): Promise {
- const data = await undeleteHistoryDataset(historyId, datasetId);
+export async function undeleteDatasetById(datasetId: string): Promise {
+ const data = await undeleteHistoryDataset(datasetId);
return data as unknown as ItemSizeSummary;
}
-export async function purgeDatasetById(historyId: string, datasetId: string): Promise {
- const data = await purgeHistoryDataset(historyId, datasetId);
+export async function purgeDatasetById(datasetId: string): Promise {
+ const data = await purgeHistoryDataset(datasetId);
return data as unknown as PurgeableItemSizeSummary;
}
diff --git a/client/src/components/User/DiskUsage/Visualizations/util.ts b/client/src/components/User/DiskUsage/Visualizations/util.ts
new file mode 100644
index 000000000000..40fd0629a102
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/util.ts
@@ -0,0 +1,119 @@
+import { onMounted, ref } from "vue";
+
+import { useConfirmDialog } from "@/composables/confirmDialog";
+import { useToast } from "@/composables/toast";
+import localize from "@/utils/localization";
+
+import type { DataValuePoint } from "./Charts";
+import { bytesLabelFormatter, bytesValueFormatter } from "./Charts/formatters";
+import { type ItemSizeSummary, purgeDatasetById, undeleteDatasetById } from "./service";
+
+interface DataLoader {
+ (): Promise;
+}
+
+interface DataReload {
+ (): void;
+}
+
+export function useDatasetsToDisplay() {
+ const numberOfDatasetsToDisplayOptions = [10, 20, 50];
+ const numberOfDatasetsToDisplay = ref(numberOfDatasetsToDisplayOptions[0] || 10);
+ const numberOfDatasetsLimit = Math.max(...numberOfDatasetsToDisplayOptions);
+ const datasetsSizeSummaryMap = new Map();
+ const topNDatasetsBySizeData = ref(null);
+
+ function isRecoverableDataPoint(dataPoint?: DataValuePoint): boolean {
+ if (dataPoint) {
+ const datasetSizeSummary = datasetsSizeSummaryMap.get(dataPoint.id || "");
+ return datasetSizeSummary?.deleted || dataPoint.id === "deleted";
+ }
+ return false;
+ }
+
+ const { success: successToast, error: errorToast } = useToast();
+ const { confirm } = useConfirmDialog();
+
+ async function onUndeleteDataset(reloadData: DataReload, datasetId: string) {
+ try {
+ const result = await undeleteDatasetById(datasetId);
+ const dataset = datasetsSizeSummaryMap.get(datasetId);
+ if (dataset && !result.deleted) {
+ dataset.deleted = result.deleted;
+ datasetsSizeSummaryMap.set(datasetId, dataset);
+ successToast(localize("Dataset undeleted successfully."));
+ reloadData();
+ }
+ } catch (error) {
+ errorToast(`${error}`, localize("An error occurred while undeleting the dataset."));
+ }
+ }
+
+ async function onPermanentlyDeleteDataset(reloadData: DataReload, datasetId: string) {
+ const confirmed = await confirm(
+ localize("Are you sure you want to permanently delete this dataset? This action cannot be undone."),
+ {
+ title: localize("Permanently delete dataset?"),
+ okVariant: "danger",
+ okTitle: localize("Permanently delete"),
+ cancelTitle: localize("Cancel"),
+ }
+ );
+ if (!confirmed) {
+ return;
+ }
+ try {
+ const result = await purgeDatasetById(datasetId);
+ const dataset = datasetsSizeSummaryMap.get(datasetId);
+ if (dataset && result) {
+ datasetsSizeSummaryMap.delete(datasetId);
+ successToast(localize("Dataset permanently deleted successfully."));
+ reloadData();
+ }
+ } catch (error) {
+ errorToast(`${error}`, localize("An error occurred while permanently deleting the dataset."));
+ }
+ }
+
+ return {
+ numberOfDatasetsToDisplayOptions,
+ numberOfDatasetsToDisplay,
+ numberOfDatasetsLimit,
+ datasetsSizeSummaryMap,
+ topNDatasetsBySizeData,
+ isRecoverableDataPoint,
+ onUndeleteDataset,
+ onPermanentlyDeleteDataset,
+ };
+}
+
+export function useDataLoading() {
+ const isLoading = ref(true);
+
+ const loadDataOnMount = (dataLoader: DataLoader) => {
+ onMounted(async () => {
+ isLoading.value = true;
+ await dataLoader();
+ isLoading.value = false;
+ });
+ };
+ return {
+ isLoading,
+ loadDataOnMount,
+ };
+}
+
+export function buildTopNDatasetsBySizeData(datasetsSizeSummary: ItemSizeSummary[], n: number): DataValuePoint[] {
+ const topTenDatasetsBySize = datasetsSizeSummary.sort((a, b) => b.size - a.size).slice(0, n);
+ return topTenDatasetsBySize.map((dataset) => ({
+ id: dataset.id,
+ label: dataset.name,
+ value: dataset.size,
+ }));
+}
+
+export const byteFormattingForChart = {
+ "enable-selection": true,
+ labelFormatter: bytesLabelFormatter,
+ valueFormatter: bytesValueFormatter,
+};
diff --git a/client/src/entry/analysis/routes/storageDashboardRoutes.ts b/client/src/entry/analysis/routes/storageDashboardRoutes.ts
index 12abc3fce492..d1b2833512a2 100644
--- a/client/src/entry/analysis/routes/storageDashboardRoutes.ts
+++ b/client/src/entry/analysis/routes/storageDashboardRoutes.ts
@@ -2,6 +2,8 @@ import StorageManager from "@/components/User/DiskUsage/Management/StorageManage
import StorageDashboard from "@/components/User/DiskUsage/StorageDashboard.vue";
import HistoriesStorageOverview from "@/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue";
import HistoryStorageOverview from "@/components/User/DiskUsage/Visualizations/HistoryStorageOverview.vue";
+import ObjectStoresStorageOverview from "@/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue";
+import ObjectStoreStorageOverview from "@/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue";
import Base from "@/entry/analysis/modules/Base.vue";
export default [
@@ -32,6 +34,18 @@ export default [
component: HistoryStorageOverview,
props: true,
},
+ {
+ path: "objectstores",
+ name: "ObjectStoresOverview",
+ component: ObjectStoresStorageOverview,
+ props: true,
+ },
+ {
+ path: "objectstores/:objectStoreId",
+ name: "ObjectStoreOverview",
+ component: ObjectStoreStorageOverview,
+ props: true,
+ },
{
path: "*",
redirect: { name: "StorageDashboard" },
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 77d518b60618..223c8f2e0f20 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -657,6 +657,35 @@ def calculate_user_disk_usage_statements(user_id, quota_source_map, for_sqlite=F
return statements
+UNIQUE_DATASET_USER_USAGE_PER_OBJECTSTORE = """
+WITH per_user_histories AS
+(
+ SELECT id
+ FROM history
+ WHERE user_id = :id
+ AND NOT purged
+),
+per_hist_hdas AS (
+ SELECT DISTINCT dataset_id
+ FROM history_dataset_association
+ WHERE NOT purged
+ AND history_id IN (SELECT id FROM per_user_histories)
+)
+SELECT COALESCE(SUM(COALESCE(dataset.total_size, dataset.file_size, 0)), 0), dataset.object_store_id
+FROM dataset
+LEFT OUTER JOIN library_dataset_dataset_association ON dataset.id = library_dataset_dataset_association.dataset_id
+WHERE dataset.id IN (SELECT dataset_id FROM per_hist_hdas)
+ AND library_dataset_dataset_association.id IS NULL
+GROUP BY dataset.object_store_id
+"""
+
+
+def calculate_disk_usage_per_objectstore(sa_session, user_id: str):
+ statement = UNIQUE_DATASET_USER_USAGE_PER_OBJECTSTORE
+ params = {"id": user_id}
+ return sa_session.execute(statement, params).all()
+
+
# move these to galaxy.schema.schema once galaxy-data depends on
# galaxy-schema.
class UserQuotaBasicUsage(BaseModel):
@@ -670,6 +699,11 @@ class UserQuotaUsage(UserQuotaBasicUsage):
quota: Optional[str] = None
+class UserObjectstoreUsage(BaseModel):
+ object_store_id: str
+ total_disk_usage: float
+
+
class User(Base, Dictifiable, RepresentById):
"""
Data for a Galaxy user or admin and relations to their
@@ -1141,6 +1175,11 @@ def attempt_create_private_role(self):
with transaction(session):
session.commit()
+ def dictify_objectstore_usage(self) -> List[UserObjectstoreUsage]:
+ session = object_session(self)
+ rows = calculate_disk_usage_per_objectstore(session, self.id)
+ return [UserObjectstoreUsage(object_store_id=r[1], total_disk_usage=r[0]) for r in rows if r[1]]
+
def dictify_usage(self, object_store=None) -> List[UserQuotaBasicUsage]:
"""Include object_store to include empty/unused usage info."""
used_labels: Set[Union[str, None]] = set()
diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 4f1933514014..611c9888dd85 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -802,6 +802,23 @@ def update_typed(
trans, history_id, id, payload.model_dump(exclude_unset=True), serialization_params, contents_type=type
)
+ @router.put(
+ "/api/datasets/{dataset_id}",
+ summary="Updates the values for the history dataset (HDA) item with the given ``ID``.",
+ operation_id="datasets__update_dataset",
+ )
+ def update_dataset(
+ self,
+ dataset_id: HistoryItemIDPathParam,
+ trans: ProvidesHistoryContext = DependsOnTrans,
+ serialization_params: SerializationParams = Depends(query_serialization_params),
+ payload: UpdateHistoryContentsPayload = Body(...),
+ ) -> AnyHistoryContentItem:
+ """Updates the values for the history content item with the given ``ID``."""
+ return self.service.update(
+ trans, None, dataset_id, payload.model_dump(exclude_unset=True), serialization_params, contents_type=HistoryContentType.dataset
+ )
+
@router.put(
"/api/histories/{history_id}/contents/{id}",
summary="Updates the values for the history content item with the given ``ID`` and query specified type. ``/api/histories/{history_id}/contents/{type}s/{id}`` should be used instead.",
@@ -894,6 +911,40 @@ def delete_legacy(
payload,
)
+ @router.delete(
+ "/api/datasets/{dataset_id}",
+ summary="Delete the history dataset content with the given ``ID``.",
+ responses=CONTENT_DELETE_RESPONSES,
+ operation_id="datasets__delete",
+ )
+ def delete_dataset(
+ self,
+ response: Response,
+ dataset_id: HistoryItemIDPathParam,
+ trans: ProvidesHistoryContext = DependsOnTrans,
+ serialization_params: SerializationParams = Depends(query_serialization_params),
+ purge: Optional[bool] = PurgeQueryParam,
+ recursive: Optional[bool] = RecursiveQueryParam,
+ stop_job: Optional[bool] = StopJobQueryParam,
+ payload: DeleteHistoryContentPayload = Body(None),
+ ):
+ """
+ Delete the history content with the given ``ID`` and path specified type.
+
+ **Note**: Currently does not stop any active jobs for which this dataset is an output.
+ """
+ return self._delete(
+ response,
+ trans,
+ dataset_id,
+ HistoryContentType.dataset,
+ serialization_params,
+ purge,
+ recursive,
+ stop_job,
+ payload,
+ )
+
def _delete(
self,
response: Response,
diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py
index 03e1511c732d..4487dea7668e 100644
--- a/lib/galaxy/webapps/galaxy/api/users.py
+++ b/lib/galaxy/webapps/galaxy/api/users.py
@@ -38,6 +38,7 @@
HistoryDatasetAssociation,
Role,
UserAddress,
+ UserObjectstoreUsage,
UserQuotaUsage,
)
from galaxy.model.base import transaction
@@ -304,6 +305,21 @@ def usage(
else:
return []
+ @router.get(
+ "/api/users/{user_id}/objectstore_usage",
+ name="get_user_objectstore_usage",
+ summary="Return the user's object store usage summary broken down by object store ID",
+ )
+ def objectstore_usage(
+ self,
+ trans: ProvidesUserContext = DependsOnTrans,
+ user_id: FlexibleUserIdType = FlexibleUserIdPathParam,
+ ) -> List[UserObjectstoreUsage]:
+ if user := self.service.get_user_full(trans, user_id, False):
+ return user.dictify_objectstore_usage()
+ else:
+ return []
+
@router.get(
"/api/users/{user_id}/usage/{label}",
name="get_user_usage_for_label",
diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py
index 8ce9db729e45..4634b5113e71 100644
--- a/lib/galaxy/webapps/galaxy/services/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/services/history_contents.py
@@ -607,7 +607,7 @@ def update_permissions(
def update(
self,
trans,
- history_id: DecodedDatabaseIdField,
+ history_id: Optional[DecodedDatabaseIdField],
id: DecodedDatabaseIdField,
payload: Dict[str, Any],
serialization_params: SerializationParams,
@@ -624,6 +624,10 @@ def update(
:returns: an error object if an error occurred or a dictionary containing
any values that were different from the original and, therefore, updated
"""
+ if history_id is None:
+ hda = self.hda_manager.get_owned(id, trans.user, current_history=trans.history)
+ history_id = hda.history.id
+
history = self.history_manager.get_mutable(history_id, trans.user, current_history=trans.history)
if contents_type == HistoryContentType.dataset:
return self.__update_dataset(trans, history, id, payload, serialization_params)
diff --git a/test/unit/data/test_quota.py b/test/unit/data/test_quota.py
index 5aea03dac771..1582ab9c0f57 100644
--- a/test/unit/data/test_quota.py
+++ b/test/unit/data/test_quota.py
@@ -142,6 +142,20 @@ def test_calculate_usage_readjusts_incorrect_quota(self):
u.calculate_and_set_disk_usage(object_store)
self._refresh_user_and_assert_disk_usage_is(10)
+ def test_calculate_objectstore_usage(self):
+ # not strictly a quota check but such similar code and ideas...
+ u = self.u
+
+ self._add_dataset(10, "not_tracked")
+ self._add_dataset(15, "tracked")
+
+ usage = u.dictify_objectstore_usage()
+ assert len(usage) == 2
+
+ usage_dict = dict([(u.object_store_id, u.total_disk_usage) for u in usage])
+ assert int(usage_dict["not_tracked"]) == 10
+ assert int(usage_dict["tracked"]) == 15
+
def test_calculate_usage_disabled_quota(self):
u = self.u