From 1c07b630be885c431cf35bbcd07139821c7f2fc7 Mon Sep 17 00:00:00 2001
From: John Chilton
Date: Mon, 19 Feb 2024 12:07:23 -0500
Subject: [PATCH 1/7] OverviewPage component for re-use.
---
.../HistoriesStorageOverview.vue | 8 +++-----
.../Visualizations/HistoryStorageOverview.vue | 19 ++++++++-----------
.../DiskUsage/Visualizations/OverviewPage.vue | 19 +++++++++++++++++++
3 files changed, 30 insertions(+), 16 deletions(-)
create mode 100644 client/src/components/User/DiskUsage/Visualizations/OverviewPage.vue
diff --git a/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue b/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
index ae6a131ae39b..9d63b587c5e0 100644
--- a/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
+++ b/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
@@ -12,9 +12,9 @@ import { bytesLabelFormatter, bytesValueFormatter } from "./Charts/formatters";
import { fetchAllHistoriesSizeSummary, type ItemSizeSummary, purgeHistoryById, undeleteHistoryById } from "./service";
import BarChart from "./Charts/BarChart.vue";
+import OverviewPage from "./OverviewPage.vue";
import RecoverableItemSizeTooltip from "./RecoverableItemSizeTooltip.vue";
import SelectedItemActions from "./SelectedItemActions.vue";
-import Heading from "@/components/Common/Heading.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
const historyStore = useHistoryStore();
@@ -151,9 +151,7 @@ async function onPermanentlyDeleteHistory(historyId: string) {
}
-
-
{{ localize("Back to Dashboard") }}
-
Histories Storage Overview
+
Here you can find various graphs displaying the storage size taken by all your histories.
@@ -230,5 +228,5 @@ async function onPermanentlyDeleteHistory(historyId: string) {
-
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/HistoryStorageOverview.vue b/client/src/components/User/DiskUsage/Visualizations/HistoryStorageOverview.vue
index 42129841b95b..f6bf8c8434db 100644
--- a/client/src/components/User/DiskUsage/Visualizations/HistoryStorageOverview.vue
+++ b/client/src/components/User/DiskUsage/Visualizations/HistoryStorageOverview.vue
@@ -17,9 +17,9 @@ import {
} from "./service";
import BarChart from "./Charts/BarChart.vue";
+import OverviewPage from "./OverviewPage.vue";
import RecoverableItemSizeTooltip from "./RecoverableItemSizeTooltip.vue";
import SelectedItemActions from "./SelectedItemActions.vue";
-import Heading from "@/components/Common/Heading.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
const router = useRouter();
@@ -27,12 +27,11 @@ const { success: successToast, error: errorToast } = useToast();
const { confirm } = useConfirmDialog();
const { getHistoryNameById } = useHistoryStore();
-const props = defineProps({
- historyId: {
- type: String,
- required: true,
- },
-});
+interface Props {
+ historyId: string;
+}
+
+const props = defineProps();
const datasetsSizeSummaryMap = new Map();
const topTenDatasetsBySizeData = ref(null);
@@ -149,9 +148,7 @@ async function onPermanentlyDeleteDataset(datasetId: string) {
}
-
-
{{ localize("Back to Dashboard") }}
-
History Storage Overview
+
Here you will find some Graphs displaying the storage taken by datasets in your history:
{{ getHistoryNameById(props.historyId) }}
-
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/OverviewPage.vue b/client/src/components/User/DiskUsage/Visualizations/OverviewPage.vue
new file mode 100644
index 000000000000..932742f5bafd
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/OverviewPage.vue
@@ -0,0 +1,19 @@
+
+
+
+
+ {{ localize("Back to Dashboard") }}
+ {{ localize(title) }}
+
+
+
From 77ef4a3896e045a05cdae5009cc24553ceef3cc7 Mon Sep 17 00:00:00 2001
From: John Chilton
Date: Mon, 19 Feb 2024 13:40:39 -0500
Subject: [PATCH 2/7] objectstores storage management (UI+API)
---
client/src/api/datasets.ts | 18 +--
client/src/api/users.ts | 1 +
.../User/DiskUsage/StorageDashboard.vue | 35 ++++-
.../HistoriesStorageOverview.vue | 26 ++--
.../Visualizations/HistoryStorageOverview.vue | 135 +++++-------------
.../Visualizations/ObjectStoreActions.vue | 48 +++++++
.../ObjectStoreStorageOverview.vue | 123 ++++++++++++++++
.../ObjectStoresStorageOverview.vue | 71 +++++++++
.../Visualizations/ShowObjectStore.vue | 50 +++++++
.../Visualizations/WarnDeletedDatasets.vue | 9 ++
.../Visualizations/WarnDeletedHistories.vue | 9 ++
.../User/DiskUsage/Visualizations/service.ts | 19 ++-
.../User/DiskUsage/Visualizations/util.ts | 119 +++++++++++++++
.../analysis/routes/storageDashboardRoutes.ts | 14 ++
lib/galaxy/model/__init__.py | 39 +++++
.../webapps/galaxy/api/history_contents.py | 56 ++++++++
lib/galaxy/webapps/galaxy/api/users.py | 16 +++
.../galaxy/services/history_contents.py | 6 +-
test/unit/data/test_quota.py | 14 ++
19 files changed, 671 insertions(+), 137 deletions(-)
create mode 100644 client/src/components/User/DiskUsage/Visualizations/ObjectStoreActions.vue
create mode 100644 client/src/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue
create mode 100644 client/src/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue
create mode 100644 client/src/components/User/DiskUsage/Visualizations/ShowObjectStore.vue
create mode 100644 client/src/components/User/DiskUsage/Visualizations/WarnDeletedDatasets.vue
create mode 100644 client/src/components/User/DiskUsage/Visualizations/WarnDeletedHistories.vue
create mode 100644 client/src/components/User/DiskUsage/Visualizations/util.ts
diff --git a/client/src/api/datasets.ts b/client/src/api/datasets.ts
index 47ee27b2b6a7..120555901521 100644
--- a/client/src/api/datasets.ts
+++ b/client/src/api/datasets.ts
@@ -49,25 +49,21 @@ export async function fetchDatasetDetails(params: { id: string }): Promise
@@ -60,10 +72,19 @@ function goToHistoriesOverview() {
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue b/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
index 9d63b587c5e0..770c274e4740 100644
--- a/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
+++ b/client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
@@ -1,5 +1,5 @@
@@ -157,36 +107,28 @@ async function onPermanentlyDeleteDataset(datasetId: string) {
Histories Storage Overview page to see
the storage taken by all your histories.
-
- Note: these graphs include deleted datasets. Remember that, even if you delete datasets, they still
- take up storage space. However, you can free up the storage space by permanently deleting them from the
- Discarded Items section of the
- Storage Manager page or by selecting them
- individually in the graph and clicking the Permanently Delete button.
-
-
+
+ v-bind="byteFormattingForChart">
{{ localize(`Top ${numberOfDatasetsToDisplay} Datasets by Size`) }}
@@ -204,8 +146,8 @@ async function onPermanentlyDeleteDataset(datasetId: string) {
item-type="dataset"
:is-recoverable="isRecoverableDataPoint(data)"
@view-item="onViewDataset"
- @undelete-item="onUndeleteDataset"
- @permanently-delete-item="onPermanentlyDeleteDataset" />
+ @undelete-item="onUndelete"
+ @permanently-delete-item="onPermDelete" />
@@ -218,8 +160,7 @@ async function onPermanentlyDeleteDataset(datasetId: string) {
)
"
:data="activeVsDeletedTotalSizeData"
- :label-formatter="bytesLabelFormatter"
- :value-formatter="bytesValueFormatter">
+ v-bind="byteFormattingForChart">
+import { library } from "@fortawesome/fontawesome-svg-core";
+import { faChartBar } from "@fortawesome/free-solid-svg-icons";
+import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
+import { BButton } from "bootstrap-vue";
+import { computed } from "vue";
+
+import localize from "@/utils/localization";
+
+import type { DataValuePoint } from "./Charts";
+
+interface Props {
+ data: DataValuePoint;
+}
+
+const props = defineProps();
+
+library.add(faChartBar);
+
+const label = computed(() => props.data.id);
+const viewDetailsIcon = computed(() => "chart-bar");
+
+const emit = defineEmits<{
+ (e: "view-item", itemId: string): void;
+}>();
+
+function onViewItem() {
+ emit("view-item", props.data.id);
+}
+
+
+
+
+ {{ label }}
+
+
+
+
+
+
+
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue b/client/src/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue
new file mode 100644
index 000000000000..06237e694f62
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue
@@ -0,0 +1,123 @@
+
+
+
+
+
+ Here you will find some Graphs displaying the storage taken by datasets in the object store:
+ {{ objectStoreId }}. You can use these graphs to identify the datasets that take the most space in this object store.
+
+
+
+
+
+
+
+
+ {{ localize(`Top ${numberOfDatasetsToDisplay} Datasets by Size`) }}
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue b/client/src/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue
new file mode 100644
index 000000000000..79af90cf7bb3
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue
@@ -0,0 +1,71 @@
+
+
+
+
+ Here you can find various graphs displaying the storage size taken by all your histories grouped by object
+ store.
+
+
+
+
+
+
+
+
+ {{ localize(`Object Stores by Usage`) }}
+
+
+
+
+
+
+
+
+
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/ShowObjectStore.vue b/client/src/components/User/DiskUsage/Visualizations/ShowObjectStore.vue
new file mode 100644
index 000000000000..313f2f6c0142
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/ShowObjectStore.vue
@@ -0,0 +1,50 @@
+
+
+
+
+
+
+
+ {{ error }}
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/WarnDeletedDatasets.vue b/client/src/components/User/DiskUsage/Visualizations/WarnDeletedDatasets.vue
new file mode 100644
index 000000000000..65ac06f9a170
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/WarnDeletedDatasets.vue
@@ -0,0 +1,9 @@
+
+
+ Note: these graphs include deleted datasets. Remember that, even if you delete datasets, they still take
+ up storage space. However, you can free up the storage space by permanently deleting them from the
+ Discarded Items section of the
+ Storage Manager page or by selecting them
+ individually in the graph and clicking the Permanently Delete button.
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/WarnDeletedHistories.vue b/client/src/components/User/DiskUsage/Visualizations/WarnDeletedHistories.vue
new file mode 100644
index 000000000000..aa11326c2e07
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/WarnDeletedHistories.vue
@@ -0,0 +1,9 @@
+
+
+ Note: these graphs include deleted histories. Remember that, even if you delete histories, they still
+ take up storage space. However, you can free up the storage space by permanently deleting them from the
+ Discarded Items section of the
+ Storage Manager page or by selecting them
+ individually in the graph and clicking the Permanently Delete button.
+
+
diff --git a/client/src/components/User/DiskUsage/Visualizations/service.ts b/client/src/components/User/DiskUsage/Visualizations/service.ts
index 2a83827c8cd8..e88481eca64f 100644
--- a/client/src/components/User/DiskUsage/Visualizations/service.ts
+++ b/client/src/components/User/DiskUsage/Visualizations/service.ts
@@ -45,6 +45,17 @@ export async function fetchHistoryContentsSizeSummary(historyId: string, limit =
return response.data as unknown as ItemSizeSummary[];
}
+export async function fetchObjectStoreContentsSizeSummary(objectStoreId: string, limit = 5000) {
+ const response = await datasetsFetcher({
+ keys: itemSizeSummaryFields,
+ limit,
+ order: "size-dsc",
+ q: ["purged", "history_content_type", "object_store_id"],
+ qv: ["false", "dataset", objectStoreId],
+ });
+ return response.data as unknown as ItemSizeSummary[];
+}
+
export async function undeleteHistoryById(historyId: string): Promise {
const response = await undeleteHistory({ history_id: historyId });
return response.data as unknown as ItemSizeSummary;
@@ -55,12 +66,12 @@ export async function purgeHistoryById(historyId: string): Promise {
- const data = await undeleteHistoryDataset(historyId, datasetId);
+export async function undeleteDatasetById(datasetId: string): Promise {
+ const data = await undeleteHistoryDataset(datasetId);
return data as unknown as ItemSizeSummary;
}
-export async function purgeDatasetById(historyId: string, datasetId: string): Promise {
- const data = await purgeHistoryDataset(historyId, datasetId);
+export async function purgeDatasetById(datasetId: string): Promise {
+ const data = await purgeHistoryDataset(datasetId);
return data as unknown as PurgeableItemSizeSummary;
}
diff --git a/client/src/components/User/DiskUsage/Visualizations/util.ts b/client/src/components/User/DiskUsage/Visualizations/util.ts
new file mode 100644
index 000000000000..40fd0629a102
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Visualizations/util.ts
@@ -0,0 +1,119 @@
+import { onMounted, ref } from "vue";
+
+import { useConfirmDialog } from "@/composables/confirmDialog";
+import { useToast } from "@/composables/toast";
+import localize from "@/utils/localization";
+
+import type { DataValuePoint } from "./Charts";
+import { bytesLabelFormatter, bytesValueFormatter } from "./Charts/formatters";
+import { type ItemSizeSummary, purgeDatasetById, undeleteDatasetById } from "./service";
+
+interface DataLoader {
+ (): Promise;
+}
+
+interface DataReload {
+ (): void;
+}
+
+export function useDatasetsToDisplay() {
+ const numberOfDatasetsToDisplayOptions = [10, 20, 50];
+ const numberOfDatasetsToDisplay = ref(numberOfDatasetsToDisplayOptions[0] || 10);
+ const numberOfDatasetsLimit = Math.max(...numberOfDatasetsToDisplayOptions);
+ const datasetsSizeSummaryMap = new Map();
+ const topNDatasetsBySizeData = ref(null);
+
+ function isRecoverableDataPoint(dataPoint?: DataValuePoint): boolean {
+ if (dataPoint) {
+ const datasetSizeSummary = datasetsSizeSummaryMap.get(dataPoint.id || "");
+ return datasetSizeSummary?.deleted || dataPoint.id === "deleted";
+ }
+ return false;
+ }
+
+ const { success: successToast, error: errorToast } = useToast();
+ const { confirm } = useConfirmDialog();
+
+ async function onUndeleteDataset(reloadData: DataReload, datasetId: string) {
+ try {
+ const result = await undeleteDatasetById(datasetId);
+ const dataset = datasetsSizeSummaryMap.get(datasetId);
+ if (dataset && !result.deleted) {
+ dataset.deleted = result.deleted;
+ datasetsSizeSummaryMap.set(datasetId, dataset);
+ successToast(localize("Dataset undeleted successfully."));
+ reloadData();
+ }
+ } catch (error) {
+ errorToast(`${error}`, localize("An error occurred while undeleting the dataset."));
+ }
+ }
+
+ async function onPermanentlyDeleteDataset(reloadData: DataReload, datasetId: string) {
+ const confirmed = await confirm(
+ localize("Are you sure you want to permanently delete this dataset? This action cannot be undone."),
+ {
+ title: localize("Permanently delete dataset?"),
+ okVariant: "danger",
+ okTitle: localize("Permanently delete"),
+ cancelTitle: localize("Cancel"),
+ }
+ );
+ if (!confirmed) {
+ return;
+ }
+ try {
+ const result = await purgeDatasetById(datasetId);
+ const dataset = datasetsSizeSummaryMap.get(datasetId);
+ if (dataset && result) {
+ datasetsSizeSummaryMap.delete(datasetId);
+ successToast(localize("Dataset permanently deleted successfully."));
+ reloadData();
+ }
+ } catch (error) {
+ errorToast(`${error}`, localize("An error occurred while permanently deleting the dataset."));
+ }
+ }
+
+ return {
+ numberOfDatasetsToDisplayOptions,
+ numberOfDatasetsToDisplay,
+ numberOfDatasetsLimit,
+ datasetsSizeSummaryMap,
+ topNDatasetsBySizeData,
+ isRecoverableDataPoint,
+ onUndeleteDataset,
+ onPermanentlyDeleteDataset,
+ };
+}
+
+export function useDataLoading() {
+ const isLoading = ref(true);
+
+ const loadDataOnMount = (dataLoader: DataLoader) => {
+ onMounted(async () => {
+ isLoading.value = true;
+ await dataLoader();
+ isLoading.value = false;
+ });
+ };
+ return {
+ isLoading,
+ loadDataOnMount,
+ };
+}
+
+export function buildTopNDatasetsBySizeData(datasetsSizeSummary: ItemSizeSummary[], n: number): DataValuePoint[] {
+ const topTenDatasetsBySize = datasetsSizeSummary.sort((a, b) => b.size - a.size).slice(0, n);
+ return topTenDatasetsBySize.map((dataset) => ({
+ id: dataset.id,
+ label: dataset.name,
+ value: dataset.size,
+ }));
+}
+
+export const byteFormattingForChart = {
+ "enable-selection": true,
+ labelFormatter: bytesLabelFormatter,
+ valueFormatter: bytesValueFormatter,
+};
diff --git a/client/src/entry/analysis/routes/storageDashboardRoutes.ts b/client/src/entry/analysis/routes/storageDashboardRoutes.ts
index 12abc3fce492..d1b2833512a2 100644
--- a/client/src/entry/analysis/routes/storageDashboardRoutes.ts
+++ b/client/src/entry/analysis/routes/storageDashboardRoutes.ts
@@ -2,6 +2,8 @@ import StorageManager from "@/components/User/DiskUsage/Management/StorageManage
import StorageDashboard from "@/components/User/DiskUsage/StorageDashboard.vue";
import HistoriesStorageOverview from "@/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue";
import HistoryStorageOverview from "@/components/User/DiskUsage/Visualizations/HistoryStorageOverview.vue";
+import ObjectStoresStorageOverview from "@/components/User/DiskUsage/Visualizations/ObjectStoresStorageOverview.vue";
+import ObjectStoreStorageOverview from "@/components/User/DiskUsage/Visualizations/ObjectStoreStorageOverview.vue";
import Base from "@/entry/analysis/modules/Base.vue";
export default [
@@ -32,6 +34,18 @@ export default [
component: HistoryStorageOverview,
props: true,
},
+ {
+ path: "objectstores",
+ name: "ObjectStoresOverview",
+ component: ObjectStoresStorageOverview,
+ props: true,
+ },
+ {
+ path: "objectstores/:objectStoreId",
+ name: "ObjectStoreOverview",
+ component: ObjectStoreStorageOverview,
+ props: true,
+ },
{
path: "*",
redirect: { name: "StorageDashboard" },
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 235eb89ee3e8..aea13441379d 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -657,6 +657,35 @@ def calculate_user_disk_usage_statements(user_id, quota_source_map, for_sqlite=F
return statements
+UNIQUE_DATASET_USER_USAGE_PER_OBJECTSTORE = """
+WITH per_user_histories AS
+(
+ SELECT id
+ FROM history
+ WHERE user_id = :id
+ AND NOT purged
+),
+per_hist_hdas AS (
+ SELECT DISTINCT dataset_id
+ FROM history_dataset_association
+ WHERE NOT purged
+ AND history_id IN (SELECT id FROM per_user_histories)
+)
+SELECT COALESCE(SUM(COALESCE(dataset.total_size, dataset.file_size, 0)), 0), dataset.object_store_id
+FROM dataset
+LEFT OUTER JOIN library_dataset_dataset_association ON dataset.id = library_dataset_dataset_association.dataset_id
+WHERE dataset.id IN (SELECT dataset_id FROM per_hist_hdas)
+ AND library_dataset_dataset_association.id IS NULL
+GROUP BY dataset.object_store_id
+"""
+
+
+def calculate_disk_usage_per_objectstore(sa_session, user_id: str):
+ statement = UNIQUE_DATASET_USER_USAGE_PER_OBJECTSTORE
+ params = {"id": user_id}
+ return sa_session.execute(statement, params).all()
+
+
# move these to galaxy.schema.schema once galaxy-data depends on
# galaxy-schema.
class UserQuotaBasicUsage(BaseModel):
@@ -670,6 +699,11 @@ class UserQuotaUsage(UserQuotaBasicUsage):
quota: Optional[str] = None
+class UserObjectstoreUsage(BaseModel):
+ object_store_id: str
+ total_disk_usage: float
+
+
class User(Base, Dictifiable, RepresentById):
"""
Data for a Galaxy user or admin and relations to their
@@ -1141,6 +1175,11 @@ def attempt_create_private_role(self):
with transaction(session):
session.commit()
+ def dictify_objectstore_usage(self) -> List[UserObjectstoreUsage]:
+ session = object_session(self)
+ rows = calculate_disk_usage_per_objectstore(session, self.id)
+ return [UserObjectstoreUsage(object_store_id=r[1], total_disk_usage=r[0]) for r in rows if r[1]]
+
def dictify_usage(self, object_store=None) -> List[UserQuotaBasicUsage]:
"""Include object_store to include empty/unused usage info."""
used_labels: Set[Union[str, None]] = set()
diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index 3cad6b3d0475..8ea3736b4299 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -810,6 +810,28 @@ def update_typed(
trans, history_id, id, payload.model_dump(exclude_unset=True), serialization_params, contents_type=type
)
+ @router.put(
+ "/api/datasets/{dataset_id}",
+ summary="Updates the values for the history dataset (HDA) item with the given ``ID``.",
+ operation_id="datasets__update_dataset",
+ )
+ def update_dataset(
+ self,
+ dataset_id: HistoryItemIDPathParam,
+ trans: ProvidesHistoryContext = DependsOnTrans,
+ serialization_params: SerializationParams = Depends(query_serialization_params),
+ payload: UpdateHistoryContentsPayload = Body(...),
+ ) -> AnyHistoryContentItem:
+ """Updates the values for the history content item with the given ``ID``."""
+ return self.service.update(
+ trans,
+ None,
+ dataset_id,
+ payload.model_dump(exclude_unset=True),
+ serialization_params,
+ contents_type=HistoryContentType.dataset,
+ )
+
@router.put(
"/api/histories/{history_id}/contents/{id}",
summary="Updates the values for the history content item with the given ``ID`` and query specified type. ``/api/histories/{history_id}/contents/{type}s/{id}`` should be used instead.",
@@ -903,6 +925,40 @@ def delete_legacy(
payload,
)
+ @router.delete(
+ "/api/datasets/{dataset_id}",
+ summary="Delete the history dataset content with the given ``ID``.",
+ responses=CONTENT_DELETE_RESPONSES,
+ operation_id="datasets__delete",
+ )
+ def delete_dataset(
+ self,
+ response: Response,
+ dataset_id: HistoryItemIDPathParam,
+ trans: ProvidesHistoryContext = DependsOnTrans,
+ serialization_params: SerializationParams = Depends(query_serialization_params),
+ purge: Optional[bool] = PurgeQueryParam,
+ recursive: Optional[bool] = RecursiveQueryParam,
+ stop_job: Optional[bool] = StopJobQueryParam,
+ payload: DeleteHistoryContentPayload = Body(None),
+ ):
+ """
+ Delete the history content with the given ``ID`` and path specified type.
+
+ **Note**: Currently does not stop any active jobs for which this dataset is an output.
+ """
+ return self._delete(
+ response,
+ trans,
+ dataset_id,
+ HistoryContentType.dataset,
+ serialization_params,
+ purge,
+ recursive,
+ stop_job,
+ payload,
+ )
+
def _delete(
self,
response: Response,
diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py
index 5dee3cdb63e2..7b75e936d86b 100644
--- a/lib/galaxy/webapps/galaxy/api/users.py
+++ b/lib/galaxy/webapps/galaxy/api/users.py
@@ -38,6 +38,7 @@
HistoryDatasetAssociation,
Role,
UserAddress,
+ UserObjectstoreUsage,
UserQuotaUsage,
)
from galaxy.model.base import transaction
@@ -304,6 +305,21 @@ def usage(
else:
return []
+ @router.get(
+ "/api/users/{user_id}/objectstore_usage",
+ name="get_user_objectstore_usage",
+ summary="Return the user's object store usage summary broken down by object store ID",
+ )
+ def objectstore_usage(
+ self,
+ trans: ProvidesUserContext = DependsOnTrans,
+ user_id: FlexibleUserIdType = FlexibleUserIdPathParam,
+ ) -> List[UserObjectstoreUsage]:
+ if user := self.service.get_user_full(trans, user_id, False):
+ return user.dictify_objectstore_usage()
+ else:
+ return []
+
@router.get(
"/api/users/{user_id}/usage/{label}",
name="get_user_usage_for_label",
diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py
index 3a69614db529..6480c4d00469 100644
--- a/lib/galaxy/webapps/galaxy/services/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/services/history_contents.py
@@ -610,7 +610,7 @@ def update_permissions(
def update(
self,
trans,
- history_id: DecodedDatabaseIdField,
+ history_id: Optional[DecodedDatabaseIdField],
id: DecodedDatabaseIdField,
payload: Dict[str, Any],
serialization_params: SerializationParams,
@@ -627,6 +627,10 @@ def update(
:returns: an error object if an error occurred or a dictionary containing
any values that were different from the original and, therefore, updated
"""
+ if history_id is None:
+ hda = self.hda_manager.get_owned(id, trans.user, current_history=trans.history)
+ history_id = hda.history.id
+
history = self.history_manager.get_mutable(history_id, trans.user, current_history=trans.history)
if contents_type == HistoryContentType.dataset:
return self.__update_dataset(trans, history, id, payload, serialization_params)
diff --git a/test/unit/data/test_quota.py b/test/unit/data/test_quota.py
index 5aea03dac771..1582ab9c0f57 100644
--- a/test/unit/data/test_quota.py
+++ b/test/unit/data/test_quota.py
@@ -142,6 +142,20 @@ def test_calculate_usage_readjusts_incorrect_quota(self):
u.calculate_and_set_disk_usage(object_store)
self._refresh_user_and_assert_disk_usage_is(10)
+ def test_calculate_objectstore_usage(self):
+ # not strictly a quota check but such similar code and ideas...
+ u = self.u
+
+ self._add_dataset(10, "not_tracked")
+ self._add_dataset(15, "tracked")
+
+ usage = u.dictify_objectstore_usage()
+ assert len(usage) == 2
+
+ usage_dict = dict([(u.object_store_id, u.total_disk_usage) for u in usage])
+ assert int(usage_dict["not_tracked"]) == 10
+ assert int(usage_dict["tracked"]) == 15
+
def test_calculate_usage_disabled_quota(self):
u = self.u
From 40a220359be59ad091a91b38c4ed967e6eda7bd7 Mon Sep 17 00:00:00 2001
From: John Chilton
Date: Mon, 25 Mar 2024 14:01:15 -0400
Subject: [PATCH 3/7] Rename purgeHistoryDataset to purgeDataset...
... per David's comment.
---
client/src/api/datasets.ts | 2 +-
.../src/components/User/DiskUsage/Visualizations/service.ts | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/client/src/api/datasets.ts b/client/src/api/datasets.ts
index 120555901521..914a87a21651 100644
--- a/client/src/api/datasets.ts
+++ b/client/src/api/datasets.ts
@@ -62,7 +62,7 @@ export async function undeleteHistoryDataset(datasetId: string) {
const deleteDataset = fetcher.path("/api/datasets/{dataset_id}").method("delete").create();
-export async function purgeHistoryDataset(datasetId: string) {
+export async function purgeDataset(datasetId: string) {
const { data } = await deleteDataset({ dataset_id: datasetId, purge: true });
return data;
}
diff --git a/client/src/components/User/DiskUsage/Visualizations/service.ts b/client/src/components/User/DiskUsage/Visualizations/service.ts
index e88481eca64f..e656f0014cb0 100644
--- a/client/src/components/User/DiskUsage/Visualizations/service.ts
+++ b/client/src/components/User/DiskUsage/Visualizations/service.ts
@@ -1,4 +1,4 @@
-import { datasetsFetcher, purgeHistoryDataset, undeleteHistoryDataset } from "@/api/datasets";
+import { datasetsFetcher, purgeDataset, undeleteHistoryDataset } from "@/api/datasets";
import { archivedHistoriesFetcher, deleteHistory, historiesFetcher, undeleteHistory } from "@/api/histories";
export interface ItemSizeSummary {
@@ -72,6 +72,6 @@ export async function undeleteDatasetById(datasetId: string): Promise {
- const data = await purgeHistoryDataset(datasetId);
+ const data = await purgeDataset(datasetId);
return data as unknown as PurgeableItemSizeSummary;
}
From 77d43ac02a4080aed0a54124ff918b53b8fa62ff Mon Sep 17 00:00:00 2001
From: John Chilton
Date: Mon, 25 Mar 2024 14:05:21 -0400
Subject: [PATCH 4/7] Improve SQL per jdavcs' comments.
---
lib/galaxy/model/__init__.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index aea13441379d..eea682b87e53 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -671,7 +671,7 @@ def calculate_user_disk_usage_statements(user_id, quota_source_map, for_sqlite=F
WHERE NOT purged
AND history_id IN (SELECT id FROM per_user_histories)
)
-SELECT COALESCE(SUM(COALESCE(dataset.total_size, dataset.file_size, 0)), 0), dataset.object_store_id
+SELECT SUM(COALESCE(dataset.total_size, dataset.file_size, 0)) as usage, dataset.object_store_id
FROM dataset
LEFT OUTER JOIN library_dataset_dataset_association ON dataset.id = library_dataset_dataset_association.dataset_id
WHERE dataset.id IN (SELECT dataset_id FROM per_hist_hdas)
@@ -1178,7 +1178,11 @@ def attempt_create_private_role(self):
def dictify_objectstore_usage(self) -> List[UserObjectstoreUsage]:
session = object_session(self)
rows = calculate_disk_usage_per_objectstore(session, self.id)
- return [UserObjectstoreUsage(object_store_id=r[1], total_disk_usage=r[0]) for r in rows if r[1]]
+ return [
+ UserObjectstoreUsage(object_store_id=r.object_store_id, total_disk_usage=r.usage)
+ for r in rows
+ if r.object_store_id
+ ]
def dictify_usage(self, object_store=None) -> List[UserQuotaBasicUsage]:
"""Include object_store to include empty/unused usage info."""
From 798994783b11bf2b7c8a853c70ed5892786af44e Mon Sep 17 00:00:00 2001
From: John Chilton
Date: Mon, 25 Mar 2024 14:06:14 -0400
Subject: [PATCH 5/7] Rename for more standard typescript.
---
client/src/components/User/DiskUsage/Visualizations/util.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/client/src/components/User/DiskUsage/Visualizations/util.ts b/client/src/components/User/DiskUsage/Visualizations/util.ts
index 40fd0629a102..cfa80d2c0cd9 100644
--- a/client/src/components/User/DiskUsage/Visualizations/util.ts
+++ b/client/src/components/User/DiskUsage/Visualizations/util.ts
@@ -9,7 +9,7 @@ import { bytesLabelFormatter, bytesValueFormatter } from "./Charts/formatters";
import { type ItemSizeSummary, purgeDatasetById, undeleteDatasetById } from "./service";
interface DataLoader {
- (): Promise;
+ (): Promise;
}
interface DataReload {
From 383057d9b88386b3ba38ea0a0368592f6f52354b Mon Sep 17 00:00:00 2001
From: John Chilton
Date: Wed, 27 Mar 2024 10:21:24 -0400
Subject: [PATCH 6/7] Linting fix.
---
test/unit/data/test_quota.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/unit/data/test_quota.py b/test/unit/data/test_quota.py
index 1582ab9c0f57..96187045829d 100644
--- a/test/unit/data/test_quota.py
+++ b/test/unit/data/test_quota.py
@@ -152,7 +152,7 @@ def test_calculate_objectstore_usage(self):
usage = u.dictify_objectstore_usage()
assert len(usage) == 2
- usage_dict = dict([(u.object_store_id, u.total_disk_usage) for u in usage])
+ usage_dict = {u.object_store_id: u.total_disk_usage for u in usage}
assert int(usage_dict["not_tracked"]) == 10
assert int(usage_dict["tracked"]) == 15
From 5bbbd04f4b18f296441ab4a04ec0f3bec5eb75a6 Mon Sep 17 00:00:00 2001
From: John Chilton
Date: Mon, 19 Feb 2024 14:35:09 -0500
Subject: [PATCH 7/7] Rebuild schema.
---
client/src/api/schema/schema.ts | 160 ++++++++++++++++++++++++++++++++
1 file changed, 160 insertions(+)
diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts
index 8865c361e282..cbed1a54e0b8 100644
--- a/client/src/api/schema/schema.ts
+++ b/client/src/api/schema/schema.ts
@@ -132,6 +132,18 @@ export interface paths {
* To get more information please check the source code.
*/
get: operations["show_api_datasets__dataset_id__get"];
+ /**
+ * Updates the values for the history dataset (HDA) item with the given ``ID``.
+ * @description Updates the values for the history content item with the given ``ID``.
+ */
+ put: operations["datasets__update_dataset"];
+ /**
+ * Delete the history dataset content with the given ``ID``.
+ * @description Delete the history content with the given ``ID`` and path specified type.
+ *
+ * **Note**: Currently does not stop any active jobs for which this dataset is an output.
+ */
+ delete: operations["datasets__delete"];
};
"/api/datasets/{dataset_id}/content/{content_type}": {
/** Retrieve information about the content of a dataset. */
@@ -1705,6 +1717,10 @@ export interface paths {
/** Remove the object from user's favorites */
delete: operations["remove_favorite_api_users__user_id__favorites__object_type___object_id__delete"];
};
+ "/api/users/{user_id}/objectstore_usage": {
+ /** Return the user's object store usage summary broken down by object store ID */
+ get: operations["get_user_objectstore_usage_api_users__user_id__objectstore_usage_get"];
+ };
"/api/users/{user_id}/recalculate_disk_usage": {
/** Triggers a recalculation of the current user disk usage. */
put: operations["recalculate_disk_usage_by_user_id_api_users__user_id__recalculate_disk_usage_put"];
@@ -12459,6 +12475,13 @@ export interface components {
*/
notification_ids: string[];
};
+ /** UserObjectstoreUsage */
+ UserObjectstoreUsage: {
+ /** Object Store Id */
+ object_store_id: string;
+ /** Total Disk Usage */
+ total_disk_usage: number;
+ };
/** UserQuota */
UserQuota: {
/**
@@ -13599,6 +13622,116 @@ export interface operations {
};
};
};
+ datasets__update_dataset: {
+ /**
+ * Updates the values for the history dataset (HDA) item with the given ``ID``.
+ * @description Updates the values for the history content item with the given ``ID``.
+ */
+ parameters: {
+ /** @description View to be passed to the serializer */
+ /** @description Comma-separated list of keys to be passed to the serializer */
+ query?: {
+ view?: string | null;
+ keys?: string | null;
+ };
+ /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+ header?: {
+ "run-as"?: string | null;
+ };
+ /** @description The ID of the item (`HDA`/`HDCA`) */
+ path: {
+ dataset_id: string;
+ };
+ };
+ requestBody: {
+ content: {
+ "application/json": components["schemas"]["UpdateHistoryContentsPayload"];
+ };
+ };
+ responses: {
+ /** @description Successful Response */
+ 200: {
+ content: {
+ "application/json":
+ | components["schemas"]["HDACustom"]
+ | components["schemas"]["HDADetailed"]
+ | components["schemas"]["HDASummary"]
+ | components["schemas"]["HDCADetailed"]
+ | components["schemas"]["HDCASummary"];
+ };
+ };
+ /** @description Validation Error */
+ 422: {
+ content: {
+ "application/json": components["schemas"]["HTTPValidationError"];
+ };
+ };
+ };
+ };
+ datasets__delete: {
+ /**
+ * Delete the history dataset content with the given ``ID``.
+ * @description Delete the history content with the given ``ID`` and path specified type.
+ *
+ * **Note**: Currently does not stop any active jobs for which this dataset is an output.
+ */
+ parameters: {
+ /**
+ * @deprecated
+ * @description Whether to remove from disk the target HDA or child HDAs of the target HDCA.
+ */
+ /**
+ * @deprecated
+ * @description When deleting a dataset collection, whether to also delete containing datasets.
+ */
+ /**
+ * @deprecated
+ * @description Whether to stop the creating job if all outputs of the job have been deleted.
+ */
+ /** @description View to be passed to the serializer */
+ /** @description Comma-separated list of keys to be passed to the serializer */
+ query?: {
+ purge?: boolean | null;
+ recursive?: boolean | null;
+ stop_job?: boolean | null;
+ view?: string | null;
+ keys?: string | null;
+ };
+ /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+ header?: {
+ "run-as"?: string | null;
+ };
+ /** @description The ID of the item (`HDA`/`HDCA`) */
+ path: {
+ dataset_id: string;
+ };
+ };
+ requestBody?: {
+ content: {
+ "application/json": components["schemas"]["DeleteHistoryContentPayload"];
+ };
+ };
+ responses: {
+ /** @description Request has been executed. */
+ 200: {
+ content: {
+ "application/json": components["schemas"]["DeleteHistoryContentResult"];
+ };
+ };
+ /** @description Request accepted, processing will finish later. */
+ 202: {
+ content: {
+ "application/json": components["schemas"]["DeleteHistoryContentResult"];
+ };
+ };
+ /** @description Validation Error */
+ 422: {
+ content: {
+ "application/json": components["schemas"]["HTTPValidationError"];
+ };
+ };
+ };
+ };
get_structured_content_api_datasets__dataset_id__content__content_type__get: {
/** Retrieve information about the content of a dataset. */
parameters: {
@@ -22575,6 +22708,33 @@ export interface operations {
};
};
};
+ get_user_objectstore_usage_api_users__user_id__objectstore_usage_get: {
+ /** Return the user's object store usage summary broken down by object store ID */
+ parameters: {
+ /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+ header?: {
+ "run-as"?: string | null;
+ };
+ /** @description The ID of the user to get or 'current'. */
+ path: {
+ user_id: string | "current";
+ };
+ };
+ responses: {
+ /** @description Successful Response */
+ 200: {
+ content: {
+ "application/json": components["schemas"]["UserObjectstoreUsage"][];
+ };
+ };
+ /** @description Validation Error */
+ 422: {
+ content: {
+ "application/json": components["schemas"]["HTTPValidationError"];
+ };
+ };
+ };
+ };
recalculate_disk_usage_by_user_id_api_users__user_id__recalculate_disk_usage_put: {
/** Triggers a recalculation of the current user disk usage. */
parameters: {