Skip to content

Commit

Permalink
Merge pull request #17500 from jmchilton/storage_management
Browse files Browse the repository at this point in the history 
Enable storage management by object store
  • Loading branch information
@davelopez
davelopez authored Mar 28, 2024
2 parents 68aa360 + 5bbbd04 commit 99aec81
Show file tree
Hide file tree
Showing 21 changed files with 866 additions and 154 deletions.
18 changes: 7 additions & 11 deletions client/src/api/datasets.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,25 +49,21 @@ export async function fetchDatasetDetails(params: { id: string }): Promise<Datas
return data as unknown as DatasetDetails;
}

const updateHistoryDataset = fetcher.path("/api/histories/{history_id}/contents/{type}s/{id}").method("put").create();
const updateDataset = fetcher.path("/api/datasets/{dataset_id}").method("put").create();

export async function undeleteHistoryDataset(historyId: string, datasetId: string) {
const { data } = await updateHistoryDataset({
history_id: historyId,
id: datasetId,
export async function undeleteHistoryDataset(datasetId: string) {
const { data } = await updateDataset({
dataset_id: datasetId,
type: "dataset",
deleted: false,
});
return data;
}

const deleteHistoryDataset = fetcher
.path("/api/histories/{history_id}/contents/{type}s/{id}")
.method("delete")
.create();
const deleteDataset = fetcher.path("/api/datasets/{dataset_id}").method("delete").create();

export async function purgeHistoryDataset(historyId: string, datasetId: string) {
const { data } = await deleteHistoryDataset({ history_id: historyId, id: datasetId, type: "dataset", purge: true });
export async function purgeDataset(datasetId: string) {
const { data } = await deleteDataset({ dataset_id: datasetId, purge: true });
return data;
}

Expand Down
160 changes: 160 additions & 0 deletions client/src/api/schema/schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,18 @@ export interface paths {
* To get more information please check the source code.
*/
get: operations["show_api_datasets__dataset_id__get"];
/**
* Updates the values for the history dataset (HDA) item with the given ``ID``.
* @description Updates the values for the history content item with the given ``ID``.
*/
put: operations["datasets__update_dataset"];
/**
* Delete the history dataset content with the given ``ID``.
* @description Delete the history content with the given ``ID`` and path specified type.
*
* **Note**: Currently does not stop any active jobs for which this dataset is an output.
*/
delete: operations["datasets__delete"];
};
"/api/datasets/{dataset_id}/content/{content_type}": {
/** Retrieve information about the content of a dataset. */
Expand Down Expand Up @@ -1705,6 +1717,10 @@ export interface paths {
/** Remove the object from user's favorites */
delete: operations["remove_favorite_api_users__user_id__favorites__object_type___object_id__delete"];
};
"/api/users/{user_id}/objectstore_usage": {
/** Return the user's object store usage summary broken down by object store ID */
get: operations["get_user_objectstore_usage_api_users__user_id__objectstore_usage_get"];
};
"/api/users/{user_id}/recalculate_disk_usage": {
/** Triggers a recalculation of the current user disk usage. */
put: operations["recalculate_disk_usage_by_user_id_api_users__user_id__recalculate_disk_usage_put"];
Expand Down Expand Up @@ -12478,6 +12494,13 @@ export interface components {
*/
notification_ids: string[];
};
/** UserObjectstoreUsage */
UserObjectstoreUsage: {
/** Object Store Id */
object_store_id: string;
/** Total Disk Usage */
total_disk_usage: number;
};
/** UserQuota */
UserQuota: {
/**
Expand Down Expand Up @@ -13618,6 +13641,116 @@ export interface operations {
};
};
};
datasets__update_dataset: {
/**
* Updates the values for the history dataset (HDA) item with the given ``ID``.
* @description Updates the values for the history content item with the given ``ID``.
*/
parameters: {
/** @description View to be passed to the serializer */
/** @description Comma-separated list of keys to be passed to the serializer */
query?: {
view?: string | null;
keys?: string | null;
};
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
header?: {
"run-as"?: string | null;
};
/** @description The ID of the item (`HDA`/`HDCA`) */
path: {
dataset_id: string;
};
};
requestBody: {
content: {
"application/json": components["schemas"]["UpdateHistoryContentsPayload"];
};
};
responses: {
/** @description Successful Response */
200: {
content: {
"application/json":
| components["schemas"]["HDACustom"]
| components["schemas"]["HDADetailed"]
| components["schemas"]["HDASummary"]
| components["schemas"]["HDCADetailed"]
| components["schemas"]["HDCASummary"];
};
};
/** @description Validation Error */
422: {
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
datasets__delete: {
/**
* Delete the history dataset content with the given ``ID``.
* @description Delete the history content with the given ``ID`` and path specified type.
*
* **Note**: Currently does not stop any active jobs for which this dataset is an output.
*/
parameters: {
/**
* @deprecated
* @description Whether to remove from disk the target HDA or child HDAs of the target HDCA.
*/
/**
* @deprecated
* @description When deleting a dataset collection, whether to also delete containing datasets.
*/
/**
* @deprecated
* @description Whether to stop the creating job if all outputs of the job have been deleted.
*/
/** @description View to be passed to the serializer */
/** @description Comma-separated list of keys to be passed to the serializer */
query?: {
purge?: boolean | null;
recursive?: boolean | null;
stop_job?: boolean | null;
view?: string | null;
keys?: string | null;
};
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
header?: {
"run-as"?: string | null;
};
/** @description The ID of the item (`HDA`/`HDCA`) */
path: {
dataset_id: string;
};
};
requestBody?: {
content: {
"application/json": components["schemas"]["DeleteHistoryContentPayload"];
};
};
responses: {
/** @description Request has been executed. */
200: {
content: {
"application/json": components["schemas"]["DeleteHistoryContentResult"];
};
};
/** @description Request accepted, processing will finish later. */
202: {
content: {
"application/json": components["schemas"]["DeleteHistoryContentResult"];
};
};
/** @description Validation Error */
422: {
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_structured_content_api_datasets__dataset_id__content__content_type__get: {
/** Retrieve information about the content of a dataset. */
parameters: {
Expand Down Expand Up @@ -22594,6 +22727,33 @@ export interface operations {
};
};
};
get_user_objectstore_usage_api_users__user_id__objectstore_usage_get: {
/** Return the user's object store usage summary broken down by object store ID */
parameters: {
/** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
header?: {
"run-as"?: string | null;
};
/** @description The ID of the user to get or 'current'. */
path: {
user_id: string | "current";
};
};
responses: {
/** @description Successful Response */
200: {
content: {
"application/json": components["schemas"]["UserObjectstoreUsage"][];
};
};
/** @description Validation Error */
422: {
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
recalculate_disk_usage_by_user_id_api_users__user_id__recalculate_disk_usage_put: {
/** Triggers a recalculation of the current user disk usage. */
parameters: {
Expand Down
1 change: 1 addition & 0 deletions client/src/api/users.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { fetcher } from "@/api/schema";
export const createApiKey = fetcher.path("/api/users/{user_id}/api_key").method("post").create();
export const deleteUser = fetcher.path("/api/users/{user_id}").method("delete").create();
export const fetchQuotaUsages = fetcher.path("/api/users/{user_id}/usage").method("get").create();
export const fetchObjectStoreUsages = fetcher.path("/api/users/{user_id}/objectstore_usage").method("get").create();
export const recalculateDiskUsage = fetcher.path("/api/users/current/recalculate_disk_usage").method("put").create();
export const recalculateDiskUsageByUserId = fetcher
.path("/api/users/{user_id}/recalculate_disk_usage")
Expand Down
35 changes: 28 additions & 7 deletions client/src/components/User/DiskUsage/StorageDashboard.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,22 @@ const texts = reactive({
icon: "fas fa-broom fa-6x",
buttonText: localize("Free up disk usage"),
},
explore: {
title: localize("Visually explore your disk usage"),
explore_by_history: {
title: localize("Visually explore your disk usage by history"),
description: localize(
"Want to know what histories or datasets take up the most space in your account? Here you can explore your disk usage in a visual way."
"Want to know what histories or datasets take up the most space in your account? Here you can explore your disk usage in a visual way by history."
),
icon: "fas fa-chart-pie fa-6x",
buttonText: localize("Explore now"),
},
explore_by_objectstore: {
title: localize("Visually explore your disk usage by object store"),
description: localize(
"Want to know how the space in your account is being distributed by object store? Here you can explore your disk usage in a visual way by object store."
),
icon: "fas fa-hdd fa-6x",
buttonText: localize("Explore now"),
},
});
function goToStorageManager() {
Expand All @@ -37,6 +45,10 @@ function goToStorageManager() {
function goToHistoriesOverview() {
router.push({ name: "HistoriesOverview" });
}
function goToObjectStoresOverview() {
router.push({ name: "ObjectStoresOverview" });
}
</script>

<template>
Expand All @@ -60,10 +72,19 @@ function goToHistoriesOverview() {
<IconCard
class="mx-auto mb-3"
data-description="explore usage card"
:title="texts.explore.title"
:description="texts.explore.description"
:icon="texts.explore.icon"
:button-text="texts.explore.buttonText"
:title="texts.explore_by_history.title"
:description="texts.explore_by_history.description"
:icon="texts.explore_by_history.icon"
:button-text="texts.explore_by_history.buttonText"
@onButtonClick="goToHistoriesOverview" />

<IconCard
class="mx-auto mb-3"
data-description="explore object store usage card"
:title="texts.explore_by_objectstore.title"
:description="texts.explore_by_objectstore.description"
:icon="texts.explore_by_objectstore.icon"
:button-text="texts.explore_by_objectstore.buttonText"
@onButtonClick="goToObjectStoresOverview" />
</div>
</template>
34 changes: 12 additions & 22 deletions client/src/components/User/DiskUsage/Visualizations/HistoriesStorageOverview.vue
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<script setup lang="ts">
import { onMounted, ref } from "vue";
import { ref } from "vue";
import { useRouter } from "vue-router/composables";
import { useConfirmDialog } from "@/composables/confirmDialog";
Expand All @@ -8,13 +8,14 @@ import { useHistoryStore } from "@/stores/historyStore";
import localize from "@/utils/localization";
import type { DataValuePoint } from "./Charts";
import { bytesLabelFormatter, bytesValueFormatter } from "./Charts/formatters";
import { fetchAllHistoriesSizeSummary, type ItemSizeSummary, purgeHistoryById, undeleteHistoryById } from "./service";
import { byteFormattingForChart, useDataLoading } from "./util";
import BarChart from "./Charts/BarChart.vue";
import OverviewPage from "./OverviewPage.vue";
import RecoverableItemSizeTooltip from "./RecoverableItemSizeTooltip.vue";
import SelectedItemActions from "./SelectedItemActions.vue";
import Heading from "@/components/Common/Heading.vue";
import WarnDeletedHistories from "./WarnDeletedHistories.vue";
import LoadingSpan from "@/components/LoadingSpan.vue";
const historyStore = useHistoryStore();
Expand All @@ -25,17 +26,16 @@ const { confirm } = useConfirmDialog();
const historiesSizeSummaryMap = new Map<string, ItemSizeSummary>();
const topTenHistoriesBySizeData = ref<DataValuePoint[] | null>(null);
const activeVsArchivedVsDeletedTotalSizeData = ref<DataValuePoint[] | null>(null);
const isLoading = ref(true);
const numberOfHistoriesToDisplayOptions = [10, 20, 50];
const numberOfHistoriesToDisplay = ref(numberOfHistoriesToDisplayOptions[0]);
onMounted(async () => {
isLoading.value = true;
const { isLoading, loadDataOnMount } = useDataLoading();
loadDataOnMount(async () => {
const allHistoriesSizeSummary = await fetchAllHistoriesSizeSummary();
allHistoriesSizeSummary.forEach((history) => historiesSizeSummaryMap.set(history.id, history));
buildGraphsData();
isLoading.value = false;
});
function buildGraphsData() {
Expand Down Expand Up @@ -151,19 +151,11 @@ async function onPermanentlyDeleteHistory(historyId: string) {
}
</script>
<template>
<div class="mx-3">
<router-link :to="{ name: 'StorageDashboard' }">{{ localize("Back to Dashboard") }}</router-link>
<Heading h1 bold class="my-3"> Histories Storage Overview </Heading>
<OverviewPage title="Histories Storage Overview">
<p class="text-justify">
Here you can find various graphs displaying the storage size taken by <b>all your histories</b>.
</p>
<p class="text-justify">
Note: these graphs include <b>deleted histories</b>. Remember that, even if you delete histories, they still
take up storage space. However, you can free up the storage space by permanently deleting them from the
<i>Discarded Items</i> section of the
<router-link :to="{ name: 'StorageManager' }"><b>Storage Manager</b></router-link> page or by selecting them
individually in the graph and clicking the <b>Permanently Delete</b> button.
</p>
<WarnDeletedHistories />

<div v-if="isLoading" class="text-center">
<LoadingSpan class="mt-5" :message="localize('Loading your storage data. This may take a while...')" />
Expand All @@ -178,8 +170,7 @@ async function onPermanentlyDeleteHistory(historyId: string) {
"
:data="topTenHistoriesBySizeData"
:enable-selection="true"
:label-formatter="bytesLabelFormatter"
:value-formatter="bytesValueFormatter">
v-bind="byteFormattingForChart">
<template v-slot:title>
<b>{{ localize(`Top ${numberOfHistoriesToDisplay} Histories by Size`) }}</b>
<b-form-select
Expand Down Expand Up @@ -220,8 +211,7 @@ async function onPermanentlyDeleteHistory(historyId: string) {
)
"
:data="activeVsArchivedVsDeletedTotalSizeData"
:label-formatter="bytesLabelFormatter"
:value-formatter="bytesValueFormatter">
v-bind="byteFormattingForChart">
<template v-slot:tooltip="{ data }">
<RecoverableItemSizeTooltip
v-if="data"
Expand All @@ -230,5 +220,5 @@ async function onPermanentlyDeleteHistory(historyId: string) {
</template>
</BarChart>
</div>
</div>
</OverviewPage>
</template>
Loading

0 comments on commit 99aec81

Please sign in to comment.