diff --git a/client/src/components/Dataset/DatasetStorage/DatasetStorage.test.js b/client/src/components/Dataset/DatasetStorage/DatasetStorage.test.js
index 821126543925..bfdedc98bda0 100644
--- a/client/src/components/Dataset/DatasetStorage/DatasetStorage.test.js
+++ b/client/src/components/Dataset/DatasetStorage/DatasetStorage.test.js
@@ -4,37 +4,18 @@ import { getLocalVue } from "tests/jest/helpers";
import flushPromises from "flush-promises";
import MockAdapter from "axios-mock-adapter";
import axios from "axios";
-import MarkdownIt from "markdown-it";
const localVue = getLocalVue();
const TEST_STORAGE_API_RESPONSE_WITHOUT_ID = {
object_store_id: null,
-};
-const TEST_STORAGE_API_RESPONSE_WITH_ID = {
- object_store_id: "foobar",
-};
-const TEST_STORAGE_API_RESPONSE_WITH_NAME = {
- object_store_id: "foobar",
- name: "my cool storage",
- description: "My cool **markdown**",
+ private: false,
};
const TEST_DATASET_ID = "1";
const TEST_STORAGE_URL = `/api/datasets/${TEST_DATASET_ID}/storage`;
-const TEST_RENDERED_MARKDOWN_AS_HTML = "My cool markdown\n";
const TEST_ERROR_MESSAGE = "Opps all errors.";
-// works fine without mocking but I guess it is more JS unit-y with the mock?
-jest.mock("markdown-it");
-MarkdownIt.mockImplementation(() => {
- return {
- render(markdown) {
- return TEST_RENDERED_MARKDOWN_AS_HTML;
- },
- };
-});
-
-describe("Dataset Storage", () => {
+describe("DatasetStorage.vue", () => {
let axiosMock;
let wrapper;
@@ -46,9 +27,6 @@ describe("Dataset Storage", () => {
wrapper = shallowMount(DatasetStorage, {
propsData: { datasetId: TEST_DATASET_ID },
localVue,
- stubs: {
- "loading-span": true,
- },
});
}
@@ -62,6 +40,7 @@ describe("Dataset Storage", () => {
mount();
await wrapper.vm.$nextTick();
expect(wrapper.findAll("loading-span-stub").length).toBe(1);
+ expect(wrapper.findAll("describe-object-store-stub").length).toBe(0);
});
it("test error rendering...", async () => {
@@ -78,44 +57,8 @@ describe("Dataset Storage", () => {
it("test dataset storage with object store without id", async () => {
await mountWithResponse(TEST_STORAGE_API_RESPONSE_WITHOUT_ID);
expect(wrapper.findAll("loading-span-stub").length).toBe(0);
- expect(wrapper.vm.descriptionRendered).toBeNull();
- const header = wrapper.findAll("h2");
- expect(header.length).toBe(1);
- expect(header.at(0).text()).toBe("Dataset Storage");
- const byIdSpan = wrapper.findAll(".display-os-by-id");
- expect(byIdSpan.length).toBe(0);
- const byNameSpan = wrapper.findAll(".display-os-by-name");
- expect(byNameSpan.length).toBe(0);
- const byDefaultSpan = wrapper.findAll(".display-os-default");
- expect(byDefaultSpan.length).toBe(1);
- });
-
- it("test dataset storage with object store id", async () => {
- await mountWithResponse(TEST_STORAGE_API_RESPONSE_WITH_ID);
- expect(wrapper.findAll("loading-span-stub").length).toBe(0);
- expect(wrapper.vm.storageInfo.object_store_id).toBe("foobar");
- expect(wrapper.vm.descriptionRendered).toBeNull();
- const header = wrapper.findAll("h2");
- expect(header.length).toBe(1);
- expect(header.at(0).text()).toBe("Dataset Storage");
- const byIdSpan = wrapper.findAll(".display-os-by-id");
- expect(byIdSpan.length).toBe(1);
- const byNameSpan = wrapper.findAll(".display-os-by-name");
- expect(byNameSpan.length).toBe(0);
- });
-
- it("test dataset storage with object store name", async () => {
- await mountWithResponse(TEST_STORAGE_API_RESPONSE_WITH_NAME);
- expect(wrapper.findAll("loading-span-stub").length).toBe(0);
- expect(wrapper.vm.storageInfo.object_store_id).toBe("foobar");
- expect(wrapper.vm.descriptionRendered).toBe(TEST_RENDERED_MARKDOWN_AS_HTML);
- const header = wrapper.findAll("h2");
- expect(header.length).toBe(1);
- expect(header.at(0).text()).toBe("Dataset Storage");
- const byIdSpan = wrapper.findAll(".display-os-by-id");
- expect(byIdSpan.length).toBe(0);
- const byNameSpan = wrapper.findAll(".display-os-by-name");
- expect(byNameSpan.length).toBe(1);
+ expect(wrapper.findAll("describe-object-store-stub").length).toBe(1);
+ expect(wrapper.vm.storageInfo.private).toEqual(false);
});
afterEach(() => {
diff --git a/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue b/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
index 8edb5fc3967c..6c35e101caa0 100644
--- a/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
+++ b/client/src/components/Dataset/DatasetStorage/DatasetStorage.vue
@@ -16,17 +16,7 @@
-
- This dataset is stored in
-
- a Galaxy object store named {{ storageInfo.name }}
-
-
- a Galaxy object store with id {{ storageInfo.object_store_id }}
-
- the default configured Galaxy object store .
-
-
+
+ This target object store has been set at the history level.
+
+
+ This target object store has been inherited from your user preferences (set in User -> Preferences ->
+ Preferred Object Store). If that option is updated, this history will target that new default.
+
+
+
+ Change this preference object store target by clicking on the storage button in the history panel.
+
+
+
+
+
diff --git a/client/src/components/History/CurrentHistory/SelectPreferredStore.test.js b/client/src/components/History/CurrentHistory/SelectPreferredStore.test.js
new file mode 100644
index 000000000000..0e9a627293e0
--- /dev/null
+++ b/client/src/components/History/CurrentHistory/SelectPreferredStore.test.js
@@ -0,0 +1,63 @@
+import { mount } from "@vue/test-utils";
+import { getLocalVue } from "tests/jest/helpers";
+import { setupSelectableMock } from "../../ObjectStore/mockServices";
+setupSelectableMock();
+
+import axios from "axios";
+import MockAdapter from "axios-mock-adapter";
+import flushPromises from "flush-promises";
+
+import SelectPreferredStore from "./SelectPreferredStore.vue";
+
+const localVue = getLocalVue(true);
+
+const TEST_HISTORY_ID = "myTestHistoryId";
+
+const TEST_HISTORY = {
+ id: TEST_HISTORY_ID,
+ preferred_object_store_id: null,
+};
+
+function mountComponent() {
+ const wrapper = mount(SelectPreferredStore, {
+ propsData: { userPreferredObjectStoreId: null, history: TEST_HISTORY },
+ localVue,
+ });
+ return wrapper;
+}
+
+import { ROOT_COMPONENT } from "@/utils/navigation";
+
+const PREFERENCES = ROOT_COMPONENT.preferences;
+
+describe("SelectPreferredStore.vue", () => {
+ let axiosMock;
+
+ beforeEach(async () => {
+ axiosMock = new MockAdapter(axios);
+ });
+
+ afterEach(async () => {
+ axiosMock.restore();
+ });
+
+ it("updates object store to default on selection null", async () => {
+ const wrapper = mountComponent();
+ await flushPromises();
+ const els = wrapper.findAll(PREFERENCES.object_store_selection.option_buttons.selector);
+ expect(els.length).toBe(3);
+ const galaxyDefaultOption = wrapper.find(
+ PREFERENCES.object_store_selection.option_button({ object_store_id: "__null__" }).selector
+ );
+ expect(galaxyDefaultOption.exists()).toBeTruthy();
+ axiosMock
+ .onPut(`/api/histories/${TEST_HISTORY_ID}`, expect.objectContaining({ preferred_object_store_id: null }))
+ .reply(202);
+ await galaxyDefaultOption.trigger("click");
+ await flushPromises();
+ const errorEl = wrapper.find(".object-store-selection-error");
+ expect(errorEl.exists()).toBeFalsy();
+ const emitted = wrapper.emitted();
+ expect(emitted["updated"][0][0]).toEqual(null);
+ });
+});
diff --git a/client/src/components/History/CurrentHistory/SelectPreferredStore.vue b/client/src/components/History/CurrentHistory/SelectPreferredStore.vue
new file mode 100644
index 000000000000..0fd3195918f4
--- /dev/null
+++ b/client/src/components/History/CurrentHistory/SelectPreferredStore.vue
@@ -0,0 +1,76 @@
+
+
+
+
+
diff --git a/client/src/components/Libraries/LibraryFolder/TopToolbar/library-model.js b/client/src/components/Libraries/LibraryFolder/TopToolbar/library-model.js
index 3db3e4a2e6ef..c16ff708ab3f 100644
--- a/client/src/components/Libraries/LibraryFolder/TopToolbar/library-model.js
+++ b/client/src/components/Libraries/LibraryFolder/TopToolbar/library-model.js
@@ -172,7 +172,7 @@ var HistoryContents = Backbone.Collection.extend({
this.id = options.id;
},
url: function () {
- return `${this.urlRoot + this.id}/contents`;
+ return `${this.urlRoot + this.id}/contents?shareable=true`;
},
model: HistoryItem,
});
diff --git a/client/src/components/ObjectStore/DescribeObjectStore.test.js b/client/src/components/ObjectStore/DescribeObjectStore.test.js
new file mode 100644
index 000000000000..1c1acff73753
--- /dev/null
+++ b/client/src/components/ObjectStore/DescribeObjectStore.test.js
@@ -0,0 +1,83 @@
+import { shallowMount } from "@vue/test-utils";
+import DescribeObjectStore from "./DescribeObjectStore";
+import { getLocalVue } from "tests/jest/helpers";
+import MarkdownIt from "markdown-it";
+
+const localVue = getLocalVue();
+
+const TEST_STORAGE_API_RESPONSE_WITHOUT_ID = {
+ object_store_id: null,
+ private: false,
+ badges: [],
+};
+const TEST_RENDERED_MARKDOWN_AS_HTML = "My cool markdown\n";
+
+const TEST_STORAGE_API_RESPONSE_WITH_ID = {
+ object_store_id: "foobar",
+ private: false,
+ badges: [],
+};
+const TEST_STORAGE_API_RESPONSE_WITH_NAME = {
+ object_store_id: "foobar",
+ name: "my cool storage",
+ description: "My cool **markdown**",
+ private: true,
+ badges: [],
+};
+
+// works fine without mocking but I guess it is more JS unit-y with the mock?
+jest.mock("markdown-it");
+MarkdownIt.mockImplementation(() => {
+ return {
+ render(markdown) {
+ return TEST_RENDERED_MARKDOWN_AS_HTML;
+ },
+ };
+});
+
+describe("DescribeObjectStore.vue", () => {
+ let wrapper;
+
+ async function mountWithResponse(response) {
+ wrapper = shallowMount(DescribeObjectStore, {
+ propsData: { storageInfo: response, what: "where i am throwing my test dataset" },
+ localVue,
+ });
+ }
+
+ it("test dataset storage with object store without id", async () => {
+ await mountWithResponse(TEST_STORAGE_API_RESPONSE_WITHOUT_ID);
+ expect(wrapper.findAll("loading-span-stub").length).toBe(0);
+ expect(wrapper.vm.descriptionRendered).toBeNull();
+ const byIdSpan = wrapper.findAll(".display-os-by-id");
+ expect(byIdSpan.length).toBe(0);
+ const byNameSpan = wrapper.findAll(".display-os-by-name");
+ expect(byNameSpan.length).toBe(0);
+ const byDefaultSpan = wrapper.findAll(".display-os-default");
+ expect(byDefaultSpan.length).toBe(1);
+ });
+
+ it("test dataset storage with object store id", async () => {
+ await mountWithResponse(TEST_STORAGE_API_RESPONSE_WITH_ID);
+ expect(wrapper.findAll("loading-span-stub").length).toBe(0);
+ expect(wrapper.vm.storageInfo.object_store_id).toBe("foobar");
+ expect(wrapper.vm.descriptionRendered).toBeNull();
+ const byIdSpan = wrapper.findAll(".display-os-by-id");
+ expect(byIdSpan.length).toBe(1);
+ const byNameSpan = wrapper.findAll(".display-os-by-name");
+ expect(byNameSpan.length).toBe(0);
+ expect(wrapper.find("object-store-restriction-span-stub").props("isPrivate")).toBeFalsy();
+ });
+
+ it("test dataset storage with object store name", async () => {
+ await mountWithResponse(TEST_STORAGE_API_RESPONSE_WITH_NAME);
+ expect(wrapper.findAll("loading-span-stub").length).toBe(0);
+ expect(wrapper.vm.storageInfo.object_store_id).toBe("foobar");
+ expect(wrapper.vm.descriptionRendered).toBe(TEST_RENDERED_MARKDOWN_AS_HTML);
+ const byIdSpan = wrapper.findAll(".display-os-by-id");
+ expect(byIdSpan.length).toBe(0);
+ const byNameSpan = wrapper.findAll(".display-os-by-name");
+ expect(byNameSpan.length).toBe(1);
+ expect(wrapper.find("object-store-restriction-span-stub").props("isPrivate")).toBeTruthy();
+ });
+});
diff --git a/client/src/components/ObjectStore/DescribeObjectStore.vue b/client/src/components/ObjectStore/DescribeObjectStore.vue
new file mode 100644
index 000000000000..475f583957e6
--- /dev/null
+++ b/client/src/components/ObjectStore/DescribeObjectStore.vue
@@ -0,0 +1,70 @@
+
+
+
+ {{ what }}
+
+ a Galaxy object store named
+ {{ storageInfo.name }}
+
+
+ a Galaxy object store with id
+ {{ storageInfo.object_store_id }}
+
+
+ the default configured Galaxy object store .
+
+
+
+
+
+
+
Galaxy has no quota configured for this object store.
+
+
{{ stockMessage }}
+
+
+
+
+
+
+
+
diff --git a/client/src/components/ObjectStore/ObjectStoreBadges.vue b/client/src/components/ObjectStore/ObjectStoreBadges.vue
new file mode 100644
index 000000000000..1d6d8e21a149
--- /dev/null
+++ b/client/src/components/ObjectStore/ObjectStoreBadges.vue
@@ -0,0 +1,35 @@
+
+
+
+
+
+
+
+
diff --git a/client/src/components/ObjectStore/ObjectStoreRestrictionSpan.test.js b/client/src/components/ObjectStore/ObjectStoreRestrictionSpan.test.js
new file mode 100644
index 000000000000..a022b92aa5c4
--- /dev/null
+++ b/client/src/components/ObjectStore/ObjectStoreRestrictionSpan.test.js
@@ -0,0 +1,27 @@
+import { shallowMount } from "@vue/test-utils";
+import { getLocalVue } from "tests/jest/helpers";
+import ObjectStoreRestrictionSpan from "./ObjectStoreRestrictionSpan";
+
+const localVue = getLocalVue();
+
+describe("ObjectStoreRestrictionSpan", () => {
+ let wrapper;
+
+ it("should render info about private storage if isPrivate", () => {
+ wrapper = shallowMount(ObjectStoreRestrictionSpan, {
+ propsData: { isPrivate: true },
+ localVue,
+ });
+ expect(wrapper.find(".stored-how").text()).toBe("private");
+ expect(wrapper.find(".stored-how").attributes("title")).toBeTruthy();
+ });
+
+ it("should render info about unrestricted storage if not isPrivate", () => {
+ wrapper = shallowMount(ObjectStoreRestrictionSpan, {
+ propsData: { isPrivate: false },
+ localVue,
+ });
+ expect(wrapper.find(".stored-how").text()).toBe("unrestricted");
+ expect(wrapper.find(".stored-how").attributes("title")).toBeTruthy();
+ });
+});
diff --git a/client/src/components/ObjectStore/ObjectStoreRestrictionSpan.vue b/client/src/components/ObjectStore/ObjectStoreRestrictionSpan.vue
new file mode 100644
index 000000000000..5f1bb689db60
--- /dev/null
+++ b/client/src/components/ObjectStore/ObjectStoreRestrictionSpan.vue
@@ -0,0 +1,38 @@
+
+ {{ text }}
+
+
+
+
+
diff --git a/client/src/components/ObjectStore/SelectObjectStore.vue b/client/src/components/ObjectStore/SelectObjectStore.vue
new file mode 100644
index 000000000000..0ce3cb152923
--- /dev/null
+++ b/client/src/components/ObjectStore/SelectObjectStore.vue
@@ -0,0 +1,143 @@
+
+
+
+
+
+
+
+ {{ error }}
+
+
+
+
+ {{ defaultOptionTitle | localize }}
+ {{ object_store.name }}
+
+
+
+
+
+
+
+
+ {{ whyIsSelectionPreferredText }}
+
+
+
+
+ {{ defaultOptionTitle }}
+ {{ defaultOptionDescription }}
+
+
+ {{ object_store.name }}
+
+
+
+
@@ -179,6 +182,7 @@ export default {
validationInternal: null,
validationScrollTo: null,
currentVersion: this.version,
+ preferredObjectStoreId: null,
};
},
computed: {
@@ -288,6 +292,9 @@ export default {
this.showLoading = false;
});
},
+ onUpdatePreferredObjectStoreId(preferredObjectStoreId) {
+ this.preferredObjectStoreId = preferredObjectStoreId;
+ },
onExecute(config, historyId) {
if (this.validationInternal) {
this.validationScrollTo = this.validationInternal.slice();
@@ -311,6 +318,9 @@ export default {
if (this.useCachedJobs) {
jobDef.inputs["use_cached_job"] = true;
}
+ if (this.preferredObjectStoreId) {
+ jobDef.preferred_object_store_id = this.preferredObjectStoreId;
+ }
console.debug("toolForm::onExecute()", jobDef);
submitJob(jobDef).then(
(jobResponse) => {
diff --git a/client/src/components/Tool/ToolSelectPreferredObjectStore.vue b/client/src/components/Tool/ToolSelectPreferredObjectStore.vue
new file mode 100644
index 000000000000..8b055f9810aa
--- /dev/null
+++ b/client/src/components/Tool/ToolSelectPreferredObjectStore.vue
@@ -0,0 +1,48 @@
+
+
+
+
diff --git a/client/src/components/Tool/ToolTargetPreferredObjectStorePopover.vue b/client/src/components/Tool/ToolTargetPreferredObjectStorePopover.vue
new file mode 100644
index 000000000000..a284d85d46d0
--- /dev/null
+++ b/client/src/components/Tool/ToolTargetPreferredObjectStorePopover.vue
@@ -0,0 +1,35 @@
+
+
+ {{ title }}
+
+ This target object store has been set at the tool level, by default history or user preferences will be used
+ and if those are not set Galaxy will pick an adminstrator configured default.
+
+
+
+
+ No selection has been made for this tool execution. Defaults from history, user, or Galaxy will be used.
+
+
+ Change this preference object store target by clicking on the storage button in the tool header.
+
+
+
+
+
diff --git a/client/src/components/User/DiskUsage/Quota/ProvidedQuotaSourceUsageBar.vue b/client/src/components/User/DiskUsage/Quota/ProvidedQuotaSourceUsageBar.vue
new file mode 100644
index 000000000000..7a9f6161c036
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Quota/ProvidedQuotaSourceUsageBar.vue
@@ -0,0 +1,42 @@
+
+
+
+
+
+
+
+
diff --git a/client/src/components/User/DiskUsage/Quota/QuotaUsageBar.vue b/client/src/components/User/DiskUsage/Quota/QuotaUsageBar.vue
index 8b584ec3d8ac..ce3dcca4dc81 100644
--- a/client/src/components/User/DiskUsage/Quota/QuotaUsageBar.vue
+++ b/client/src/components/User/DiskUsage/Quota/QuotaUsageBar.vue
@@ -5,9 +5,14 @@ import { DEFAULT_QUOTA_SOURCE_LABEL, QuotaUsage } from "./model/QuotaUsage";
interface QuotaUsageBarProps {
quotaUsage: QuotaUsage;
+ embedded?: boolean;
+ compact?: boolean;
}
-const props = defineProps
();
+const props = withDefaults(defineProps(), {
+ embedded: false,
+ compact: false,
+});
const storageSourceText = ref(localize("storage source"));
const percentOfDiskQuotaUsedText = ref(localize("% of disk quota used"));
@@ -33,6 +38,14 @@ const progressVariant = computed(() => {
return "danger";
});
+const sourceTag = computed(() => {
+ return props.embedded ? "div" : "h2";
+});
+
+const usageTag = computed(() => {
+ return props.embedded ? "div" : "h3";
+});
+
defineExpose({
isDefaultQuota,
quotaHasLimit,
@@ -40,20 +53,24 @@ defineExpose({
-
-
+
+
{{ quotaUsage.sourceLabel }}
{{ storageSourceText }}
-
-
+
+
{{ quotaUsage.niceTotalDiskUsage }}
of {{ quotaUsage.niceQuota }} used
-
-
+
+
{{ quotaUsage.quotaPercent }}{{ percentOfDiskQuotaUsedText }}
-
+
diff --git a/client/src/components/User/DiskUsage/Quota/QuotaUsageProvider.js b/client/src/components/User/DiskUsage/Quota/QuotaUsageProvider.js
new file mode 100644
index 000000000000..5679fa3b7c17
--- /dev/null
+++ b/client/src/components/User/DiskUsage/Quota/QuotaUsageProvider.js
@@ -0,0 +1,25 @@
+import axios from "axios";
+import { SingleQueryProvider } from "components/providers/SingleQueryProvider";
+import { getAppRoot } from "onload/loadConfig";
+import { rethrowSimple } from "utils/simple-error";
+import { QuotaUsage } from "./model";
+
+/**
+ * Fetches the disk usage corresponding to one quota source label -
+ * or the default quota sources if the supplied label is null.
+ * @returns {}
+ */
+async function fetchQuotaSourceUsage({ quotaSourceLabel = null }) {
+ if (quotaSourceLabel == null) {
+ quotaSourceLabel = "__null__";
+ }
+ const url = `${getAppRoot()}api/users/current/usage/${quotaSourceLabel}`;
+ try {
+ const { data } = await axios.get(url);
+ return new QuotaUsage(data);
+ } catch (e) {
+ rethrowSimple(e);
+ }
+}
+
+export const QuotaSourceUsageProvider = SingleQueryProvider(fetchQuotaSourceUsage);
diff --git a/client/src/components/User/UserPreferences.vue b/client/src/components/User/UserPreferences.vue
index 09afa1e0532f..64b5855cca16 100644
--- a/client/src/components/User/UserPreferences.vue
+++ b/client/src/components/User/UserPreferences.vue
@@ -87,6 +87,15 @@
>>
+
+
+
+
+
+
{
+ let axiosMock;
+
+ beforeEach(async () => {
+ axiosMock = new MockAdapter(axios);
+ });
+
+ afterEach(async () => {
+ axiosMock.restore();
+ });
+
+ it("contains a localized link", async () => {
+ const wrapper = mountComponent();
+ expect(wrapper.vm.$refs["modal"].isHidden).toBeTruthy();
+ const el = await wrapper.find(ROOT_COMPONENT.preferences.object_store.selector);
+ expect(el.text()).toBeLocalizationOf("Preferred Object Store");
+ await el.trigger("click");
+ expect(wrapper.vm.$refs["modal"].isHidden).toBeFalsy();
+ });
+
+ it("updates object store to default on selection null", async () => {
+ const wrapper = mountComponent();
+ const el = await wrapper.find(ROOT_COMPONENT.preferences.object_store.selector);
+ await el.trigger("click");
+ await flushPromises();
+ const els = wrapper.findAll(ROOT_COMPONENT.preferences.object_store_selection.option_buttons.selector);
+ expect(els.length).toBe(3);
+ const galaxyDefaultOption = wrapper.find(
+ ROOT_COMPONENT.preferences.object_store_selection.option_button({ object_store_id: "__null__" }).selector
+ );
+ expect(galaxyDefaultOption.exists()).toBeTruthy();
+ axiosMock.onPut("/api/users/current", expect.objectContaining({ preferred_object_store_id: null })).reply(202);
+ await galaxyDefaultOption.trigger("click");
+ await flushPromises();
+ const errorEl = wrapper.find(".object-store-selection-error");
+ expect(errorEl.exists()).toBeFalsy();
+ });
+
+ it("updates object store to default on actual selection", async () => {
+ const wrapper = mountComponent();
+ const el = await wrapper.find(ROOT_COMPONENT.preferences.object_store.selector);
+ await el.trigger("click");
+ const objectStore2Option = wrapper.find(
+ ROOT_COMPONENT.preferences.object_store_selection.option_button({ object_store_id: "object_store_2" })
+ .selector
+ );
+ expect(objectStore2Option.exists()).toBeTruthy();
+ axiosMock
+ .onPut("/api/users/current", expect.objectContaining({ preferred_object_store_id: "object_store_2" }))
+ .reply(202);
+ await objectStore2Option.trigger("click");
+ await flushPromises();
+ const errorEl = wrapper.find(".object-store-selection-error");
+ expect(errorEl.exists()).toBeFalsy();
+ });
+
+ it("displayed error is user update fails", async () => {
+ const wrapper = mountComponent();
+ const el = await wrapper.find(ROOT_COMPONENT.preferences.object_store.selector);
+ await el.trigger("click");
+ const galaxyDefaultOption = wrapper.find(
+ ROOT_COMPONENT.preferences.object_store_selection.option_button({ object_store_id: "__null__" }).selector
+ );
+ expect(galaxyDefaultOption.exists()).toBeTruthy();
+ axiosMock
+ .onPut("/api/users/current", expect.objectContaining({ preferred_object_store_id: null }))
+ .reply(400, { err_msg: "problem with selection.." });
+ await galaxyDefaultOption.trigger("click");
+ await flushPromises();
+ const errorEl = await wrapper.find(".object-store-selection-error");
+ expect(errorEl.exists()).toBeTruthy();
+ expect(wrapper.vm.error).toBe("problem with selection..");
+ });
+});
diff --git a/client/src/components/User/UserPreferredObjectStore.vue b/client/src/components/User/UserPreferredObjectStore.vue
new file mode 100644
index 000000000000..16efca8ddb2c
--- /dev/null
+++ b/client/src/components/User/UserPreferredObjectStore.vue
@@ -0,0 +1,97 @@
+
+
+
+
+
Preferred Object Store
+
+ Select a preferred default object store for the outputs of new jobs to be created in.
+
+
+
+
+
-
-
Workflow: {{ model.name }}
-
-
-
-
-
-
- Send results to a new history
- Attempt to re-use jobs with identical parameters?
-
-
+
+
+
+
+ Workflow: {{ model.name }}
+
+
+
+
+
+
+ Send results to a new history
+ Attempt to re-use jobs with identical parameters?
+ Send outputs and intermediate to different object stores?
+
+
+
+
+
+
+
+
Expand to full workflow form.
-
+
+
diff --git a/client/src/components/Workflow/Run/WorkflowStorageConfiguration.test.js b/client/src/components/Workflow/Run/WorkflowStorageConfiguration.test.js
new file mode 100644
index 000000000000..4c95222add56
--- /dev/null
+++ b/client/src/components/Workflow/Run/WorkflowStorageConfiguration.test.js
@@ -0,0 +1,64 @@
+import WorkflowStorageConfiguration from "./WorkflowStorageConfiguration";
+import { mount } from "@vue/test-utils";
+import { getLocalVue, findViaNavigation } from "tests/jest/helpers";
+import { ROOT_COMPONENT } from "utils/navigation";
+
+const localVue = getLocalVue(true);
+
+describe("WorkflowStorageConfiguration.vue", () => {
+ let wrapper;
+
+ async function doMount(split) {
+ const propsData = {
+ splitObjectStore: split,
+ invocationPreferredObjectStoreId: null,
+ invocationPreferredIntermediateObjectStoreId: null,
+ };
+ wrapper = mount(WorkflowStorageConfiguration, {
+ propsData,
+ localVue,
+ });
+ }
+
+ describe("rendering buttons", () => {
+ it("should show two buttons on splitObjectStore", async () => {
+ doMount(true);
+ const primaryEl = findViaNavigation(wrapper, ROOT_COMPONENT.workflow_run.primary_storage_indciator);
+ expect(primaryEl.exists()).toBeTruthy();
+ const intermediateEl = findViaNavigation(
+ wrapper,
+ ROOT_COMPONENT.workflow_run.intermediate_storage_indciator
+ );
+ expect(intermediateEl.exists()).toBeTruthy();
+ });
+
+ it("should show one button on not splitObjectStore", async () => {
+ doMount(false);
+ const primaryEl = findViaNavigation(wrapper, ROOT_COMPONENT.workflow_run.primary_storage_indciator);
+ expect(primaryEl.exists()).toBeTruthy();
+ const intermediateEl = findViaNavigation(
+ wrapper,
+ ROOT_COMPONENT.workflow_run.intermediate_storage_indciator
+ );
+ expect(intermediateEl.exists()).toBeFalsy();
+ });
+ });
+
+ describe("event handling", () => {
+ it("should fire update events when primary selection is updated", async () => {
+ doMount(true);
+ await wrapper.vm.onUpdate("storage123");
+ const emitted = wrapper.emitted();
+ expect(emitted["updated"][0][0]).toEqual("storage123");
+ expect(emitted["updated"][0][1]).toEqual(false);
+ });
+
+ it("should fire an update event when intermediate selection is updated", async () => {
+ doMount(true);
+ await wrapper.vm.onUpdateIntermediate("storage123");
+ const emitted = wrapper.emitted();
+ expect(emitted["updated"][0][0]).toEqual("storage123");
+ expect(emitted["updated"][0][1]).toEqual(true);
+ });
+ });
+});
diff --git a/client/src/components/Workflow/Run/WorkflowStorageConfiguration.vue b/client/src/components/Workflow/Run/WorkflowStorageConfiguration.vue
new file mode 100644
index 000000000000..3d5a62022754
--- /dev/null
+++ b/client/src/components/Workflow/Run/WorkflowStorageConfiguration.vue
@@ -0,0 +1,116 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/client/src/components/Workflow/Run/WorkflowTargetPreferredObjectStorePopover.vue b/client/src/components/Workflow/Run/WorkflowTargetPreferredObjectStorePopover.vue
new file mode 100644
index 000000000000..197e2968a0ab
--- /dev/null
+++ b/client/src/components/Workflow/Run/WorkflowTargetPreferredObjectStorePopover.vue
@@ -0,0 +1,36 @@
+
+
+ {{ title }}
+ This target object store has been set at the invocation level.
+
+
+
+ No selection has been made for this worklfow invocation. Defaults from history, user, or Galaxy will be
+ used.
+
+
+ Change this preference object store target by clicking on the storage button in the worklfow run header.
+
+
+
+
+
diff --git a/client/src/components/plugins/icons.js b/client/src/components/plugins/icons.js
index b6b6c7f6661b..bc066dc8293b 100644
--- a/client/src/components/plugins/icons.js
+++ b/client/src/components/plugins/icons.js
@@ -21,6 +21,7 @@ import {
faFileExport,
faFilter,
faFolder,
+ faHdd,
faInfoCircle,
faList,
faLink,
@@ -62,6 +63,7 @@ library.add(
faFileExport,
faFilter,
faFolder,
+ faHdd,
faInfoCircle,
faLink,
faList,
diff --git a/client/src/components/providers/ObjectStoreProvider.js b/client/src/components/providers/ObjectStoreProvider.js
new file mode 100644
index 000000000000..0cfc2dafb381
--- /dev/null
+++ b/client/src/components/providers/ObjectStoreProvider.js
@@ -0,0 +1,16 @@
+import axios from "axios";
+import { getAppRoot } from "onload/loadConfig";
+import { SingleQueryProvider } from "components/providers/SingleQueryProvider";
+import { rethrowSimple } from "utils/simple-error";
+
+async function objectStoreDetails({ id }) {
+ const url = `${getAppRoot()}api/object_store/${id}`;
+ try {
+ const { data } = await axios.get(url);
+ return data;
+ } catch (e) {
+ rethrowSimple(e);
+ }
+}
+
+export const ObjectStoreDetailsProvider = SingleQueryProvider(objectStoreDetails);
diff --git a/client/src/schema/schema.ts b/client/src/schema/schema.ts
index ed8ed9a13730..1f37909e345b 100644
--- a/client/src/schema/schema.ts
+++ b/client/src/schema/schema.ts
@@ -889,6 +889,14 @@ export interface paths {
*/
post: operations["create_api_metrics_post"];
};
+ "/api/object_store": {
+ /** Get a list of (currently only concrete) object stores configured with this Galaxy instance. */
+ get: operations["index_api_object_store_get"];
+ };
+ "/api/object_store/{object_store_id}": {
+ /** Get information about a concrete object store configured with Galaxy. */
+ get: operations["show_info_api_object_store__object_store_id__get"];
+ };
"/api/pages": {
/**
* Lists all Pages viewable by the user.
@@ -1181,6 +1189,14 @@ export interface paths {
*/
post: operations["set_beacon_api_users__user_id__beacon_post"];
};
+ "/api/users/{user_id}/usage": {
+ /** Return the user's quota usage summary broken down by quota source */
+ get: operations["get_user_usage_api_users__user_id__usage_get"];
+ };
+ "/api/users/{user_id}/usage/{label}": {
+ /** Return the user's quota usage summary for a given quota source label */
+ get: operations["get_user_usage_for_label_api_users__user_id__usage__label__get"];
+ };
"/api/version": {
/**
* Return Galaxy version information: major/minor version, optional extra info
@@ -1424,6 +1440,34 @@ export interface components {
/** Queue of task being done derived from Celery AsyncResult */
queue?: string;
};
+ /** BadgeDict */
+ BadgeDict: {
+ /** Message */
+ message: string;
+ /**
+ * Source
+ * @enum {string}
+ */
+ source: "admin" | "galaxy";
+ /**
+ * Type
+ * @enum {string}
+ */
+ type:
+ | "faster"
+ | "slower"
+ | "short_term"
+ | "cloud"
+ | "backed_up"
+ | "not_backed_up"
+ | "more_secure"
+ | "less_secure"
+ | "more_stable"
+ | "less_stable"
+ | "quota"
+ | "no_quota"
+ | "restricted";
+ };
/**
* BasicRoleModel
* @description Base model definition with common configuration used by all derived models.
@@ -1710,6 +1754,20 @@ export interface components {
*/
hash_function?: components["schemas"]["HashFunctionNameEnum"];
};
+ /** ConcreteObjectStoreModel */
+ ConcreteObjectStoreModel: {
+ /** Badges */
+ badges: components["schemas"]["BadgeDict"][];
+ /** Description */
+ description?: string;
+ /** Name */
+ name?: string;
+ /** Object Store Id */
+ object_store_id?: string;
+ /** Private */
+ private: boolean;
+ quota: components["schemas"]["QuotaModel"];
+ };
/** ContentsObject */
ContentsObject: {
/**
@@ -2083,6 +2141,11 @@ export interface components {
* @default =
*/
operation?: components["schemas"]["QuotaOperation"];
+ /**
+ * Quota Source Label
+ * @description If set, quota source label to apply this quota operation to. Otherwise, the default quota is used.
+ */
+ quota_source_label?: string;
};
/**
* CreateQuotaResult
@@ -2114,6 +2177,11 @@ export interface components {
* @description The name of the quota. This must be unique within a Galaxy instance.
*/
name: string;
+ /**
+ * Quota Source Label
+ * @description Quota source label
+ */
+ quota_source_label?: string;
/**
* URL
* @deprecated
@@ -2445,6 +2513,11 @@ export interface components {
* @description Base model definition with common configuration used by all derived models.
*/
DatasetStorageDetails: {
+ /**
+ * Badges
+ * @description A mapping of object store labels to badges describing object store properties.
+ */
+ badges: Record
[];
/**
* Dataset State
* @description The model state of the supplied dataset instance.
@@ -2475,6 +2548,16 @@ export interface components {
* @description The percentage indicating how full the store is.
*/
percent_used?: number;
+ /**
+ * Quota
+ * @description Information about quota sources around dataset storage.
+ */
+ quota: Record;
+ /**
+ * Shareable
+ * @description Is this dataset shareable.
+ */
+ shareable: boolean;
/**
* Sources
* @description The file sources associated with the supplied dataset instance.
@@ -6198,6 +6281,11 @@ export interface components {
* @default =
*/
operation?: components["schemas"]["QuotaOperation"];
+ /**
+ * Quota Source Label
+ * @description Quota source label
+ */
+ quota_source_label?: string;
/**
* Users
* @description A list of specific users associated with this quota.
@@ -6205,6 +6293,13 @@ export interface components {
*/
users?: components["schemas"]["UserQuota"][];
};
+ /** QuotaModel */
+ QuotaModel: {
+ /** Enabled */
+ enabled: boolean;
+ /** Source */
+ source?: string;
+ };
/**
* QuotaOperation
* @description An enumeration.
@@ -6236,6 +6331,11 @@ export interface components {
* @description The name of the quota. This must be unique within a Galaxy instance.
*/
name: string;
+ /**
+ * Quota Source Label
+ * @description Quota source label
+ */
+ quota_source_label?: string;
/**
* URL
* @deprecated
@@ -7294,6 +7394,19 @@ export interface components {
*/
user: components["schemas"]["UserModel"];
};
+ /** UserQuotaUsage */
+ UserQuotaUsage: {
+ /** Quota */
+ quota?: string;
+ /** Quota Bytes */
+ quota_bytes?: number;
+ /** Quota Percent */
+ quota_percent?: number;
+ /** Quota Source Label */
+ quota_source_label?: string;
+ /** Total Disk Usage */
+ total_disk_usage: number;
+ };
/** ValidationError */
ValidationError: {
/** Location */
@@ -10039,6 +10152,7 @@ export interface operations {
* @deprecated
* @description Whether to return visible or hidden datasets only. Leave unset for both.
*/
+ /** @description Whether to return only shareable or not shareable datasets. Leave unset for both. */
/** @description View to be passed to the serializer */
/** @description Comma-separated list of keys to be passed to the serializer */
/**
@@ -10062,6 +10176,7 @@ export interface operations {
types?: string[];
deleted?: boolean;
visible?: boolean;
+ shareable?: boolean;
view?: string;
keys?: string;
q?: string[];
@@ -10848,6 +10963,7 @@ export interface operations {
* @deprecated
* @description Whether to return visible or hidden datasets only. Leave unset for both.
*/
+ /** @description Whether to return only shareable or not shareable datasets. Leave unset for both. */
/** @description View to be passed to the serializer */
/** @description Comma-separated list of keys to be passed to the serializer */
/**
@@ -10871,6 +10987,7 @@ export interface operations {
types?: string[];
deleted?: boolean;
visible?: boolean;
+ shareable?: boolean;
view?: string;
keys?: string;
q?: string[];
@@ -12520,6 +12637,60 @@ export interface operations {
};
};
};
+ index_api_object_store_get: {
+ /** Get a list of (currently only concrete) object stores configured with this Galaxy instance. */
+ parameters?: {
+ /** @description Restrict index query to user selectable object stores, the current implementation requires this to be true. */
+ query?: {
+ selectable?: boolean;
+ };
+ /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+ header?: {
+ "run-as"?: string;
+ };
+ };
+ responses: {
+ /** @description A list of the configured object stores. */
+ 200: {
+ content: {
+ "application/json": components["schemas"]["ConcreteObjectStoreModel"][];
+ };
+ };
+ /** @description Validation Error */
+ 422: {
+ content: {
+ "application/json": components["schemas"]["HTTPValidationError"];
+ };
+ };
+ };
+ };
+ show_info_api_object_store__object_store_id__get: {
+ /** Get information about a concrete object store configured with Galaxy. */
+ parameters: {
+ /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+ header?: {
+ "run-as"?: string;
+ };
+ /** @description The concrete object store ID. */
+ path: {
+ object_store_id: string;
+ };
+ };
+ responses: {
+ /** @description Successful Response */
+ 200: {
+ content: {
+ "application/json": components["schemas"]["ConcreteObjectStoreModel"];
+ };
+ };
+ /** @description Validation Error */
+ 422: {
+ content: {
+ "application/json": components["schemas"]["HTTPValidationError"];
+ };
+ };
+ };
+ };
index_api_pages_get: {
/**
* Lists all Pages viewable by the user.
@@ -13999,6 +14170,62 @@ export interface operations {
};
};
};
+ get_user_usage_api_users__user_id__usage_get: {
+ /** Return the user's quota usage summary broken down by quota source */
+ parameters: {
+ /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+ header?: {
+ "run-as"?: string;
+ };
+ /** @description The ID of the user to get or __current__. */
+ path: {
+ user_id: string;
+ };
+ };
+ responses: {
+ /** @description Successful Response */
+ 200: {
+ content: {
+ "application/json": components["schemas"]["UserQuotaUsage"][];
+ };
+ };
+ /** @description Validation Error */
+ 422: {
+ content: {
+ "application/json": components["schemas"]["HTTPValidationError"];
+ };
+ };
+ };
+ };
+ get_user_usage_for_label_api_users__user_id__usage__label__get: {
+ /** Return the user's quota usage summary for a given quota source label */
+ parameters: {
+ /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */
+ header?: {
+ "run-as"?: string;
+ };
+ /** @description The ID of the user to get or __current__. */
+ /** @description The label corresponding to the quota source to fetch usage information about. */
+ path: {
+ user_id: string;
+ label: string;
+ };
+ };
+ responses: {
+ /** @description Successful Response */
+ 200: {
+ content: {
+ "application/json": components["schemas"]["UserQuotaUsage"];
+ };
+ };
+ /** @description Validation Error */
+ 422: {
+ content: {
+ "application/json": components["schemas"]["HTTPValidationError"];
+ };
+ };
+ };
+ };
version_api_version_get: {
/**
* Return Galaxy version information: major/minor version, optional extra info
diff --git a/client/src/utils/navigation/navigation.yml b/client/src/utils/navigation/navigation.yml
index f2d0890bcaf8..3a2fd2db7068 100644
--- a/client/src/utils/navigation/navigation.yml
+++ b/client/src/utils/navigation/navigation.yml
@@ -80,12 +80,18 @@ preferences:
current_email: "#user-preferences-current-email"
get_new_key: '.create-button'
api_key_input: '[data-test-id="api-key-input"]'
+ object_store: '#select-preferred-object-store'
delete_account: '#delete-account'
delete_account_input: '#name-input'
delete_account_ok_btn: '.modal-footer .btn-primary'
email_input: "input[id='email']"
username_input: "input[id='username']"
+ object_store_selection:
+ selectors:
+ option_buttons: '.preferred-object-store-select-button'
+ option_button: '.preferred-object-store-select-button[data-object-store-id="${object_store_id}"]'
+
toolbox_filters:
selectors:
input:
@@ -119,6 +125,10 @@ dataset_details:
transform_action: '[data-transform-action="${action}"]'
deferred_source_uri: '.deferred-dataset-source-uri'
+object_store_details:
+ selectors:
+ badge_of_type: '.object-store-badge-wrapper [data-badge-type="${type}"]'
+
history_panel:
menu:
labels:
@@ -581,6 +591,8 @@ workflow_run:
input_select_field:
type: xpath
selector: '//div[@data-label="${label}"]//span[@class="select2-chosen"]'
+ primary_storage_indciator: '.workflow-storage-indicator-primary'
+ intermediate_storage_indciator: '.workflow-storage-indicator-intermediate'
workflow_editor:
node:
diff --git a/client/tests/jest/helpers.js b/client/tests/jest/helpers.js
index 94612f2f0cc0..eefa5702167e 100644
--- a/client/tests/jest/helpers.js
+++ b/client/tests/jest/helpers.js
@@ -15,6 +15,10 @@ import { PiniaVuePlugin } from "pinia";
const defaultComparator = (a, b) => a == b;
+export function findViaNavigation(wrapper, component) {
+ return wrapper.find(component.selector);
+}
+
function testLocalize(text) {
if (text) {
return `test_localized<${text}>`;
diff --git a/lib/galaxy/job_execution/output_collect.py b/lib/galaxy/job_execution/output_collect.py
index 818c2a03d89c..22399d45073f 100644
--- a/lib/galaxy/job_execution/output_collect.py
+++ b/lib/galaxy/job_execution/output_collect.py
@@ -338,7 +338,7 @@ def add_library_dataset_to_folder(self, library_folder, ld):
trans.app.security_agent.copy_library_permissions(trans, ld, ldda)
# Copy the current user's DefaultUserPermissions to the new LibraryDatasetDatasetAssociation.dataset
trans.app.security_agent.set_all_dataset_permissions(
- ldda.dataset, trans.app.security_agent.user_get_default_permissions(trans.user)
+ ldda.dataset, trans.app.security_agent.user_get_default_permissions(trans.user), flush=False, new=True
)
library_folder.add_library_dataset(ld, genome_build=ldda.dbkey)
trans.sa_session.add(library_folder)
diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py
index 722af5fc6bb5..d376337c543b 100644
--- a/lib/galaxy/jobs/__init__.py
+++ b/lib/galaxy/jobs/__init__.py
@@ -1597,8 +1597,17 @@ def _set_object_store_ids(self, job):
# jobs may have this set. Skip this following code if that is the case.
return
- object_store_populator = ObjectStorePopulator(self.app, job.user)
+ object_store = self.app.object_store
+ if not object_store.object_store_allows_id_selection:
+ self._set_object_store_ids_basic(job)
+ else:
+ self._set_object_store_ids_full(job)
+
+ def _set_object_store_ids_basic(self, job):
object_store_id = self.get_destination_configuration("object_store_id", None)
+ object_store_populator = ObjectStorePopulator(self.app, job.user)
+ require_shareable = job.requires_shareable_storage(self.app.security_agent)
+
if object_store_id:
object_store_populator.object_store_id = object_store_id
@@ -1610,11 +1619,88 @@ def _set_object_store_ids(self, job):
# afterward. State below needs to happen the same way.
for dataset_assoc in job.output_datasets + job.output_library_datasets:
dataset = dataset_assoc.dataset
- object_store_populator.set_object_store_id(dataset)
+ object_store_populator.set_object_store_id(dataset, require_shareable=require_shareable)
job.object_store_id = object_store_populator.object_store_id
self._setup_working_directory(job=job)
+ def _set_object_store_ids_full(self, job):
+ user = job.user
+ object_store_id = self.get_destination_configuration("object_store_id", None)
+ split_object_stores = None
+ object_store_id_overrides = None
+
+ if object_store_id is None:
+ object_store_id = job.preferred_object_store_id
+ if object_store_id is None and job.workflow_invocation_step:
+ workflow_invocation_step = job.workflow_invocation_step
+ invocation_object_stores = workflow_invocation_step.preferred_object_stores
+ if invocation_object_stores.is_split_configuration:
+ # Redo for subworkflows...
+ outputs_object_store_populator = ObjectStorePopulator(self.app, user)
+ preferred_outputs_object_store_id = invocation_object_stores.preferred_outputs_object_store_id
+ outputs_object_store_populator.object_store_id = preferred_outputs_object_store_id
+
+ intermediate_object_store_populator = ObjectStorePopulator(self.app, user)
+ preferred_intermediate_object_store_id = invocation_object_stores.preferred_intermediate_object_store_id
+ intermediate_object_store_populator.object_store_id = preferred_intermediate_object_store_id
+
+ # default for the job... probably isn't used in anyway but for job working
+ # directory?
+ object_store_id = invocation_object_stores.preferred_outputs_object_store_id
+ object_store_populator = intermediate_object_store_populator
+ output_names = [o.output_name for o in workflow_invocation_step.workflow_step.unique_workflow_outputs]
+ if invocation_object_stores.step_effective_outputs is not None:
+ output_names = [
+ o for o in output_names if invocation_object_stores.is_output_name_an_effective_output(o)
+ ]
+
+ # we resolve the precreated datasets here with object store populators
+ # but for dynamically created datasets after the job we need to record
+ # the outputs and set them accordingly
+ object_store_id_overrides = {o: preferred_outputs_object_store_id for o in output_names}
+
+ def split_object_stores(output_name):
+ if "|__part__|" in output_name:
+ output_name = output_name.split("|__part__|", 1)[0]
+ if output_name in output_names:
+ return outputs_object_store_populator
+ else:
+ return intermediate_object_store_populator
+
+ else:
+ object_store_id = invocation_object_stores.preferred_object_store_id
+
+ if object_store_id is None:
+ history = job.history
+ if history is not None:
+ object_store_id = history.preferred_object_store_id
+ if object_store_id is None:
+ if user is not None:
+ object_store_id = user.preferred_object_store_id
+
+ require_shareable = job.requires_shareable_storage(self.app.security_agent)
+ if not split_object_stores:
+ object_store_populator = ObjectStorePopulator(self.app, user)
+
+ if object_store_id:
+ object_store_populator.object_store_id = object_store_id
+
+ for dataset_assoc in job.output_datasets + job.output_library_datasets:
+ dataset = dataset_assoc.dataset
+ object_store_populator.set_object_store_id(dataset, require_shareable=require_shareable)
+
+ job.object_store_id = object_store_populator.object_store_id
+ self._setup_working_directory(job=job)
+ else:
+ for dataset_assoc in job.output_datasets + job.output_library_datasets:
+ dataset = dataset_assoc.dataset
+ dataset_object_store_populator = split_object_stores(dataset_assoc.name)
+ dataset_object_store_populator.set_object_store_id(dataset, require_shareable=require_shareable)
+ job.object_store_id = object_store_populator.object_store_id
+ job.object_store_id_overrides = object_store_id_overrides
+ self._setup_working_directory(job=job)
+
def _finish_dataset(self, output_name, dataset, job, context, final_job_state, remote_metadata_directory):
implicit_collection_jobs = job.implicit_collection_jobs_association
purged = dataset.dataset.purged
@@ -1893,13 +1979,17 @@ def fail(message=job.info, exception=None):
# custom post process setup
collected_bytes = 0
+ quota_source_info = None
# Once datasets are collected, set the total dataset size (includes extra files)
for dataset_assoc in job.output_datasets:
if not dataset_assoc.dataset.dataset.purged:
+ # assume all datasets in a job get written to the same objectstore
+ quota_source_info = dataset_assoc.dataset.dataset.quota_source_info
collected_bytes += dataset_assoc.dataset.set_total_size()
- if job.user:
- job.user.adjust_total_disk_usage(collected_bytes)
+ user = job.user
+ if user and collected_bytes > 0 and quota_source_info is not None and quota_source_info.use:
+ user.adjust_total_disk_usage(collected_bytes, quota_source_info.label)
# Certain tools require tasks to be completed after job execution
# ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ).
diff --git a/lib/galaxy/jobs/runners/__init__.py b/lib/galaxy/jobs/runners/__init__.py
index 2ec20b6ccf4f..20f40a159157 100644
--- a/lib/galaxy/jobs/runners/__init__.py
+++ b/lib/galaxy/jobs/runners/__init__.py
@@ -171,7 +171,15 @@ def run_next(self):
def put(self, job_wrapper: "MinimalJobWrapper"):
"""Add a job to the queue (by job identifier), indicate that the job is ready to run."""
put_timer = ExecutionTimer()
- queue_job = job_wrapper.enqueue()
+ try:
+ queue_job = job_wrapper.enqueue()
+ except Exception as e:
+ queue_job = False
+ # Required for exceptions thrown by object store incompatiblity.
+ # tested by test/integration/objectstore/test_private_handling.py
+ job_wrapper.fail(str(e), exception=e)
+ log.debug(f"Job [{job_wrapper.job_id}] failed to queue {put_timer}")
+ return
if queue_job:
self.mark_as_queued(job_wrapper)
log.debug(f"Job [{job_wrapper.job_id}] queued {put_timer}")
diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py
index a1f8fba0d76f..279eb865199a 100644
--- a/lib/galaxy/managers/configuration.py
+++ b/lib/galaxy/managers/configuration.py
@@ -112,6 +112,7 @@ def _use_config(item, key: str, **context):
def _config_is_truthy(item, key, **context):
return True if item.get(key) else False
+ object_store = self.app.object_store
self.serializers: Dict[str, base.Serializer] = {
# TODO: this is available from user data, remove
"is_admin_user": lambda *a, **c: False,
@@ -195,6 +196,11 @@ def _config_is_truthy(item, key, **context):
"expose_user_email": _use_config,
"enable_tool_source_display": _use_config,
"enable_celery_tasks": _use_config,
+ "quota_source_labels": lambda item, key, **context: list(
+ object_store.get_quota_source_map().get_quota_source_labels()
+ ),
+ "object_store_allows_id_selection": lambda item, key, **context: object_store.object_store_allows_id_selection(),
+ "object_store_ids_allowing_selection": lambda item, key, **context: object_store.object_store_ids_allowing_selection(),
"user_library_import_dir_available": lambda item, key, **context: bool(item.get("user_library_import_dir")),
"welcome_directory": _use_config,
"themes": _use_config,
diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py
index 7abbee7f3b7f..224f3c2afffc 100644
--- a/lib/galaxy/managers/hdas.py
+++ b/lib/galaxy/managers/hdas.py
@@ -215,8 +215,9 @@ def _purge(self, hda, flush=True):
quota_amount_reduction = hda.quota_amount(user)
super().purge(hda, flush=flush)
# decrease the user's space used
- if quota_amount_reduction:
- user.adjust_total_disk_usage(-quota_amount_reduction)
+ quota_source_info = hda.dataset.quota_source_info
+ if quota_amount_reduction and quota_source_info.use:
+ user.adjust_total_disk_usage(-quota_amount_reduction, quota_source_info.label)
# .... states
def error_if_uploading(self, hda):
diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py
index 9191289ec786..9713672577b4 100644
--- a/lib/galaxy/managers/histories.py
+++ b/lib/galaxy/managers/histories.py
@@ -33,6 +33,7 @@
sharable,
)
from galaxy.managers.base import (
+ ModelDeserializingError,
Serializer,
SortableManager,
)
@@ -44,6 +45,7 @@
HDABasicInfo,
ShareHistoryExtra,
)
+from galaxy.security.validate_user_input import validate_preferred_object_store_id
from galaxy.structured_app import MinimalManagerApp
log = logging.getLogger(__name__)
@@ -474,6 +476,7 @@ def __init__(
"annotation",
"tags",
"update_time",
+ "preferred_object_store_id",
],
)
self.add_view(
@@ -496,6 +499,7 @@ def __init__(
"state_details",
"state_ids",
"hid_counter",
+ "preferred_object_store_id",
# 'community_rating',
# 'user_rating',
],
@@ -519,6 +523,7 @@ def __init__(
# 'contents_states',
"contents_active",
"hid_counter",
+ "preferred_object_store_id",
],
include_keys_from="summary",
)
@@ -680,9 +685,17 @@ def add_deserializers(self):
{
"name": self.deserialize_basestring,
"genome_build": self.deserialize_genome_build,
+ "preferred_object_store_id": self.deserialize_preferred_object_store_id,
}
)
+ def deserialize_preferred_object_store_id(self, item, key, val, **context):
+ preferred_object_store_id = val
+ validation_error = validate_preferred_object_store_id(self.app.object_store, preferred_object_store_id)
+ if validation_error:
+ raise ModelDeserializingError(validation_error)
+ return self.default_deserializer(item, key, preferred_object_store_id, **context)
+
class HistoryFilters(sharable.SharableModelFilters, deletable.PurgableFiltersMixin):
model_class = model.History
diff --git a/lib/galaxy/managers/quotas.py b/lib/galaxy/managers/quotas.py
index 886f06489bb4..33013474de6a 100644
--- a/lib/galaxy/managers/quotas.py
+++ b/lib/galaxy/managers/quotas.py
@@ -59,7 +59,11 @@ def create_quota(self, payload: dict, decode_id=None) -> Tuple[model.Quota, str]
raise ActionInputError("Operation for an unlimited quota must be '='.")
# Create the quota
quota = model.Quota(
- name=params.name, description=params.description, amount=create_amount, operation=params.operation
+ name=params.name,
+ description=params.description,
+ amount=create_amount,
+ operation=params.operation,
+ quota_source_label=params.quota_source_label,
)
self.sa_session.add(quota)
# If this is a default quota, create the DefaultQuotaAssociation
diff --git a/lib/galaxy/managers/users.py b/lib/galaxy/managers/users.py
index 0190386099f6..2c58dd7e1052 100644
--- a/lib/galaxy/managers/users.py
+++ b/lib/galaxy/managers/users.py
@@ -7,7 +7,12 @@
import re
import time
from datetime import datetime
-from typing import Optional
+from typing import (
+ Any,
+ Dict,
+ List,
+ Optional,
+)
from markupsafe import escape
from sqlalchemy import (
@@ -29,10 +34,12 @@
base,
deletable,
)
+from galaxy.model import UserQuotaUsage
from galaxy.security.validate_user_input import (
VALID_EMAIL_RE,
validate_email,
validate_password,
+ validate_preferred_object_store_id,
validate_publicname,
)
from galaxy.structured_app import (
@@ -381,13 +388,13 @@ def sharing_roles(self, user):
def default_permissions(self, user):
return self.app.security_agent.user_get_default_permissions(user)
- def quota(self, user, total=False):
+ def quota(self, user, total=False, quota_source_label=None):
if total:
- return self.app.quota_agent.get_quota_nice_size(user)
- return self.app.quota_agent.get_percent(user=user)
+ return self.app.quota_agent.get_quota_nice_size(user, quota_source_label=quota_source_label)
+ return self.app.quota_agent.get_percent(user=user, quota_source_label=quota_source_label)
- def quota_bytes(self, user):
- return self.app.quota_agent.get_quota(user=user)
+ def quota_bytes(self, user, quota_source_label: Optional[str] = None):
+ return self.app.quota_agent.get_quota(user=user, quota_source_label=quota_source_label)
def tags_used(self, user, tag_models=None):
"""
@@ -620,6 +627,7 @@ def __init__(self, app: MinimalManagerApp):
"tags_used",
# all annotations
# 'annotations'
+ "preferred_object_store_id",
],
include_keys_from="summary",
)
@@ -643,6 +651,39 @@ def add_serializers(self):
}
)
+ def serialize_disk_usage(self, user: model.User) -> List[UserQuotaUsage]:
+ usages = user.dictify_usage(self.app.object_store)
+ rval: List[UserQuotaUsage] = []
+ for usage in usages:
+ quota_source_label = usage.quota_source_label
+ quota_percent = self.user_manager.quota(user, quota_source_label=quota_source_label)
+ quota = self.user_manager.quota(user, total=True, quota_source_label=quota_source_label)
+ quota_bytes = self.user_manager.quota_bytes(user, quota_source_label=quota_source_label)
+ rval.append(
+ UserQuotaUsage(
+ quota_source_label=quota_source_label,
+ total_disk_usage=usage.total_disk_usage,
+ quota_percent=quota_percent,
+ quota=quota,
+ quota_bytes=quota_bytes,
+ )
+ )
+ return rval
+
+ def serialize_disk_usage_for(self, user: model.User, label: Optional[str]) -> UserQuotaUsage:
+ usage = user.dictify_usage_for(label)
+ quota_source_label = usage.quota_source_label
+ quota_percent = self.user_manager.quota(user, quota_source_label=quota_source_label)
+ quota = self.user_manager.quota(user, total=True, quota_source_label=quota_source_label)
+ quota_bytes = self.user_manager.quota_bytes(user, quota_source_label=quota_source_label)
+ return UserQuotaUsage(
+ quota_source_label=quota_source_label,
+ total_disk_usage=usage.total_disk_usage,
+ quota_percent=quota_percent,
+ quota=quota,
+ quota_bytes=quota_bytes,
+ )
+
class UserDeserializer(base.ModelDeserializer):
"""
@@ -654,11 +695,18 @@ class UserDeserializer(base.ModelDeserializer):
def add_deserializers(self):
super().add_deserializers()
- self.deserializers.update(
- {
- "username": self.deserialize_username,
- }
- )
+ history_deserializers: Dict[str, base.Deserializer] = {
+ "username": self.deserialize_username,
+ "preferred_object_store_id": self.deserialize_preferred_object_store_id,
+ }
+ self.deserializers.update(history_deserializers)
+
+ def deserialize_preferred_object_store_id(self, item: Any, key: Any, val: Any, **context):
+ preferred_object_store_id = val
+ validation_error = validate_preferred_object_store_id(self.app.object_store, preferred_object_store_id)
+ if validation_error:
+ raise base.ModelDeserializingError(validation_error)
+ return self.default_deserializer(item, key, preferred_object_store_id, **context)
def deserialize_username(self, item, key, username, trans=None, **context):
# TODO: validate_publicname requires trans and should(?) raise exceptions
diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py
index 616ddd755589..c60127d8f28e 100644
--- a/lib/galaxy/model/__init__.py
+++ b/lib/galaxy/model/__init__.py
@@ -27,6 +27,7 @@
List,
NamedTuple,
Optional,
+ Set,
Tuple,
Type,
TYPE_CHECKING,
@@ -39,6 +40,7 @@
import sqlalchemy
from boltons.iterutils import remap
+from pydantic import BaseModel
from social_core.storage import (
AssociationMixin,
CodeMixin,
@@ -51,6 +53,7 @@
and_,
asc,
BigInteger,
+ bindparam,
Boolean,
Column,
DateTime,
@@ -98,7 +101,10 @@
)
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.sql import exists
-from typing_extensions import Protocol
+from typing_extensions import (
+ Protocol,
+ TypedDict,
+)
import galaxy.exceptions
import galaxy.model.metadata
@@ -175,6 +181,7 @@
# Tags that get automatically propagated from inputs to outputs when running jobs.
AUTO_PROPAGATED_TAGS = ["name"]
YIELD_PER_ROWS = 100
+CANNOT_SHARE_PRIVATE_DATASET_MESSAGE = "Attempting to share a non-shareable dataset."
if TYPE_CHECKING:
@@ -525,6 +532,122 @@ def stderr(self, stderr):
raise NotImplementedError("Attempt to set stdout, must set tool_stderr or job_stderr")
+UNIQUE_DATASET_USER_USAGE = """
+WITH per_user_histories AS
+(
+ SELECT id
+ FROM history
+ WHERE user_id = :id
+ AND NOT purged
+),
+per_hist_hdas AS (
+ SELECT DISTINCT dataset_id
+ FROM history_dataset_association
+ WHERE NOT purged
+ AND history_id IN (SELECT id FROM per_user_histories)
+)
+SELECT COALESCE(SUM(COALESCE(dataset.total_size, dataset.file_size, 0)), 0)
+FROM dataset
+LEFT OUTER JOIN library_dataset_dataset_association ON dataset.id = library_dataset_dataset_association.dataset_id
+WHERE dataset.id IN (SELECT dataset_id FROM per_hist_hdas)
+ AND library_dataset_dataset_association.id IS NULL
+ AND (
+ {dataset_condition}
+ )
+"""
+
+
+def calculate_user_disk_usage_statements(user_id, quota_source_map, for_sqlite=False):
+ """Standalone function so can be reused for postgres directly in pgcleanup.py."""
+ statements = []
+ default_quota_enabled = quota_source_map.default_quota_enabled
+ default_exclude_ids = quota_source_map.default_usage_excluded_ids()
+ default_cond = "dataset.object_store_id IS NULL" if default_quota_enabled else ""
+ exclude_cond = "dataset.object_store_id NOT IN :exclude_object_store_ids" if default_exclude_ids else ""
+ use_or = " OR " if (default_cond != "" and exclude_cond != "") else ""
+ default_usage_dataset_condition = "{default_cond} {use_or} {exclude_cond}".format(
+ default_cond=default_cond,
+ exclude_cond=exclude_cond,
+ use_or=use_or,
+ )
+ default_usage = UNIQUE_DATASET_USER_USAGE.format(dataset_condition=default_usage_dataset_condition)
+ default_usage = (
+ """
+UPDATE galaxy_user SET disk_usage = (%s)
+WHERE id = :id
+"""
+ % default_usage
+ )
+ params = {"id": user_id}
+ if default_exclude_ids:
+ params["exclude_object_store_ids"] = default_exclude_ids
+ statements.append((default_usage, params))
+ source = quota_source_map.ids_per_quota_source()
+ # TODO: Merge a lot of these settings together by generating a temp table for
+ # the object_store_id to quota_source_label into a temp table of values
+ for quota_source_label, object_store_ids in source.items():
+ label_usage = UNIQUE_DATASET_USER_USAGE.format(
+ dataset_condition="dataset.object_store_id IN :include_object_store_ids"
+ )
+ if for_sqlite:
+ # hacky alternative for older sqlite
+ statement = """
+WITH new (user_id, quota_source_label, disk_usage) AS (
+ VALUES(:id, :label, ({label_usage}))
+)
+INSERT OR REPLACE INTO user_quota_source_usage (id, user_id, quota_source_label, disk_usage)
+SELECT old.id, new.user_id, new.quota_source_label, new.disk_usage
+FROM new
+ LEFT JOIN user_quota_source_usage AS old
+ ON new.user_id = old.user_id
+ AND new.quota_source_label = old.quota_source_label
+""".format(
+ label_usage=label_usage
+ )
+ else:
+ statement = """
+INSERT INTO user_quota_source_usage(user_id, quota_source_label, disk_usage)
+VALUES(:user_id, :label, ({label_usage}))
+ON CONFLICT
+ON constraint uqsu_unique_label_per_user
+DO UPDATE SET disk_usage = excluded.disk_usage
+""".format(
+ label_usage=label_usage
+ )
+ statements.append(
+ (statement, {"id": user_id, "label": quota_source_label, "include_object_store_ids": object_store_ids})
+ )
+
+ params = {"id": user_id}
+ source_labels = list(source.keys())
+ if len(source_labels) > 0:
+ clean_old_statement = """
+DELETE FROM user_quota_source_usage
+WHERE user_id = :id AND quota_source_label NOT IN :labels
+"""
+ params["labels"] = source_labels
+ else:
+ clean_old_statement = """
+DELETE FROM user_quota_source_usage
+WHERE user_id = :id AND quota_source_label IS NOT NULL
+"""
+ statements.append((clean_old_statement, params))
+ return statements
+
+
+# move these to galaxy.schema.schema once galaxy-data depends on
+# galaxy-schema.
+class UserQuotaBasicUsage(BaseModel):
+ quota_source_label: Optional[str]
+ total_disk_usage: float
+
+
+class UserQuotaUsage(UserQuotaBasicUsage):
+ quota_percent: Optional[float]
+ quota_bytes: Optional[int]
+ quota: Optional[str]
+
+
class User(Base, Dictifiable, RepresentById):
"""
Data for a Galaxy user or admin and relations to their
@@ -546,6 +669,7 @@ class User(Base, Dictifiable, RepresentById):
last_password_change = Column(DateTime, default=now)
external = Column(Boolean, default=False)
form_values_id = Column(Integer, ForeignKey("form_values.id"), index=True)
+ preferred_object_store_id = Column(String(255), nullable=True)
deleted = Column(Boolean, index=True, default=False)
purged = Column(Boolean, index=True, default=False)
disk_usage = Column(Numeric(15, 0), index=True)
@@ -571,6 +695,7 @@ class User(Base, Dictifiable, RepresentById):
"GalaxySession", back_populates="user", order_by=lambda: desc(GalaxySession.update_time) # type: ignore[has-type]
)
quotas = relationship("UserQuotaAssociation", back_populates="user")
+ quota_source_usages = relationship("UserQuotaSourceUsage", back_populates="user")
social_auth = relationship("UserAuthnzToken", back_populates="user")
stored_workflow_menu_entries = relationship(
"StoredWorkflowMenuEntry",
@@ -620,6 +745,7 @@ class User(Base, Dictifiable, RepresentById):
"deleted",
"active",
"last_password_change",
+ "preferred_object_store_id",
]
def __init__(self, email=None, password=None, username=None):
@@ -727,14 +853,31 @@ def all_roles_exploiting_cache(self):
roles.append(role)
return roles
- def get_disk_usage(self, nice_size=False):
+ def get_disk_usage(self, nice_size=False, quota_source_label=None):
"""
Return byte count of disk space used by user or a human-readable
string if `nice_size` is `True`.
"""
- rval = 0
- if self.disk_usage is not None:
- rval = self.disk_usage
+ if quota_source_label is None:
+ rval = 0
+ if self.disk_usage is not None:
+ rval = self.disk_usage
+ else:
+ statement = """
+SELECT DISK_USAGE
+FROM user_quota_source_usage
+WHERE user_id = :user_id and quota_source_label = :label
+"""
+ sa_session = object_session(self)
+ params = {
+ "user_id": self.id,
+ "label": quota_source_label,
+ }
+ row = sa_session.execute(statement, params).fetchone()
+ if row is not None:
+ rval = row[0]
+ else:
+ rval = 0
if nice_size:
rval = galaxy.util.nice_size(rval)
return rval
@@ -747,9 +890,36 @@ def set_disk_usage(self, bytes):
total_disk_usage = property(get_disk_usage, set_disk_usage)
- def adjust_total_disk_usage(self, amount):
+ def adjust_total_disk_usage(self, amount, quota_source_label):
+ assert amount is not None
if amount != 0:
- self.disk_usage = func.coalesce(self.table.c.disk_usage, 0) + amount
+ if quota_source_label is None:
+ self.disk_usage = func.coalesce(self.table.c.disk_usage, 0) + amount
+ else:
+ # else would work on newer sqlite - 3.24.0
+ sa_session = object_session(self)
+ if "sqlite" in sa_session.bind.dialect.name:
+ # hacky alternative for older sqlite
+ statement = """
+WITH new (user_id, quota_source_label) AS ( VALUES(:user_id, :label) )
+INSERT OR REPLACE INTO user_quota_source_usage (id, user_id, quota_source_label, disk_usage)
+SELECT old.id, new.user_id, new.quota_source_label, COALESCE(old.disk_usage + :amount, :amount)
+FROM new LEFT JOIN user_quota_source_usage AS old ON new.user_id = old.user_id AND NEW.quota_source_label = old.quota_source_label;
+"""
+ else:
+ statement = """
+INSERT INTO user_quota_source_usage(user_id, disk_usage, quota_source_label)
+VALUES(:user_id, :amount, :label)
+ON CONFLICT
+ ON constraint uqsu_unique_label_per_user
+ DO UPDATE SET disk_usage = user_quota_source_usage.disk_usage + :amount
+"""
+ params = {
+ "user_id": self.id,
+ "amount": int(amount),
+ "label": quota_source_label,
+ }
+ sa_session.execute(statement, params)
@property
def nice_total_disk_usage(self):
@@ -758,53 +928,54 @@ def nice_total_disk_usage(self):
"""
return self.get_disk_usage(nice_size=True)
- def calculate_disk_usage(self):
+ def calculate_disk_usage_default_source(self, object_store):
"""
Return byte count total of disk space used by all non-purged, non-library
- HDAs in non-purged histories.
+ HDAs in non-purged histories assigned to default quota source.
"""
- # maintain a list so that we don't double count
- return self._calculate_or_set_disk_usage(dryrun=True)
+ # only used in set_user_disk_usage.py
+ assert object_store is not None
+ quota_source_map = object_store.get_quota_source_map()
+ default_quota_enabled = quota_source_map.default_quota_enabled
+ default_cond = "dataset.object_store_id IS NULL OR" if default_quota_enabled else ""
+ default_usage_dataset_condition = (
+ "{default_cond} dataset.object_store_id NOT IN :exclude_object_store_ids".format(
+ default_cond=default_cond,
+ )
+ )
+ default_usage = UNIQUE_DATASET_USER_USAGE.format(dataset_condition=default_usage_dataset_condition)
+ sql_calc = text(default_usage)
+ sql_calc = sql_calc.bindparams(bindparam("id"), bindparam("exclude_object_store_ids", expanding=True))
+ params = {"id": self.id, "exclude_object_store_ids": quota_source_map.default_usage_excluded_ids()}
+ sa_session = object_session(self)
+ usage = sa_session.scalar(sql_calc, params)
+ return usage
- def calculate_and_set_disk_usage(self):
+ def calculate_and_set_disk_usage(self, object_store):
"""
Calculates and sets user disk usage.
"""
- self._calculate_or_set_disk_usage(dryrun=False)
+ self._calculate_or_set_disk_usage(object_store=object_store)
- def _calculate_or_set_disk_usage(self, dryrun=True):
+ def _calculate_or_set_disk_usage(self, object_store):
"""
Utility to calculate and return the disk usage. If dryrun is False,
the new value is set immediately.
"""
- sql_calc = text(
- """
- WITH per_user_histories AS
- (
- SELECT id
- FROM history
- WHERE user_id = :id
- AND NOT purged
- ),
- per_hist_hdas AS (
- SELECT DISTINCT dataset_id
- FROM history_dataset_association
- WHERE NOT purged
- AND history_id IN (SELECT id FROM per_user_histories)
- )
- SELECT SUM(COALESCE(dataset.total_size, dataset.file_size, 0))
- FROM dataset
- LEFT OUTER JOIN library_dataset_dataset_association ON dataset.id = library_dataset_dataset_association.dataset_id
- WHERE dataset.id IN (SELECT dataset_id FROM per_hist_hdas)
- AND library_dataset_dataset_association.id IS NULL
- """
- )
+ assert object_store is not None
+ quota_source_map = object_store.get_quota_source_map()
sa_session = object_session(self)
- usage = sa_session.scalar(sql_calc, {"id": self.id})
- if not dryrun:
- self.set_disk_usage(usage)
+ for_sqlite = "sqlite" in sa_session.bind.dialect.name
+ statements = calculate_user_disk_usage_statements(self.id, quota_source_map, for_sqlite)
+ for sql, args in statements:
+ statement = text(sql)
+ binds = []
+ for key, _ in args.items():
+ expand_binding = key.endswith("s")
+ binds.append(bindparam(key, expanding=expand_binding))
+ statement = statement.bindparams(*binds)
+ sa_session.execute(statement, args)
sa_session.flush()
- return usage
@staticmethod
def user_template_environment(user):
@@ -868,6 +1039,66 @@ def attempt_create_private_role(self):
session.add(assoc)
session.flush()
+ def dictify_usage(self, object_store=None) -> List[UserQuotaBasicUsage]:
+ """Include object_store to include empty/unused usage info."""
+ used_labels: Set[Union[str, None]] = set()
+ rval: List[UserQuotaBasicUsage] = [
+ UserQuotaBasicUsage(
+ quota_source_label=None,
+ total_disk_usage=float(self.disk_usage or 0),
+ )
+ ]
+ used_labels.add(None)
+ for quota_source_usage in self.quota_source_usages:
+ label = quota_source_usage.quota_source_label
+ rval.append(
+ UserQuotaBasicUsage(
+ quota_source_label=label,
+ total_disk_usage=float(quota_source_usage.disk_usage),
+ )
+ )
+ used_labels.add(label)
+
+ if object_store is not None:
+ for label in object_store.get_quota_source_map().ids_per_quota_source().keys():
+ if label not in used_labels:
+ rval.append(
+ UserQuotaBasicUsage(
+ quota_source_label=label,
+ total_disk_usage=0.0,
+ )
+ )
+
+ return rval
+
+ def dictify_usage_for(self, quota_source_label: Optional[str]) -> UserQuotaBasicUsage:
+ rval: UserQuotaBasicUsage
+ if quota_source_label is None:
+ rval = UserQuotaBasicUsage(
+ quota_source_label=None,
+ total_disk_usage=float(self.disk_usage or 0),
+ )
+ else:
+ quota_source_usage = self.quota_source_usage_for(quota_source_label)
+ if quota_source_usage is None:
+ rval = UserQuotaBasicUsage(
+ quota_source_label=quota_source_label,
+ total_disk_usage=0.0,
+ )
+ else:
+ rval = UserQuotaBasicUsage(
+ quota_source_label=quota_source_label,
+ total_disk_usage=float(quota_source_usage.disk_usage),
+ )
+
+ return rval
+
+ def quota_source_usage_for(self, quota_source_label: Optional[str]) -> Optional["UserQuotaSourceUsage"]:
+ for quota_source_usage in self.quota_source_usages:
+ if quota_source_usage.quota_source_label == quota_source_label:
+ return quota_source_usage
+ return None
+
class PasswordResetToken(Base):
__tablename__ = "password_reset_token"
@@ -1010,6 +1241,8 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable):
imported = Column(Boolean, default=False, index=True)
params = Column(TrimmedString(255), index=True)
handler = Column(TrimmedString(255), index=True)
+ preferred_object_store_id = Column(String(255), nullable=True)
+ object_store_id_overrides = Column(JSONType)
user = relationship("User")
galaxy_session = relationship("GalaxySession")
@@ -1483,6 +1716,19 @@ def remap_objects(p, k, obj):
job_attrs["params"] = params_dict
return job_attrs
+ def requires_shareable_storage(self, security_agent):
+ # An easy optimization would be to calculate this in galaxy.tools.actions when the
+ # job is created and all the output permissions are already known. Having to reload
+ # these permissions in the job code shouldn't strictly be needed.
+
+ requires_sharing = False
+ for dataset_assoc in self.output_datasets + self.output_library_datasets:
+ if not security_agent.dataset_is_private_to_a_user(dataset_assoc.dataset.dataset):
+ requires_sharing = True
+ break
+
+ return requires_sharing
+
def to_dict(self, view="collection", system_details=False):
if view == "admin_job_list":
rval = super().to_dict(view="collection")
@@ -2512,6 +2758,7 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable
importable = Column(Boolean, default=False)
slug = Column(TEXT)
published = Column(Boolean, index=True, default=False)
+ preferred_object_store_id = Column(String(255), nullable=True)
datasets = relationship(
"HistoryDatasetAssociation", back_populates="history", cascade_backrefs=False, order_by=lambda: asc(HistoryDatasetAssociation.hid) # type: ignore[has-type]
@@ -2610,6 +2857,7 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable
"importable",
"slug",
"empty",
+ "preferred_object_store_id",
]
default_name = "Unnamed history"
@@ -2714,7 +2962,9 @@ def add_dataset(self, dataset, parent_id=None, genome_build=None, set_hid=True,
dataset.hid = self._next_hid()
add_object_to_object_session(dataset, self)
if quota and is_dataset and self.user:
- self.user.adjust_total_disk_usage(dataset.quota_amount(self.user))
+ quota_source_info = dataset.dataset.quota_source_info
+ if quota_source_info.use:
+ self.user.adjust_total_disk_usage(dataset.quota_amount(self.user), quota_source_info.label)
dataset.history = self
if is_dataset and genome_build not in [None, "?"]:
self.genome_build = genome_build
@@ -2732,7 +2982,10 @@ def add_datasets(
self.__add_datasets_optimized(datasets, genome_build=genome_build)
if quota and self.user:
disk_usage = sum(d.get_total_size() for d in datasets if is_hda(d))
- self.user.adjust_total_disk_usage(disk_usage)
+ if disk_usage:
+ quota_source_info = datasets[0].dataset.quota_source_info
+ if quota_source_info.use:
+ self.user.adjust_total_disk_usage(disk_usage, quota_source_info.label)
sa_session.add_all(datasets)
if flush:
sa_session.flush()
@@ -3032,7 +3285,14 @@ def __filter_contents(self, content_class, **kwds):
visible = galaxy.util.string_as_bool_or_none(kwds.get("visible", None))
if visible is not None:
query = query.filter(content_class.visible == visible)
+ if "object_store_ids" in kwds:
+ if content_class == HistoryDatasetAssociation:
+ query = query.join(content_class.dataset).filter(
+ Dataset.table.c.object_store_id.in_(kwds.get("object_store_ids"))
+ )
+ # else ignoring object_store_ids on HDCAs...
if "ids" in kwds:
+ assert "object_store_ids" not in kwds
ids = kwds["ids"]
max_in_filter_length = kwds.get("max_in_filter_length", MAX_IN_FILTER_LENGTH)
if len(ids) < max_in_filter_length:
@@ -3125,6 +3385,20 @@ def __init__(self, name=None, description=None, type=types.SYSTEM, deleted=False
self.deleted = deleted
+class UserQuotaSourceUsage(Base, Dictifiable, RepresentById):
+ __tablename__ = "user_quota_source_usage"
+ __table_args__ = (UniqueConstraint("user_id", "quota_source_label", name="uqsu_unique_label_per_user"),)
+
+ dict_element_visible_keys = ["disk_usage", "quota_source_label"]
+
+ id = Column(Integer, primary_key=True)
+ user_id = Column(Integer, ForeignKey("galaxy_user.id"), index=True)
+ quota_source_label = Column(String(32), index=True)
+ # user had an index on disk_usage - does that make any sense? -John
+ disk_usage = Column(Numeric(15, 0), default=0, nullable=False)
+ user = relationship("User", back_populates="quota_source_usages")
+
+
class UserQuotaAssociation(Base, Dictifiable, RepresentById):
__tablename__ = "user_quota_association"
@@ -3165,6 +3439,7 @@ def __init__(self, group, quota):
class Quota(Base, Dictifiable, RepresentById):
__tablename__ = "quota"
+ __table_args__ = (Index("ix_quota_quota_source_label", "quota_source_label"),)
id = Column(Integer, primary_key=True)
create_time = Column(DateTime, default=now)
@@ -3174,11 +3449,12 @@ class Quota(Base, Dictifiable, RepresentById):
bytes = Column(BigInteger)
operation = Column(String(8))
deleted = Column(Boolean, index=True, default=False)
+ quota_source_label = Column(String(32), default=None)
default = relationship("DefaultQuotaAssociation", back_populates="quota")
groups = relationship("GroupQuotaAssociation", back_populates="quota")
users = relationship("UserQuotaAssociation", back_populates="quota")
- dict_collection_visible_keys = ["id", "name"]
+ dict_collection_visible_keys = ["id", "name", "quota_source_label"]
dict_element_visible_keys = [
"id",
"name",
@@ -3189,10 +3465,11 @@ class Quota(Base, Dictifiable, RepresentById):
"default",
"users",
"groups",
+ "quota_source_label",
]
valid_operations = ("+", "-", "=")
- def __init__(self, name=None, description=None, amount=0, operation="="):
+ def __init__(self, name=None, description=None, amount=0, operation="=", quota_source_label=None):
self.name = name
self.description = description
if amount is None:
@@ -3200,6 +3477,7 @@ def __init__(self, name=None, description=None, amount=0, operation="="):
else:
self.bytes = amount
self.operation = operation
+ self.quota_source_label = quota_source_label
def get_amount(self):
if self.bytes == -1:
@@ -3228,7 +3506,7 @@ class DefaultQuotaAssociation(Base, Dictifiable, RepresentById):
id = Column(Integer, primary_key=True)
create_time = Column(DateTime, default=now)
update_time = Column(DateTime, default=now, onupdate=now)
- type = Column(String(32), index=True, unique=True)
+ type = Column(String(32), index=True)
quota_id = Column(Integer, ForeignKey("quota.id"), index=True)
quota = relationship("Quota", back_populates="default")
@@ -3538,14 +3816,27 @@ def is_new(self):
def in_ready_state(self):
return self.state in self.ready_states
+ @property
+ def shareable(self):
+ """Return True if placed into an objectstore not labeled as ``private``."""
+ if self.external_filename:
+ return True
+ else:
+ object_store = self._assert_object_store_set()
+ return not object_store.is_private(self)
+
+ def ensure_shareable(self):
+ if not self.shareable:
+ raise Exception(CANNOT_SHARE_PRIVATE_DATASET_MESSAGE)
+
def get_file_name(self):
if self.purged:
log.warning(f"Attempt to get file name of purged dataset {self.id}")
return ""
if not self.external_filename:
- assert self.object_store is not None, f"Object Store has not been initialized for dataset {self.id}"
- if self.object_store.exists(self):
- file_name = self.object_store.get_filename(self)
+ object_store = self._assert_object_store_set()
+ if object_store.exists(self):
+ file_name = object_store.get_filename(self)
else:
file_name = ""
if not file_name and self.state not in (self.states.NEW, self.states.QUEUED):
@@ -3558,6 +3849,16 @@ def get_file_name(self):
# Make filename absolute
return os.path.abspath(filename)
+ @property
+ def quota_source_label(self):
+ return self.quota_source_info.label
+
+ @property
+ def quota_source_info(self):
+ object_store_id = self.object_store_id
+ quota_source_map = self.object_store.get_quota_source_map()
+ return quota_source_map.get_quota_source_info(object_store_id)
+
def set_file_name(self, filename):
if not filename:
self.external_filename = None
@@ -3566,6 +3867,10 @@ def set_file_name(self, filename):
file_name = property(get_file_name, set_file_name)
+ def _assert_object_store_set(self):
+ assert self.object_store is not None, f"Object Store has not been initialized for dataset {self.id}"
+ return self.object_store
+
def get_extra_files_path(self):
# Unlike get_file_name - external_extra_files_path is not backed by an
# actual database column so if SA instantiates this object - the
@@ -4601,6 +4906,9 @@ def to_library_dataset_dataset_association(
"""
Copy this HDA to a library optionally replacing an existing LDDA.
"""
+ if not self.dataset.shareable:
+ raise Exception("Attempting to share a non-shareable dataset.")
+
if replace_dataset:
# The replace_dataset param ( when not None ) refers to a LibraryDataset that
# is being replaced with a new version.
@@ -4665,10 +4973,10 @@ def get_access_roles(self, security_agent):
"""
return self.dataset.get_access_roles(security_agent)
- def purge_usage_from_quota(self, user):
+ def purge_usage_from_quota(self, user, quota_source_info):
"""Remove this HDA's quota_amount from user's quota."""
- if user:
- user.adjust_total_disk_usage(-self.quota_amount(user))
+ if user and quota_source_info.use:
+ user.adjust_total_disk_usage(-self.quota_amount(user), quota_source_info.label)
def quota_amount(self, user):
"""
@@ -8014,6 +8322,48 @@ def history_id(self):
return self.workflow_invocation.history_id
+class EffectiveOutput(TypedDict):
+ """An output for the sake or determining full workflow outputs.
+
+ A workflow output might not be an effective output if it is an
+ output on a subworkflow or a parent workflow that doesn't declare
+ it an output.
+
+ This is currently only used for determining object store selections.
+ We don't want to capture subworkflow outputs that the user would like
+ to ignore and discard as effective workflow outputs.
+ """
+
+ output_name: str
+ step_id: int
+
+
+class WorkflowInvocationStepObjectStores(NamedTuple):
+ preferred_object_store_id: Optional[str]
+ preferred_outputs_object_store_id: Optional[str]
+ preferred_intermediate_object_store_id: Optional[str]
+ step_effective_outputs: Optional[List["EffectiveOutput"]]
+
+ def is_output_name_an_effective_output(self, output_name: str) -> bool:
+ if self.step_effective_outputs is None:
+ return True
+ else:
+ for effective_output in self.step_effective_outputs:
+ if effective_output["output_name"] == output_name:
+ return True
+
+ return False
+
+ @property
+ def is_split_configuration(self):
+ preferred_outputs_object_store_id = self.preferred_outputs_object_store_id
+ preferred_intermediate_object_store_id = self.preferred_intermediate_object_store_id
+ has_typed_preferences = (
+ preferred_outputs_object_store_id is not None or preferred_intermediate_object_store_id is not None
+ )
+ return has_typed_preferences and preferred_outputs_object_store_id != preferred_intermediate_object_store_id
+
+
class WorkflowInvocationStep(Base, Dictifiable, Serializable):
__tablename__ = "workflow_invocation_step"
@@ -8110,6 +8460,36 @@ def jobs(self):
else:
return []
+ @property
+ def preferred_object_stores(self) -> WorkflowInvocationStepObjectStores:
+ meta_type = WorkflowRequestInputParameter.types.META_PARAMETERS
+ preferred_object_store_id = None
+ preferred_outputs_object_store_id = None
+ preferred_intermediate_object_store_id = None
+ step_effective_outputs: Optional[List["EffectiveOutput"]] = None
+
+ workflow_invocation = self.workflow_invocation
+ for input_parameter in workflow_invocation.input_parameters:
+ if input_parameter.type != meta_type:
+ continue
+ if input_parameter.name == "preferred_object_store_id":
+ preferred_object_store_id = input_parameter.value
+ elif input_parameter.name == "preferred_outputs_object_store_id":
+ preferred_outputs_object_store_id = input_parameter.value
+ elif input_parameter.name == "preferred_intermediate_object_store_id":
+ preferred_intermediate_object_store_id = input_parameter.value
+ elif input_parameter.name == "effective_outputs":
+ all_effective_outputs = json.loads(input_parameter.value)
+ step_id = self.workflow_step_id
+ step_effective_outputs = [e for e in all_effective_outputs if e["step_id"] == step_id]
+
+ return WorkflowInvocationStepObjectStores(
+ preferred_object_store_id,
+ preferred_outputs_object_store_id,
+ preferred_intermediate_object_store_id,
+ step_effective_outputs,
+ )
+
def _serialize(self, id_encoder, serialization_options):
step_attrs = dict_for(self)
step_attrs["state"] = self.state
diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/9540a051226e_preferred_object_store_ids.py b/lib/galaxy/model/migrations/alembic/versions_gxy/9540a051226e_preferred_object_store_ids.py
new file mode 100644
index 000000000000..807e520bf09e
--- /dev/null
+++ b/lib/galaxy/model/migrations/alembic/versions_gxy/9540a051226e_preferred_object_store_ids.py
@@ -0,0 +1,36 @@
+"""preferred_object_store_ids
+
+Revision ID: 9540a051226e
+Revises: d0583094c8cd
+Create Date: 2022-06-10 10:38:25.212102
+
+"""
+from alembic import op
+from sqlalchemy import (
+ Column,
+ String,
+)
+
+from galaxy.model.custom_types import JSONType
+from galaxy.model.migrations.util import drop_column
+
+# revision identifiers, used by Alembic.
+revision = "9540a051226e"
+down_revision = "d0583094c8cd"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ preferred_object_store_type = String(255)
+ op.add_column("galaxy_user", Column("preferred_object_store_id", preferred_object_store_type, default=None))
+ op.add_column("history", Column("preferred_object_store_id", preferred_object_store_type, default=None))
+ op.add_column("job", Column("preferred_object_store_id", preferred_object_store_type, default=None))
+ op.add_column("job", Column("object_store_id_overrides", JSONType))
+
+
+def downgrade():
+ drop_column("galaxy_user", "preferred_object_store_id")
+ drop_column("history", "preferred_object_store_id")
+ drop_column("job", "preferred_object_store_id")
+ drop_column("job", "object_store_id_overrides")
diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/d0583094c8cd_add_quota_source_labels.py b/lib/galaxy/model/migrations/alembic/versions_gxy/d0583094c8cd_add_quota_source_labels.py
new file mode 100644
index 000000000000..867ac31aa43f
--- /dev/null
+++ b/lib/galaxy/model/migrations/alembic/versions_gxy/d0583094c8cd_add_quota_source_labels.py
@@ -0,0 +1,49 @@
+"""add quota source labels
+
+Revision ID: d0583094c8cd
+Revises: c39f1de47a04
+Create Date: 2022-06-09 12:24:44.329038
+
+"""
+from alembic import op
+from sqlalchemy import (
+ Column,
+ ForeignKey,
+ Integer,
+ Numeric,
+ String,
+)
+
+from galaxy.model.migrations.util import (
+ add_unique_constraint,
+ drop_column,
+)
+
+# revision identifiers, used by Alembic.
+revision = "d0583094c8cd"
+down_revision = "c39f1de47a04"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.add_column("quota", Column("quota_source_label", String(32), default=None))
+
+ op.create_table(
+ "user_quota_source_usage",
+ Column("id", Integer, primary_key=True),
+ Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
+ Column("quota_source_label", String(32), index=True),
+ # user had an index on disk_usage - does that make any sense? -John
+ Column("disk_usage", Numeric(15, 0)),
+ )
+ add_unique_constraint("uqsu_unique_label_per_user", "user_quota_source_usage", ["user_id", "quota_source_label"])
+ op.drop_index("ix_default_quota_association_type", "default_quota_association")
+ op.create_index("ix_quota_quota_source_label", "quota", ["quota_source_label"])
+
+
+def downgrade():
+ op.create_index("ix_default_quota_association_type", "default_quota_association", ["type"], unique=True)
+ op.drop_table("user_quota_source_usage")
+ op.drop_index("ix_quota_quota_source_label", "quota")
+ drop_column("quota", "quota_source_label")
diff --git a/lib/galaxy/model/migrations/util.py b/lib/galaxy/model/migrations/util.py
index 37d4eb2d95f5..fc5e38ff3d3d 100644
--- a/lib/galaxy/model/migrations/util.py
+++ b/lib/galaxy/model/migrations/util.py
@@ -1,4 +1,5 @@
import logging
+from typing import List
from alembic import (
context,
@@ -17,6 +18,22 @@ def drop_column(table_name, column_name):
batch_op.drop_column(column_name)
+def add_unique_constraint(index_name: str, table_name: str, columns: List[str]):
+ if _is_sqlite():
+ with op.batch_alter_table(table_name) as batch_op:
+ batch_op.create_unique_constraint(index_name, columns)
+ else:
+ op.create_unique_constraint(index_name, table_name, columns)
+
+
+def drop_unique_constraint(index_name: str, table_name: str):
+ if _is_sqlite():
+ with op.batch_alter_table(table_name) as batch_op:
+ batch_op.drop_constraint(index_name)
+ else:
+ op.drop_constraint(index_name, table_name)
+
+
def column_exists(table_name, column_name):
if context.is_offline_mode():
return _handle_offline_mode(f"column_exists({table_name}, {column_name})", False)
@@ -34,3 +51,8 @@ def _handle_offline_mode(code, return_value):
)
log.info(msg)
return return_value
+
+
+def _is_sqlite() -> bool:
+ bind = op.get_context().bind
+ return bool(bind and bind.engine.name == "sqlite")
diff --git a/lib/galaxy/model/security.py b/lib/galaxy/model/security.py
index 881e3276bf47..7c698feac779 100644
--- a/lib/galaxy/model/security.py
+++ b/lib/galaxy/model/security.py
@@ -899,16 +899,23 @@ def set_all_dataset_permissions(self, dataset, permissions=None, new=False, flus
# Make sure that DATASET_MANAGE_PERMISSIONS is associated with at least 1 role
has_dataset_manage_permissions = False
permissions = permissions or {}
- for action, roles in permissions.items():
- if isinstance(action, Action):
- if action == self.permitted_actions.DATASET_MANAGE_PERMISSIONS and roles:
- has_dataset_manage_permissions = True
- break
- elif action == self.permitted_actions.DATASET_MANAGE_PERMISSIONS.action and roles:
- has_dataset_manage_permissions = True
- break
+ for _ in _walk_action_roles(permissions, self.permitted_actions.DATASET_MANAGE_PERMISSIONS):
+ has_dataset_manage_permissions = True
+ break
if not has_dataset_manage_permissions:
return "At least 1 role must be associated with manage permissions on this dataset."
+
+ # If this is new, the objectstore likely hasn't been set yet - defer check until
+ # the job handler assigns it.
+ if not new and not dataset.shareable:
+ # ensure dataset not shared.
+ dataset_access_roles = []
+ for _, roles in _walk_action_roles(permissions, self.permitted_actions.DATASET_ACCESS):
+ dataset_access_roles.extend(roles)
+
+ if len(dataset_access_roles) != 1 or dataset_access_roles[0].type != self.model.Role.types.PRIVATE:
+ return galaxy.model.CANNOT_SHARE_PRIVATE_DATASET_MESSAGE
+
flush_needed = False
# Delete all of the current permissions on the dataset
if not new:
@@ -937,6 +944,12 @@ def set_dataset_permission(self, dataset, permission=None):
Permission looks like: { Action.action : [ Role, Role ] }
"""
permission = permission or {}
+
+ # if modifying access - ensure it is shareable.
+ for _ in _walk_action_roles(permission, self.permitted_actions.DATASET_ACCESS):
+ dataset.ensure_shareable()
+ break
+
flush_needed = False
for action, roles in permission.items():
if isinstance(action, Action):
@@ -976,6 +989,7 @@ def copy_dataset_permissions(self, src, dst):
self.set_all_dataset_permissions(dst, self.get_permissions(src))
def privately_share_dataset(self, dataset, users=None):
+ dataset.ensure_shareable()
intersect = None
users = users or []
for user in users:
@@ -1154,6 +1168,19 @@ def dataset_is_private_to_user(self, trans, dataset):
else:
return False
+ def dataset_is_private_to_a_user(self, dataset):
+ """
+ If the Dataset object has exactly one access role and that is
+ the current user's private role then we consider the dataset private.
+ """
+ access_roles = dataset.get_access_roles(self)
+
+ if len(access_roles) != 1:
+ return False
+ else:
+ access_role = access_roles[0]
+ return access_role.type == self.model.Role.types.PRIVATE
+
def datasets_are_public(self, trans, datasets):
"""
Given a transaction object and a list of Datasets, return
@@ -1188,6 +1215,8 @@ def datasets_are_public(self, trans, datasets):
def make_dataset_public(self, dataset):
# A dataset is considered public if there are no "access" actions associated with it. Any
# other actions ( 'manage permissions', 'edit metadata' ) are irrelevant.
+ dataset.ensure_shareable()
+
flush_needed = False
for dp in dataset.actions:
if dp.action == self.permitted_actions.DATASET_ACCESS.action:
@@ -1635,3 +1664,12 @@ def set_dataset_permissions(self, hda, user, site):
hdadaa = self.model.HistoryDatasetAssociationDisplayAtAuthorization(hda=hda, user=user, site=site)
self.sa_session.add(hdadaa)
self.sa_session.flush()
+
+
+def _walk_action_roles(permissions, query_action):
+ for action, roles in permissions.items():
+ if isinstance(action, Action):
+ if action == query_action and roles:
+ yield action, roles
+ elif action == query_action.action and roles:
+ yield action, roles
diff --git a/lib/galaxy/model/store/discover.py b/lib/galaxy/model/store/discover.py
index 2f169ea3d673..19a4c1e84194 100644
--- a/lib/galaxy/model/store/discover.py
+++ b/lib/galaxy/model/store/discover.py
@@ -82,6 +82,7 @@ def create_dataset(
created_from_basename=None,
final_job_state="ok",
creating_job_id=None,
+ output_name=None,
storage_callbacks=None,
):
tag_list = tag_list or []
@@ -190,6 +191,7 @@ def create_dataset(
extra_files=extra_files,
filename=filename,
link_data=link_data,
+ output_name=output_name,
)
else:
storage_callbacks.append(
@@ -199,14 +201,19 @@ def create_dataset(
extra_files=extra_files,
filename=filename,
link_data=link_data,
+ output_name=output_name,
)
)
return primary_data
- def finalize_storage(self, primary_data, dataset_attributes, extra_files, filename, link_data):
+ def finalize_storage(self, primary_data, dataset_attributes, extra_files, filename, link_data, output_name):
# Move data from temp location to dataset location
if not link_data:
- self.object_store.update_from_file(primary_data.dataset, file_name=filename, create=True)
+ dataset = primary_data.dataset
+ object_store_id = self.override_object_store_id(output_name)
+ if object_store_id:
+ dataset.object_store_id = object_store_id
+ self.object_store.update_from_file(dataset, file_name=filename, create=True)
else:
primary_data.link_to(filename)
if extra_files:
@@ -374,6 +381,7 @@ def _populate_elements(self, chunk, name, root_collection_builder, metadata_sour
datasets=element_datasets["datasets"],
paths=element_datasets["paths"],
extra_files=element_datasets["extra_files"],
+ output_name=name,
)
log.debug(
"(%s) Add dynamic collection datasets to history for output [%s] %s",
@@ -388,8 +396,12 @@ def add_tags_to_datasets(self, datasets, tag_lists):
for dataset, tags in zip(datasets, tag_lists):
self.tag_handler.add_tags_from_list(self.user, dataset, tags, flush=False)
- def update_object_store_with_datasets(self, datasets, paths, extra_files):
+ def update_object_store_with_datasets(self, datasets, paths, extra_files, output_name):
for dataset, path, extra_file in zip(datasets, paths, extra_files):
+ object_store_id = self.override_object_store_id(output_name)
+ if object_store_id:
+ dataset.dataset.object_store_id = object_store_id
+
self.object_store.update_from_file(dataset.dataset, file_name=path, create=True)
if extra_file:
persist_extra_files(self.object_store, extra_files, dataset)
@@ -435,6 +447,15 @@ def get_implicit_collection_jobs_association_id(self) -> Optional[str]:
def job(self) -> Optional[galaxy.model.Job]:
"""Return associated job object if bound to a job finish context connected to a database."""
+ def override_object_store_id(self, output_name: Optional[str] = None) -> Optional[str]:
+ """Object store ID to assign to a dataset before populating its contents."""
+ job = self.job
+ if not job:
+ return None
+ default_object_store_id = job.object_store_id
+ object_store_id_overrides = job.object_store_id_overrides or {}
+ return object_store_id_overrides.get(output_name, default_object_store_id)
+
@property
@abc.abstractmethod
def metadata_source_provider(self) -> "MetadataSourceProvider":
diff --git a/lib/galaxy/model/unittest_utils/data_app.py b/lib/galaxy/model/unittest_utils/data_app.py
index 60f62f1077cc..8c4430ab8b26 100644
--- a/lib/galaxy/model/unittest_utils/data_app.py
+++ b/lib/galaxy/model/unittest_utils/data_app.py
@@ -68,6 +68,7 @@ def __init__(self, root=None, **kwd):
self.new_file_path = os.path.join(self.data_dir, "tmp")
self.file_path = os.path.join(self.data_dir, "files")
self.server_name = "main"
+ self.enable_quotas = False
def __del__(self):
if self._remove_root:
diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py
index 17708ecc7db2..d0b0e35b4c05 100644
--- a/lib/galaxy/objectstore/__init__.py
+++ b/lib/galaxy/objectstore/__init__.py
@@ -16,11 +16,19 @@
Any,
Dict,
List,
+ NamedTuple,
+ Optional,
+ Set,
Tuple,
Type,
)
import yaml
+from pydantic import BaseModel
+from typing_extensions import (
+ Literal,
+ TypedDict,
+)
from galaxy.exceptions import (
ObjectInvalid,
@@ -43,9 +51,50 @@
NO_SESSION_ERROR_MESSAGE = (
"Attempted to 'create' object store entity in configuration with no database session present."
)
+DEFAULT_PRIVATE = False
+DEFAULT_QUOTA_SOURCE = None # Just track quota right on user object in Galaxy.
+DEFAULT_QUOTA_ENABLED = True # enable quota tracking in object stores by default
log = logging.getLogger(__name__)
+BadgeSourceT = Literal["admin", "galaxy"]
+BadgeT = Literal[
+ "faster",
+ "slower",
+ "short_term",
+ "cloud",
+ "backed_up",
+ "not_backed_up",
+ "more_secure",
+ "less_secure",
+ "more_stable",
+ "less_stable",
+ "quota",
+ "no_quota",
+ "restricted",
+]
+
+BADGE_SPECIFICATION = [
+ {"type": "faster", "conflicts": ["slower"]},
+ {"type": "slower", "conflicts": ["faster"]},
+ {"type": "short_term", "conflicts": []},
+ {"type": "cloud", "conflicts": []},
+ {"type": "backed_up", "conflicts": ["not_backed_up"]},
+ {"type": "not_backed_up", "conflicts": ["backed_up"]},
+ {"type": "more_secure", "conflicts": ["less_secure"]},
+ {"type": "less_secure", "conflicts": ["more_secure"]},
+ {"type": "more_stable", "conflicts": ["less_stable"]},
+ {"type": "less_stable", "conflicts": ["more_stable"]},
+]
+KNOWN_BADGE_TYPES = [s["type"] for s in BADGE_SPECIFICATION]
+BADGE_SPECIFCATION_BY_TYPE = {s["type"]: s for s in BADGE_SPECIFICATION}
+
+
+class BadgeDict(TypedDict):
+ type: BadgeT
+ message: Optional[str]
+ source: BadgeSourceT
+
class ObjectStore(metaclass=abc.ABCMeta):
@@ -105,6 +154,9 @@ def create(
This method will create a proper directory structure for
the file if the directory does not already exist.
+
+ The method returns the concrete objectstore the supplied object is stored
+ in.
"""
raise NotImplementedError()
@@ -244,6 +296,35 @@ def get_concrete_store_description_markdown(self, obj):
yet been set, this may return ``None``.
"""
+ @abc.abstractmethod
+ def get_concrete_store_badges(self, obj) -> List[BadgeDict]:
+ """Return a list of dictified badges summarizing the object store configuration."""
+
+ @abc.abstractmethod
+ def is_private(self, obj):
+ """Return True iff supplied object is stored in private ConcreteObjectStore."""
+
+ def object_store_ids(self, private=None):
+ """Return IDs of all concrete object stores - either private ones or non-private ones.
+
+ This should just return an empty list for non-DistributedObjectStore object stores,
+ i.e. concrete objectstores and the HierarchicalObjectStore since these do not
+ use the object_store_id column for objects (Galaxy Datasets).
+ """
+ return []
+
+ def object_store_allows_id_selection(self) -> bool:
+ """Return True if this object store respects object_store_id and allow selection of this."""
+ return False
+
+ def object_store_ids_allowing_selection(self) -> List[str]:
+ """Return a non-emtpy list of allowed selectable object store IDs during creation."""
+ return []
+
+ def get_concrete_store_by_object_store_id(self, object_store_id: str) -> Optional["ConcreteObjectStore"]:
+ """If this is a distributed object store, get ConcreteObjectStore by object_store_id."""
+ return None
+
@abc.abstractmethod
def get_store_usage_percent(self):
"""Return the percentage indicating how full the store is."""
@@ -261,6 +342,10 @@ def get_store_by(self, obj):
def to_dict(self) -> Dict[str, Any]:
raise NotImplementedError
+ @abc.abstractmethod
+ def get_quota_source_map(self):
+ """Return QuotaSourceMap describing mapping of object store IDs to quota sources."""
+
class BaseObjectStore(ObjectStore):
store_by: str
@@ -284,6 +369,7 @@ def __init__(self, config, config_dict=None, **kwargs):
self.running = True
self.config = config
self.check_old_style = config.object_store_check_old_style
+ self.galaxy_enable_quotas = config.enable_quotas
extra_dirs = {}
extra_dirs["job_work"] = config.jobs_directory
extra_dirs["temp"] = config.new_file_path
@@ -329,10 +415,11 @@ def to_dict(self):
extra_dirs = []
for extra_dir_type, extra_dir_path in self.extra_dirs.items():
extra_dirs.append({"type": extra_dir_type, "path": extra_dir_path})
+ store_type = self.store_type
return {
"config": config_to_dict(self.config),
"extra_dirs": extra_dirs,
- "type": self.store_type,
+ "type": store_type,
}
def _get_object_id(self, obj):
@@ -383,18 +470,45 @@ def get_concrete_store_name(self, obj):
def get_concrete_store_description_markdown(self, obj):
return self._invoke("get_concrete_store_description_markdown", obj)
+ def get_concrete_store_badges(self, obj) -> List[BadgeDict]:
+ return self._invoke("get_concrete_store_badges", obj)
+
def get_store_usage_percent(self):
return self._invoke("get_store_usage_percent")
def get_store_by(self, obj, **kwargs):
return self._invoke("get_store_by", obj, **kwargs)
+ def is_private(self, obj):
+ return self._invoke("is_private", obj)
+
+ @classmethod
+ def parse_private_from_config_xml(clazz, config_xml):
+ private = DEFAULT_PRIVATE
+ if config_xml is not None:
+ private = asbool(config_xml.attrib.get("private", DEFAULT_PRIVATE))
+ return private
+
+ @classmethod
+ def parse_badges_from_config_xml(clazz, badges_xml):
+ badges = []
+ for e in badges_xml:
+ type = e.tag
+ message = e.text
+ badges.append({"type": type, "message": message})
+ return badges
+
+ def get_quota_source_map(self):
+ # I'd rather keep this abstract... but register_singleton wants it to be instantiable...
+ raise NotImplementedError()
+
class ConcreteObjectStore(BaseObjectStore):
"""Subclass of ObjectStore for stores that don't delegate (non-nested).
- Currently only adds store_by functionality. Which doesn't make
- sense for the delegating object stores.
+ Adds store_by and quota_source functionality. These attributes do not make
+ sense for the delegating object stores, they should describe files at actually
+ persisted, not how a file is routed to a persistence source.
"""
def __init__(self, config, config_dict=None, **kwargs):
@@ -416,14 +530,95 @@ def __init__(self, config, config_dict=None, **kwargs):
self.store_by = config_dict.get("store_by", None) or getattr(config, "object_store_store_by", "id")
self.name = config_dict.get("name", None)
self.description = config_dict.get("description", None)
+ # Annotate this as true to prevent sharing of data.
+ self.private = config_dict.get("private", DEFAULT_PRIVATE)
+ # short label describing the quota source or null to use default
+ # quota source right on user object.
+ quota_config = config_dict.get("quota", {})
+ self.quota_source = quota_config.get("source", DEFAULT_QUOTA_SOURCE)
+ self.quota_enabled = quota_config.get("enabled", DEFAULT_QUOTA_ENABLED)
+ raw_badges = config_dict.get("badges", [])
+ badges = []
+ badge_types: Set[str] = set()
+ badge_conflicts: Dict[str, str] = {}
+ for badge in raw_badges:
+ # when recovering serialized badges, skip ones that are set by Galaxy
+ badge_source = badge.get("source")
+ if badge_source and badge_source != "admin":
+ continue
+ assert "type" in badge
+ badge_type = badge["type"]
+ if badge_type not in KNOWN_BADGE_TYPES:
+ raise Exception(f"badge_type {badge_type} unimplemented/unknown {badge}")
+ message = badge.get("message", None)
+ badges.append({"type": badge_type, "message": message})
+ badge_types.add(badge_type)
+ if badge_type in badge_conflicts:
+ conflicting_badge_type = badge_conflicts[badge_type]
+ raise Exception(
+ f"Conflicting badge to [{badge_type}] defined on the object store [{conflicting_badge_type}]."
+ )
+ conflicts = BADGE_SPECIFCATION_BY_TYPE[badge_type]["conflicts"]
+ for conflict in conflicts:
+ badge_conflicts[conflict] = badge_type
+ self.badges = badges
def to_dict(self):
rval = super().to_dict()
+ rval["private"] = self.private
rval["store_by"] = self.store_by
rval["name"] = self.name
rval["description"] = self.description
+ rval["quota"] = {
+ "source": self.quota_source,
+ "enabled": self.quota_enabled,
+ }
+ rval["badges"] = self._get_concrete_store_badges(None)
return rval
+ def to_model(self, object_store_id: str) -> "ConcreteObjectStoreModel":
+ return ConcreteObjectStoreModel(
+ object_store_id=object_store_id,
+ private=self.private,
+ name=self.name,
+ description=self.description,
+ quota=QuotaModel(source=self.quota_source, enabled=self.quota_enabled),
+ badges=self._get_concrete_store_badges(None),
+ )
+
+ def _get_concrete_store_badges(self, obj) -> List[BadgeDict]:
+ badge_dicts: List[BadgeDict] = []
+ for badge in self.badges:
+ badge_dict: BadgeDict = {
+ "source": "admin",
+ "type": badge["type"],
+ "message": badge["message"],
+ }
+ badge_dicts.append(badge_dict)
+
+ quota_badge_dict: BadgeDict
+ if self.galaxy_enable_quotas and self.quota_enabled:
+ quota_badge_dict = {
+ "type": "quota",
+ "message": None,
+ "source": "galaxy",
+ }
+ else:
+ quota_badge_dict = {
+ "type": "no_quota",
+ "message": None,
+ "source": "galaxy",
+ }
+ badge_dicts.append(quota_badge_dict)
+ if self.private:
+ restricted_badge_dict: BadgeDict = {
+ "type": "restricted",
+ "message": None,
+ "source": "galaxy",
+ }
+ badge_dicts.append(restricted_badge_dict)
+ return badge_dicts
+
def _get_concrete_store_name(self, obj):
return self.name
@@ -433,6 +628,16 @@ def _get_concrete_store_description_markdown(self, obj):
def _get_store_by(self, obj):
return self.store_by
+ def _is_private(self, obj):
+ return self.private
+
+ def get_quota_source_map(self):
+ quota_source_map = QuotaSourceMap(
+ self.quota_source,
+ self.quota_enabled,
+ )
+ return quota_source_map
+
class DiskObjectStore(ConcreteObjectStore):
"""
@@ -444,8 +649,8 @@ class DiskObjectStore(ConcreteObjectStore):
>>> import tempfile
>>> file_path=tempfile.mkdtemp()
>>> obj = Bunch(id=1)
- >>> s = DiskObjectStore(Bunch(umask=0o077, jobs_directory=file_path, new_file_path=file_path, object_store_check_old_style=False), dict(files_dir=file_path))
- >>> s.create(obj)
+ >>> s = DiskObjectStore(Bunch(umask=0o077, jobs_directory=file_path, new_file_path=file_path, object_store_check_old_style=False, enable_quotas=True), dict(files_dir=file_path))
+ >>> o = s.create(obj)
>>> s.exists(obj)
True
>>> assert s.get_filename(obj) == file_path + '/000/dataset_1.dat'
@@ -484,14 +689,22 @@ def parse_xml(clazz, config_xml):
if name is not None:
config_dict["name"] = name
for e in config_xml:
- if e.tag == "files_dir":
+ if e.tag == "quota":
+ config_dict["quota"] = {
+ "source": e.get("source", DEFAULT_QUOTA_SOURCE),
+ "enabled": asbool(e.get("enabled", DEFAULT_QUOTA_ENABLED)),
+ }
+ elif e.tag == "files_dir":
config_dict["files_dir"] = e.get("path")
elif e.tag == "description":
config_dict["description"] = e.text
+ elif e.tag == "badges":
+ config_dict["badges"] = BaseObjectStore.parse_badges_from_config_xml(e)
else:
extra_dirs.append({"type": e.get("type"), "path": e.get("path")})
config_dict["extra_dirs"] = extra_dirs
+ config_dict["private"] = BaseObjectStore.parse_private_from_config_xml(config_xml)
return config_dict
def to_dict(self):
@@ -572,7 +785,7 @@ def _construct_path(
hash id (e.g., /files/dataset_10.dat (old) vs.
/files/000/dataset_10.dat (new))
"""
- base = os.path.abspath(self.extra_dirs.get(base_dir, self.file_path))
+ base = os.path.abspath(self.extra_dirs.get(base_dir) or self.file_path)
# extra_dir should never be constructed from provided data but just
# make sure there are no shenannigans afoot
if extra_dir and extra_dir != os.path.normpath(extra_dir):
@@ -631,6 +844,7 @@ def _create(self, obj, **kwargs):
if not dir_only:
open(path, "w").close() # Should be rb?
umask_fix_perms(path, self.config.umask, 0o666)
+ return self
def _empty(self, obj, **kwargs):
"""Override `ObjectStore`'s stub by checking file size on disk."""
@@ -767,7 +981,8 @@ def file_ready(self, obj, **kwargs):
def _create(self, obj, **kwargs):
"""Create a backing file in a random backend."""
- random.choice(list(self.backends.values())).create(obj, **kwargs)
+ objectstore = random.choice(list(self.backends.values()))
+ return objectstore.create(obj, **kwargs)
def _empty(self, obj, **kwargs):
"""For the first backend that has this `obj`, determine if it is empty."""
@@ -806,6 +1021,12 @@ def _get_concrete_store_name(self, obj):
def _get_concrete_store_description_markdown(self, obj):
return self._call_method("_get_concrete_store_description_markdown", obj, None, False)
+ def _get_concrete_store_badges(self, obj) -> List[BadgeDict]:
+ return self._call_method("_get_concrete_store_badges", obj, [], False)
+
+ def _is_private(self, obj):
+ return self._call_method("_is_private", obj, ObjectNotFound, True)
+
def _get_store_by(self, obj):
return self._call_method("_get_store_by", obj, None, False)
@@ -859,6 +1080,7 @@ def __init__(self, config, config_dict, fsmon=False):
removing backends when they get too full.
"""
super().__init__(config, config_dict)
+ self._quota_source_map = None
self.backends = {}
self.weighted_backend_ids = []
@@ -868,11 +1090,14 @@ def __init__(self, config, config_dict, fsmon=False):
self.search_for_missing = config_dict.get("search_for_missing", True)
random.seed()
+ user_selection_allowed = []
for backend_def in config_dict["backends"]:
backened_id = backend_def["id"]
maxpctfull = backend_def.get("max_percent_full", 0)
weight = backend_def["weight"]
-
+ allow_selection = backend_def.get("allow_selection")
+ if allow_selection:
+ user_selection_allowed.append(backened_id)
backend = build_object_store_from_config(config, config_dict=backend_def, fsmon=fsmon)
self.backends[backened_id] = backend
@@ -885,7 +1110,8 @@ def __init__(self, config, config_dict, fsmon=False):
self.weighted_backend_ids.append(backened_id)
self.original_weighted_backend_ids = self.weighted_backend_ids
-
+ self.user_selection_allowed = user_selection_allowed
+ self.allow_user_selection = bool(user_selection_allowed)
self.sleeper = None
if fsmon and (self.global_max_percent_full or [_ for _ in self.max_percent_full.values() if _ != 0.0]):
self.sleeper = Sleeper()
@@ -914,6 +1140,7 @@ def parse_xml(clazz, config_xml, legacy=False):
store_maxpctfull = float(b.get("maxpctfull", 0))
store_type = b.get("type", "disk")
store_by = b.get("store_by", None)
+ allow_selection = asbool(b.get("allow_selection"))
objectstore_class, _ = type_to_object_store_class(store_type)
backend_config_dict = objectstore_class.parse_xml(b)
@@ -921,6 +1148,7 @@ def parse_xml(clazz, config_xml, legacy=False):
backend_config_dict["weight"] = store_weight
backend_config_dict["max_percent_full"] = store_maxpctfull
backend_config_dict["type"] = store_type
+ backend_config_dict["allow_selection"] = allow_selection
if store_by is not None:
backend_config_dict["store_by"] = store_by
backends.append(backend_config_dict)
@@ -980,23 +1208,27 @@ def __filesystem_monitor(self, sleeper: Sleeper):
def _create(self, obj, **kwargs):
"""The only method in which obj.object_store_id may be None."""
- if obj.object_store_id is None or not self._exists(obj, **kwargs):
- if obj.object_store_id is None or obj.object_store_id not in self.backends:
+ object_store_id = obj.object_store_id
+ if object_store_id is None or not self._exists(obj, **kwargs):
+ if object_store_id is None or object_store_id not in self.backends:
try:
- obj.object_store_id = random.choice(self.weighted_backend_ids)
+ object_store_id = random.choice(self.weighted_backend_ids)
+ obj.object_store_id = object_store_id
except IndexError:
raise ObjectInvalid(
f"objectstore.create, could not generate obj.object_store_id: {obj}, kwargs: {kwargs}"
)
log.debug(
- "Selected backend '%s' for creation of %s %s", obj.object_store_id, obj.__class__.__name__, obj.id
+ "Selected backend '%s' for creation of %s %s", object_store_id, obj.__class__.__name__, obj.id
)
else:
log.debug(
"Using preferred backend '%s' for creation of %s %s"
- % (obj.object_store_id, obj.__class__.__name__, obj.id)
+ % (object_store_id, obj.__class__.__name__, obj.id)
)
- self.backends[obj.object_store_id].create(obj, **kwargs)
+ return self.backends[object_store_id].create(obj, **kwargs)
+ else:
+ return self.backends[object_store_id]
def _call_method(self, method, obj, default, default_is_exception, **kwargs):
object_store_id = self.__get_store_id_for(obj, **kwargs)
@@ -1010,6 +1242,21 @@ def _call_method(self, method, obj, default, default_is_exception, **kwargs):
else:
return default
+ def get_quota_source_map(self):
+ if self._quota_source_map is None:
+ quota_source_map = QuotaSourceMap()
+ self._merge_quota_source_map(quota_source_map, self)
+ self._quota_source_map = quota_source_map
+ return self._quota_source_map
+
+ @classmethod
+ def _merge_quota_source_map(clz, quota_source_map, object_store):
+ for backend_id, backend in object_store.backends.items():
+ if isinstance(backend, DistributedObjectStore):
+ clz._merge_quota_source_map(quota_source_map, backend)
+ else:
+ quota_source_map.backends[backend_id] = backend.get_quota_source_map()
+
def __get_store_id_for(self, obj, **kwargs):
if obj.object_store_id is not None:
if obj.object_store_id in self.backends:
@@ -1032,9 +1279,28 @@ def __get_store_id_for(self, obj, **kwargs):
return id
return None
+ def object_store_ids(self, private=None):
+ object_store_ids = []
+ for backend_id, backend in self.backends.items():
+ object_store_ids.extend(backend.object_store_ids(private=private))
+ if backend.private is private or private is None:
+ object_store_ids.append(backend_id)
+ return object_store_ids
-class HierarchicalObjectStore(NestedObjectStore):
+ def object_store_allows_id_selection(self) -> bool:
+ """Return True if this object store respects object_store_id and allow selection of this."""
+ return self.allow_user_selection
+ def object_store_ids_allowing_selection(self) -> List[str]:
+ """Return a non-emtpy list of allowed selectable object store IDs during creation."""
+ return self.user_selection_allowed
+
+ def get_concrete_store_by_object_store_id(self, object_store_id: str) -> Optional["ConcreteObjectStore"]:
+ """If this is a distributed object store, get ConcreteObjectStore by object_store_id."""
+ return self.backends[object_store_id]
+
+
+class HierarchicalObjectStore(NestedObjectStore):
"""
ObjectStore that defers to a list of backends.
@@ -1049,22 +1315,43 @@ def __init__(self, config, config_dict, fsmon=False):
super().__init__(config, config_dict)
backends: Dict[int, ObjectStore] = {}
+ is_private = config_dict.get("private", DEFAULT_PRIVATE)
for order, backend_def in enumerate(config_dict["backends"]):
+ backend_is_private = backend_def.get("private")
+ if backend_is_private is not None:
+ assert (
+ is_private == backend_is_private
+ ), "The private attribute must be defined on the HierarchicalObjectStore and not contained concrete objectstores."
+ backend_quota = backend_def.get("quota")
+ if backend_quota is not None:
+ # Make sure just was using defaults - because cannot override what is
+ # is setup by the HierarchicalObjectStore.
+ assert backend_quota.get("source", DEFAULT_QUOTA_SOURCE) == DEFAULT_QUOTA_SOURCE
+ assert backend_quota.get("enabled", DEFAULT_QUOTA_ENABLED) == DEFAULT_QUOTA_ENABLED
+
backends[order] = build_object_store_from_config(config, config_dict=backend_def, fsmon=fsmon)
self.backends = backends
+ self.private = is_private
+ quota_config = config_dict.get("quota", {})
+ self.quota_source = quota_config.get("source", DEFAULT_QUOTA_SOURCE)
+ self.quota_enabled = quota_config.get("enabled", DEFAULT_QUOTA_ENABLED)
@classmethod
def parse_xml(clazz, config_xml):
backends_list = []
+ is_private = BaseObjectStore.parse_private_from_config_xml(config_xml)
for backend in sorted(config_xml.find("backends"), key=lambda b: int(b.get("order"))):
store_type = backend.get("type")
objectstore_class, _ = type_to_object_store_class(store_type)
backend_config_dict = objectstore_class.parse_xml(backend)
+ backend_config_dict["private"] = is_private
backend_config_dict["type"] = store_type
backends_list.append(backend_config_dict)
- return {"backends": backends_list}
+ config_dict = {"backends": backends_list}
+ config_dict["private"] = is_private
+ return config_dict
def to_dict(self):
as_dict = super().to_dict()
@@ -1073,6 +1360,7 @@ def to_dict(self):
backend_as_dict = backend.to_dict()
backends.append(backend_as_dict)
as_dict["backends"] = backends
+ as_dict["private"] = self.private
return as_dict
def _exists(self, obj, **kwargs):
@@ -1084,7 +1372,34 @@ def _exists(self, obj, **kwargs):
def _create(self, obj, **kwargs):
"""Call the primary object store."""
- self.backends[0].create(obj, **kwargs)
+ return self.backends[0].create(obj, **kwargs)
+
+ def _is_private(self, obj):
+ # Unlink the DistributedObjectStore - the HierarchicalObjectStore does not use
+ # object_store_id - so all the contained object stores need to define is_private
+ # the same way.
+ return self.private
+
+ def get_quota_source_map(self):
+ quota_source_map = QuotaSourceMap(
+ self.quota_source,
+ self.quota_enabled,
+ )
+ return quota_source_map
+
+
+class QuotaModel(BaseModel):
+ source: Optional[str]
+ enabled: bool
+
+
+class ConcreteObjectStoreModel(BaseModel):
+ object_store_id: Optional[str]
+ private: bool
+ name: Optional[str]
+ description: Optional[str]
+ quota: QuotaModel
+ badges: List[BadgeDict]
def type_to_object_store_class(store: str, fsmon: bool = False) -> Tuple[Type[BaseObjectStore], Dict[str, Any]]:
@@ -1227,6 +1542,7 @@ def config_to_dict(config):
"""Dict-ify the portion of a config object consumed by the ObjectStore class and its subclasses."""
return {
"object_store_check_old_style": config.object_store_check_old_style,
+ "enable_quotas": config.enable_quotas,
"file_path": config.file_path,
"umask": config.umask,
"jobs_directory": config.jobs_directory,
@@ -1236,6 +1552,66 @@ def config_to_dict(config):
}
+class QuotaSourceInfo(NamedTuple):
+ label: Optional[str]
+ use: bool
+
+
+class QuotaSourceMap:
+ def __init__(self, source=DEFAULT_QUOTA_SOURCE, enabled=DEFAULT_QUOTA_ENABLED):
+ self.default_quota_source = source
+ self.default_quota_enabled = enabled
+ self.info = QuotaSourceInfo(self.default_quota_source, self.default_quota_enabled)
+ self.backends = {}
+ self._labels = None
+
+ def get_quota_source_info(self, object_store_id):
+ if object_store_id in self.backends:
+ return self.backends[object_store_id].get_quota_source_info(object_store_id)
+ else:
+ return self.info
+
+ def get_quota_source_label(self, object_store_id):
+ if object_store_id in self.backends:
+ return self.backends[object_store_id].get_quota_source_label(object_store_id)
+ else:
+ return self.default_quota_source
+
+ def get_quota_source_labels(self):
+ if self._labels is None:
+ labels = set()
+ if self.default_quota_source:
+ labels.add(self.default_quota_source)
+ for backend in self.backends.values():
+ labels = labels.union(backend.get_quota_source_labels())
+ self._labels = labels
+ return self._labels
+
+ def default_usage_excluded_ids(self):
+ exclude_object_store_ids = []
+ for backend_id, backend_source_map in self.backends.items():
+ if backend_source_map.default_quota_source is not None:
+ exclude_object_store_ids.append(backend_id)
+ elif not backend_source_map.default_quota_enabled:
+ exclude_object_store_ids.append(backend_id)
+ return exclude_object_store_ids
+
+ def get_id_to_source_pairs(self):
+ pairs = []
+ for backend_id, backend_source_map in self.backends.items():
+ if backend_source_map.default_quota_source is not None and backend_source_map.default_quota_enabled:
+ pairs.append((backend_id, backend_source_map.default_quota_source))
+ return pairs
+
+ def ids_per_quota_source(self):
+ quota_sources: Dict[str, List[str]] = {}
+ for object_id, quota_source_label in self.get_id_to_source_pairs():
+ if quota_source_label not in quota_sources:
+ quota_sources[quota_source_label] = []
+ quota_sources[quota_source_label].append(object_id)
+ return quota_sources
+
+
class ObjectStorePopulator:
"""Small helper for interacting with the object store and making sure all
datasets from a job end up with the same object_store_id.
@@ -1250,16 +1626,19 @@ def __init__(self, has_object_store, user):
self.object_store_id = None
self.user = user
- def set_object_store_id(self, data):
- self.set_dataset_object_store_id(data.dataset)
+ def set_object_store_id(self, data, require_shareable=False):
+ self.set_dataset_object_store_id(data.dataset, require_shareable=require_shareable)
- def set_dataset_object_store_id(self, dataset):
+ def set_dataset_object_store_id(self, dataset, require_shareable=True):
# Create an empty file immediately. The first dataset will be
# created in the "default" store, all others will be created in
# the same store as the first.
dataset.object_store_id = self.object_store_id
try:
- self.object_store.create(dataset)
+ ensure_non_private = require_shareable
+ concrete_store = self.object_store.create(dataset, ensure_non_private=ensure_non_private)
+ if concrete_store.private and require_shareable:
+ raise Exception("Attempted to create shared output datasets in objectstore with sharing disabled")
except ObjectInvalid:
raise Exception("Unable to create output dataset: object store is full")
self.object_store_id = dataset.object_store_id # these will be the same thing after the first output
diff --git a/lib/galaxy/objectstore/azure_blob.py b/lib/galaxy/objectstore/azure_blob.py
index f90fc565458e..db8a047f553e 100644
--- a/lib/galaxy/objectstore/azure_blob.py
+++ b/lib/galaxy/objectstore/azure_blob.py
@@ -77,6 +77,7 @@ def parse_config_xml(config_xml):
"path": staging_path,
},
"extra_dirs": extra_dirs,
+ "private": ConcreteObjectStore.parse_private_from_config_xml(config_xml),
}
except Exception:
# Toss it back up after logging, we can't continue loading at this point.
diff --git a/lib/galaxy/objectstore/cloud.py b/lib/galaxy/objectstore/cloud.py
index 15f7ef987461..e4a9ed5bc14e 100644
--- a/lib/galaxy/objectstore/cloud.py
+++ b/lib/galaxy/objectstore/cloud.py
@@ -608,6 +608,7 @@ def _create(self, obj, **kwargs):
rel_path = os.path.join(rel_path, alt_name if alt_name else f"dataset_{self._get_object_id(obj)}.dat")
open(os.path.join(self.staging_path, rel_path), "w").close()
self._push_to_os(rel_path, from_string="")
+ return self
def _empty(self, obj, **kwargs):
if self._exists(obj, **kwargs):
diff --git a/lib/galaxy/objectstore/irods.py b/lib/galaxy/objectstore/irods.py
index cf0f8a7770d7..3dde6e991a1d 100644
--- a/lib/galaxy/objectstore/irods.py
+++ b/lib/galaxy/objectstore/irods.py
@@ -111,6 +111,7 @@ def parse_config_xml(config_xml):
"path": staging_path,
},
"extra_dirs": extra_dirs,
+ "private": DiskObjectStore.parse_private_from_config_xml(config_xml),
}
except Exception:
# Toss it back up after logging, we can't continue loading at this point.
@@ -599,6 +600,7 @@ def _create(self, obj, **kwargs):
open(os.path.join(self.staging_path, rel_path), "w").close()
self._push_to_irods(rel_path, from_string="")
log.debug("irods_pt _create: %s", ipt_timer)
+ return self
def _empty(self, obj, **kwargs):
if self._exists(obj, **kwargs):
diff --git a/lib/galaxy/objectstore/pithos.py b/lib/galaxy/objectstore/pithos.py
index f28baf6c31de..9eaa113bc887 100644
--- a/lib/galaxy/objectstore/pithos.py
+++ b/lib/galaxy/objectstore/pithos.py
@@ -77,6 +77,7 @@ def parse_config_xml(config_xml):
log.error(msg)
raise Exception(msg)
r["extra_dirs"] = [{k: e.get(k) for k in attrs} for e in extra_dirs]
+ r["private"] = ConcreteObjectStore.parse_private_from_config_xml(config_xml)
if "job_work" not in (d["type"] for d in r["extra_dirs"]):
msg = f'No value for {tag}:type="job_work" in XML tree'
log.error(msg)
@@ -297,6 +298,7 @@ def _create(self, obj, **kwargs):
new_file = os.path.join(self.staging_path, rel_path)
open(new_file, "w").close()
self.pithos.upload_from_string(rel_path, "")
+ return self
def _empty(self, obj, **kwargs):
"""
diff --git a/lib/galaxy/objectstore/s3.py b/lib/galaxy/objectstore/s3.py
index cca72815dc8d..52432b3bf6bf 100644
--- a/lib/galaxy/objectstore/s3.py
+++ b/lib/galaxy/objectstore/s3.py
@@ -105,6 +105,7 @@ def parse_config_xml(config_xml):
"path": staging_path,
},
"extra_dirs": extra_dirs,
+ "private": ConcreteObjectStore.parse_private_from_config_xml(config_xml),
}
except Exception:
# Toss it back up after logging, we can't continue loading at this point.
@@ -624,6 +625,7 @@ def _create(self, obj, **kwargs):
rel_path = os.path.join(rel_path, alt_name if alt_name else f"dataset_{self._get_object_id(obj)}.dat")
open(os.path.join(self.staging_path, rel_path), "w").close()
self._push_to_os(rel_path, from_string="")
+ return self
def _empty(self, obj, **kwargs):
if self._exists(obj, **kwargs):
diff --git a/lib/galaxy/objectstore/unittest_utils/__init__.py b/lib/galaxy/objectstore/unittest_utils/__init__.py
index 738a309264fa..2cc0239aef22 100644
--- a/lib/galaxy/objectstore/unittest_utils/__init__.py
+++ b/lib/galaxy/objectstore/unittest_utils/__init__.py
@@ -74,6 +74,7 @@ def __init__(self, temp_directory, config_file, store_by="id"):
self.new_file_path = temp_directory
self.umask = 0000
self.gid = 1000
+ self.enable_quotas = True
__all__ = [
diff --git a/lib/galaxy/quota/__init__.py b/lib/galaxy/quota/__init__.py
index 7c119f6bdc0c..5102f263cd95 100644
--- a/lib/galaxy/quota/__init__.py
+++ b/lib/galaxy/quota/__init__.py
@@ -1,5 +1,8 @@
"""Galaxy Quotas"""
import logging
+from typing import Optional
+
+from sqlalchemy.sql import text
import galaxy.util
@@ -21,12 +24,12 @@ class QuotaAgent: # metaclass=abc.ABCMeta
"""
# TODO: make abstractmethod after they work better with mypy
- def get_quota(self, user):
+ def get_quota(self, user, quota_source_label=None) -> Optional[int]:
"""Return quota in bytes or None if no quota is set."""
- def get_quota_nice_size(self, user):
+ def get_quota_nice_size(self, user, quota_source_label=None) -> Optional[str]:
"""Return quota as a human-readable string or 'unlimited' if no quota is set."""
- quota_bytes = self.get_quota(user)
+ quota_bytes = self.get_quota(user, quota_source_label=quota_source_label)
if quota_bytes is not None:
quota_str = galaxy.util.nice_size(quota_bytes)
else:
@@ -34,10 +37,12 @@ def get_quota_nice_size(self, user):
return quota_str
# TODO: make abstractmethod after they work better with mypy
- def get_percent(self, trans=None, user=False, history=False, usage=False, quota=False):
+ def get_percent(
+ self, trans=None, user=False, history=False, usage=False, quota=False, quota_source_label=None
+ ) -> Optional[int]:
"""Return the percentage of any storage quota applicable to the user/transaction."""
- def get_usage(self, trans=None, user=False, history=False):
+ def get_usage(self, trans=None, user=False, history=False, quota_source_label=None) -> Optional[float]:
if trans:
user = trans.user
history = trans.history
@@ -46,7 +51,14 @@ def get_usage(self, trans=None, user=False, history=False):
assert history, "Could not determine anonymous user's history."
usage = history.disk_size
else:
- usage = user.total_disk_usage
+ if quota_source_label is None:
+ usage = user.total_disk_usage
+ else:
+ quota_source_usage = user.quota_source_usage_for(quota_source_label)
+ if not quota_source_usage or quota_source_usage.disk_usage is None:
+ usage = 0.0
+ else:
+ usage = quota_source_usage.disk_usage
return usage
def is_over_quota(self, app, job, job_destination):
@@ -64,14 +76,16 @@ class NoQuotaAgent(QuotaAgent):
def __init__(self):
pass
- def get_quota(self, user):
+ def get_quota(self, user, quota_source_label=None) -> Optional[int]:
return None
@property
def default_quota(self):
return None
- def get_percent(self, trans=None, user=False, history=False, usage=False, quota=False):
+ def get_percent(
+ self, trans=None, user=False, history=False, usage=False, quota=False, quota_source_label=None
+ ) -> Optional[int]:
return None
def is_over_quota(self, app, job, job_destination):
@@ -85,7 +99,7 @@ def __init__(self, model):
self.model = model
self.sa_session = model.context
- def get_quota(self, user):
+ def get_quota(self, user, quota_source_label=None) -> Optional[int]:
"""
Calculated like so:
@@ -98,62 +112,90 @@ def get_quota(self, user):
quotas.
"""
if not user:
- return self.default_unregistered_quota
- quotas = []
- for group in [uga.group for uga in user.groups]:
- for quota in [gqa.quota for gqa in group.quotas]:
- if quota not in quotas:
- quotas.append(quota)
- for quota in [uqa.quota for uqa in user.quotas]:
- if quota not in quotas:
- quotas.append(quota)
- use_default = True
- max = 0
- adjustment = 0
- rval = 0
- for quota in quotas:
- if quota.deleted:
- continue
- if quota.operation == "=" and quota.bytes == -1:
- rval = None
- break
- elif quota.operation == "=":
- use_default = False
- if quota.bytes > max:
- max = quota.bytes
- elif quota.operation == "+":
- adjustment += quota.bytes
- elif quota.operation == "-":
- adjustment -= quota.bytes
- if use_default:
- max = self.default_registered_quota
- if max is None:
- rval = None
- if rval is not None:
- rval = max + adjustment
- if rval <= 0:
- rval = 0
- return rval
-
- @property
- def default_unregistered_quota(self):
- return self._default_quota(self.model.DefaultQuotaAssociation.types.UNREGISTERED)
-
- @property
- def default_registered_quota(self):
- return self._default_quota(self.model.DefaultQuotaAssociation.types.REGISTERED)
-
- def _default_quota(self, default_type):
- dqa = (
- self.sa_session.query(self.model.DefaultQuotaAssociation)
- .filter(self.model.DefaultQuotaAssociation.type == default_type)
- .first()
+ return self._default_unregistered_quota(quota_source_label)
+ query = text(
+ """
+SELECT (
+ COALESCE(MAX(CASE WHEN union_quota.operation = '='
+ THEN union_quota.bytes
+ ELSE NULL
+ END),
+ (SELECT default_quota.bytes
+ FROM quota as default_quota
+ LEFT JOIN default_quota_association on default_quota.id = default_quota_association.quota_id
+ WHERE default_quota_association.type = 'registered'
+ AND default_quota.deleted != :is_true
+ AND default_quota.quota_source_label {label_cond}))
+ +
+ (CASE WHEN SUM(CASE WHEN union_quota.operation = '=' AND union_quota.bytes = -1
+ THEN 1 ELSE 0
+ END) > 0
+ THEN NULL
+ ELSE 0 END)
+ +
+ (COALESCE(SUM(
+ CASE WHEN union_quota.operation = '+' THEN union_quota.bytes
+ WHEN union_quota.operation = '-' THEN -1 * union_quota.bytes
+ ELSE 0
+ END
+ ), 0))
+ )
+FROM (
+ SELECT user_quota.operation as operation, user_quota.bytes as bytes
+ FROM galaxy_user as guser
+ LEFT JOIN user_quota_association as uqa on guser.id = uqa.user_id
+ LEFT JOIN quota as user_quota on user_quota.id = uqa.quota_id
+ WHERE user_quota.deleted != :is_true
+ AND user_quota.quota_source_label {label_cond}
+ AND guser.id = :user_id
+ UNION ALL
+ SELECT group_quota.operation as operation, group_quota.bytes as bytes
+ FROM galaxy_user as guser
+ LEFT JOIN user_group_association as uga on guser.id = uga.user_id
+ LEFT JOIN galaxy_group on galaxy_group.id = uga.group_id
+ LEFT JOIN group_quota_association as gqa on galaxy_group.id = gqa.group_id
+ LEFT JOIN quota as group_quota on group_quota.id = gqa.quota_id
+ WHERE group_quota.deleted != :is_true
+ AND group_quota.quota_source_label {label_cond}
+ AND guser.id = :user_id
+) as union_quota
+""".format(
+ label_cond="IS NULL" if quota_source_label is None else " = :label"
+ )
)
- if not dqa:
- return None
- if dqa.quota.bytes < 0:
- return None
- return dqa.quota.bytes
+ conn = self.sa_session.connection()
+ with conn.begin():
+ res = conn.execute(query, is_true=True, user_id=user.id, label=quota_source_label).fetchone()
+ if res:
+ return int(res[0]) if res[0] else None
+ else:
+ return None
+
+ def _default_unregistered_quota(self, quota_source_label):
+ return self._default_quota(self.model.DefaultQuotaAssociation.types.UNREGISTERED, quota_source_label)
+
+ def _default_quota(self, default_type, quota_source_label):
+ label_condition = "IS NULL" if quota_source_label is None else "= :label"
+ query = text(
+ """
+SELECT bytes
+FROM quota as default_quota
+LEFT JOIN default_quota_association on default_quota.id = default_quota_association.quota_id
+WHERE default_quota_association.type = :default_type
+ AND default_quota.deleted != :is_true
+ AND default_quota.quota_source_label {label_condition}
+""".format(
+ label_condition=label_condition
+ )
+ )
+
+ conn = self.sa_session.connection()
+ with conn.begin():
+ res = conn.execute(query, is_true=True, label=quota_source_label, default_type=default_type).fetchone()
+ if res:
+ return res[0]
+ else:
+ return None
def set_default_quota(self, default_type, quota):
# Unset the current default(s) associated with this quota, if there are any
@@ -165,20 +207,27 @@ def set_default_quota(self, default_type, quota):
for gqa in quota.groups:
self.sa_session.delete(gqa)
# Find the old default, assign the new quota if it exists
- dqa = (
+ label = quota.quota_source_label
+ dqas = (
self.sa_session.query(self.model.DefaultQuotaAssociation)
- .filter(self.model.DefaultQuotaAssociation.type == default_type)
- .first()
+ .filter(self.model.DefaultQuotaAssociation.table.c.type == default_type)
+ .all()
)
- if dqa:
- dqa.quota = quota
+ target_default = None
+ for dqa in dqas:
+ if dqa.quota.quota_source_label == label and not dqa.quota.deleted:
+ target_default = dqa
+ if target_default:
+ target_default.quota = quota
# Or create if necessary
else:
- dqa = self.model.DefaultQuotaAssociation(default_type, quota)
- self.sa_session.add(dqa)
+ target_default = self.model.DefaultQuotaAssociation(default_type, quota)
+ self.sa_session.add(target_default)
self.sa_session.flush()
- def get_percent(self, trans=None, user=False, history=False, usage=False, quota=False):
+ def get_percent(
+ self, trans=None, user=False, history=False, usage=False, quota=False, quota_source_label=None
+ ) -> Optional[int]:
"""
Return the percentage of any storage quota applicable to the user/transaction.
"""
@@ -188,13 +237,13 @@ def get_percent(self, trans=None, user=False, history=False, usage=False, quota=
history = trans.history
# if quota wasn't passed, attempt to get the quota
if quota is False:
- quota = self.get_quota(user)
+ quota = self.get_quota(user, quota_source_label=quota_source_label)
# return none if no applicable quotas or quotas disabled
if quota is None:
return None
# get the usage, if it wasn't passed
if usage is False:
- usage = self.get_usage(trans, user, history)
+ usage = self.get_usage(trans, user, history, quota_source_label=quota_source_label)
try:
return min((int(float(usage) / quota * 100), 100))
except ZeroDivisionError:
@@ -224,10 +273,19 @@ def set_entity_quota_associations(self, quotas=None, users=None, groups=None, de
self.sa_session.flush()
def is_over_quota(self, app, job, job_destination):
- quota = self.get_quota(job.user)
+ # Doesn't work because job.object_store_id until inside handler :_(
+ # quota_source_label = job.quota_source_label
+ if job_destination is not None:
+ object_store_id = job_destination.params.get("object_store_id", None)
+ object_store = app.object_store
+ quota_source_map = object_store.get_quota_source_map()
+ quota_source_label = quota_source_map.get_quota_source_info(object_store_id).label
+ else:
+ quota_source_label = None
+ quota = self.get_quota(job.user, quota_source_label=quota_source_label)
if quota is not None:
try:
- usage = self.get_usage(user=job.user, history=job.history)
+ usage = self.get_usage(user=job.user, history=job.history, quota_source_label=quota_source_label)
if usage > quota:
return True
except AssertionError:
diff --git a/lib/galaxy/quota/_schema.py b/lib/galaxy/quota/_schema.py
index e2881570216d..b56f4e084877 100644
--- a/lib/galaxy/quota/_schema.py
+++ b/lib/galaxy/quota/_schema.py
@@ -107,6 +107,11 @@ class QuotaBase(Model):
description="The `encoded identifier` of the quota.",
)
name: str = QuotaNameField
+ quota_source_label: Optional[str] = Field(
+ None,
+ title="Quota Source Label",
+ description="Quota source label",
+ )
class QuotaSummary(QuotaBase):
@@ -183,6 +188,11 @@ class CreateQuotaParams(Model):
" equivalent to ``no``."
),
)
+ quota_source_label: Optional[str] = Field(
+ default=None,
+ title="Quota Source Label",
+ description="If set, quota source label to apply this quota operation to. Otherwise, the default quota is used.",
+ )
in_users: Optional[List[str]] = Field(
default=[],
title="Users",
diff --git a/lib/galaxy/security/validate_user_input.py b/lib/galaxy/security/validate_user_input.py
index f3bd0a680a80..86d890cdf3cb 100644
--- a/lib/galaxy/security/validate_user_input.py
+++ b/lib/galaxy/security/validate_user_input.py
@@ -6,12 +6,15 @@
"""
import logging
import re
+from typing import Optional
import dns.resolver
from dns.exception import DNSException
from sqlalchemy import func
from typing_extensions import LiteralString
+from galaxy.objectstore import ObjectStore
+
log = logging.getLogger(__name__)
# Email validity parameters
@@ -155,3 +158,12 @@ def validate_password(trans, password, confirm):
if password != confirm:
return "Passwords do not match."
return validate_password_str(password)
+
+
+def validate_preferred_object_store_id(object_store: ObjectStore, preferred_object_store_id: Optional[str]) -> str:
+ if not object_store.object_store_allows_id_selection() and preferred_object_store_id is not None:
+ return "The current configuration doesn't allow selecting preferred object stores."
+ if object_store.object_store_allows_id_selection() and preferred_object_store_id:
+ if preferred_object_store_id not in object_store.object_store_ids_allowing_selection():
+ return "Supplied object store id is not an allowed object store selection"
+ return ""
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 64f7977a1c63..6340f7269df2 100644
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -1936,7 +1936,15 @@ def expand_incoming(self, trans, incoming, request_context, input_format="legacy
log.info(validation_timer)
return all_params, all_errors, rerun_remap_job_id, collection_info
- def handle_input(self, trans, incoming, history=None, use_cached_job=False, input_format="legacy"):
+ def handle_input(
+ self,
+ trans,
+ incoming,
+ history=None,
+ use_cached_job=False,
+ preferred_object_store_id: Optional[str] = None,
+ input_format="legacy",
+ ):
"""
Process incoming parameters for this tool from the dict `incoming`,
update the tool state (or create if none existed), and either return
@@ -1970,6 +1978,7 @@ def handle_input(self, trans, incoming, history=None, use_cached_job=False, inpu
mapping_params,
history=request_context.history,
rerun_remap_job_id=rerun_remap_job_id,
+ preferred_object_store_id=preferred_object_store_id,
collection_info=collection_info,
completed_jobs=completed_jobs,
)
@@ -2017,6 +2026,7 @@ def handle_single_execution(
completed_job=None,
collection_info=None,
job_callback=None,
+ preferred_object_store_id=None,
flush_job=True,
skip=False,
):
@@ -2035,6 +2045,7 @@ def handle_single_execution(
completed_job=completed_job,
collection_info=collection_info,
job_callback=job_callback,
+ preferred_object_store_id=preferred_object_store_id,
flush_job=flush_job,
skip=skip,
)
diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py
index cef92a0aba8e..f19050731f7d 100644
--- a/lib/galaxy/tools/actions/__init__.py
+++ b/lib/galaxy/tools/actions/__init__.py
@@ -367,6 +367,7 @@ def execute(
completed_job=None,
collection_info=None,
job_callback=None,
+ preferred_object_store_id=None,
flush_job=True,
skip=False,
):
@@ -651,6 +652,7 @@ def handle_output(name, output, hidden=None):
data.state = "ok"
with open(data.dataset.file_name, "w") as out:
out.write(json.dumps(None))
+ job.preferred_object_store_id = preferred_object_store_id
self._record_inputs(trans, tool, job, incoming, inp_data, inp_dataset_collections)
self._record_outputs(job, out_data, output_collections)
# execute immediate post job actions and associate post job actions that are to be executed after the job is complete
diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py
index 412e83d0b548..cee7aef04fc2 100644
--- a/lib/galaxy/tools/actions/upload_common.py
+++ b/lib/galaxy/tools/actions/upload_common.py
@@ -140,7 +140,7 @@ def __new_history_upload(trans, uploaded_dataset, history=None, state=None):
trans.sa_session.flush()
history.add_dataset(hda, genome_build=uploaded_dataset.dbkey, quota=False)
permissions = trans.app.security_agent.history_get_default_permissions(history)
- trans.app.security_agent.set_all_dataset_permissions(hda.dataset, permissions)
+ trans.app.security_agent.set_all_dataset_permissions(hda.dataset, permissions, new=True, flush=False)
trans.sa_session.flush()
return hda
@@ -211,7 +211,7 @@ def __new_library_upload(trans, cntrller, uploaded_dataset, library_bunch, tag_h
else:
# Copy the current user's DefaultUserPermissions to the new LibraryDatasetDatasetAssociation.dataset
trans.app.security_agent.set_all_dataset_permissions(
- ldda.dataset, trans.app.security_agent.user_get_default_permissions(trans.user)
+ ldda.dataset, trans.app.security_agent.user_get_default_permissions(trans.user), new=True
)
folder.add_library_dataset(ld, genome_build=uploaded_dataset.dbkey)
trans.sa_session.add(folder)
diff --git a/lib/galaxy/tools/execute.py b/lib/galaxy/tools/execute.py
index a668588e71c2..9cfb9a3831a0 100644
--- a/lib/galaxy/tools/execute.py
+++ b/lib/galaxy/tools/execute.py
@@ -57,6 +57,7 @@ def execute(
mapping_params: MappingParameters,
history: model.History,
rerun_remap_job_id: Optional[int] = None,
+ preferred_object_store_id: Optional[str] = None,
collection_info: Optional[MatchingCollections] = None,
workflow_invocation_uuid: Optional[str] = None,
invocation_step: Optional[model.WorkflowInvocationStep] = None,
@@ -120,6 +121,7 @@ def execute_single_job(execution_slice, completed_job, skip=False):
completed_job,
collection_info,
job_callback=job_callback,
+ preferred_object_store_id=preferred_object_store_id,
flush_job=False,
skip=skip,
)
diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py
index 45df16c83afe..1cb7f6525942 100644
--- a/lib/galaxy/webapps/base/webapp.py
+++ b/lib/galaxy/webapps/base/webapp.py
@@ -825,7 +825,7 @@ def _associate_user_history(self, user, prev_galaxy_session=None):
# Increase the user's disk usage by the amount of the previous history's datasets if they didn't already
# own it.
for hda in history.datasets:
- user.adjust_total_disk_usage(hda.quota_amount(user))
+ user.adjust_total_disk_usage(hda.quota_amount(user), hda.dataset.quota_source_info.label)
# Only set default history permissions if the history is from the previous session and anonymous
set_permissions = True
elif self.galaxy_session.current_history:
diff --git a/lib/galaxy/webapps/galaxy/api/history_contents.py b/lib/galaxy/webapps/galaxy/api/history_contents.py
index add89f3c1981..472cb3e47cc7 100644
--- a/lib/galaxy/webapps/galaxy/api/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/api/history_contents.py
@@ -198,6 +198,11 @@ def get_legacy_index_query_params(
description="Whether to return visible or hidden datasets only. Leave unset for both.",
deprecated=True,
),
+ shareable: Optional[bool] = Query(
+ default=None,
+ title="Shareable",
+ description="Whether to return only shareable or not shareable datasets. Leave unset for both.",
+ ),
) -> LegacyHistoryContentsIndexParams:
"""This function is meant to be used as a dependency to render the OpenAPI documentation
correctly"""
@@ -207,6 +212,7 @@ def get_legacy_index_query_params(
details=details,
deleted=deleted,
visible=visible,
+ shareable=shareable,
)
@@ -216,6 +222,7 @@ def parse_legacy_index_query_params(
details: Optional[str] = None,
deleted: Optional[bool] = None,
visible: Optional[bool] = None,
+ shareable: Optional[bool] = None,
**_, # Additional params are ignored
) -> LegacyHistoryContentsIndexParams:
"""Parses (legacy) query parameters for the history contents `index` operation
@@ -242,6 +249,7 @@ def parse_legacy_index_query_params(
ids=id_list,
deleted=deleted,
visible=visible,
+ shareable=shareable,
dataset_details=dataset_details,
)
except ValidationError as e:
diff --git a/lib/galaxy/webapps/galaxy/api/object_store.py b/lib/galaxy/webapps/galaxy/api/object_store.py
new file mode 100644
index 000000000000..3e3a81442d04
--- /dev/null
+++ b/lib/galaxy/webapps/galaxy/api/object_store.py
@@ -0,0 +1,78 @@
+"""
+API operations on Galaxy's object store.
+"""
+import logging
+from typing import List
+
+from fastapi import (
+ Path,
+ Query,
+)
+
+from galaxy.exceptions import (
+ ObjectNotFound,
+ RequestParameterInvalidException,
+)
+from galaxy.managers.context import ProvidesUserContext
+from galaxy.objectstore import (
+ BaseObjectStore,
+ ConcreteObjectStoreModel,
+)
+from . import (
+ depends,
+ DependsOnTrans,
+ Router,
+)
+
+log = logging.getLogger(__name__)
+
+router = Router(tags=["object sstore"])
+
+ConcreteObjectStoreIdPathParam: str = Path(
+ ..., title="Concrete Object Store ID", description="The concrete object store ID."
+)
+
+SelectableQueryParam: bool = Query(
+ default=False,
+ title="Selectable",
+ description="Restrict index query to user selectable object stores, the current implementation requires this to be true.",
+)
+
+
+@router.cbv
+class FastAPIObjectStore:
+ object_store: BaseObjectStore = depends(BaseObjectStore)
+
+ @router.get(
+ "/api/object_store",
+ summary="Get a list of (currently only concrete) object stores configured with this Galaxy instance.",
+ response_description="A list of the configured object stores.",
+ )
+ def index(
+ self,
+ trans: ProvidesUserContext = DependsOnTrans,
+ selectable: bool = SelectableQueryParam,
+ ) -> List[ConcreteObjectStoreModel]:
+ if not selectable:
+ raise RequestParameterInvalidException(
+ "The object store index query currently needs to be called with selectable=true"
+ )
+ selectable_ids = self.object_store.object_store_ids_allowing_selection()
+ return [self._model_for(selectable_id) for selectable_id in selectable_ids]
+
+ @router.get(
+ "/api/object_store/{object_store_id}",
+ summary="Get information about a concrete object store configured with Galaxy.",
+ )
+ def show_info(
+ self,
+ trans: ProvidesUserContext = DependsOnTrans,
+ object_store_id: str = ConcreteObjectStoreIdPathParam,
+ ) -> ConcreteObjectStoreModel:
+ return self._model_for(object_store_id)
+
+ def _model_for(self, object_store_id: str) -> ConcreteObjectStoreModel:
+ concrete_object_store = self.object_store.get_concrete_store_by_object_store_id(object_store_id)
+ if concrete_object_store is None:
+ raise ObjectNotFound()
+ return concrete_object_store.to_model(object_store_id)
diff --git a/lib/galaxy/webapps/galaxy/api/users.py b/lib/galaxy/webapps/galaxy/api/users.py
index bd960968df24..82a1a75e5561 100644
--- a/lib/galaxy/webapps/galaxy/api/users.py
+++ b/lib/galaxy/webapps/galaxy/api/users.py
@@ -5,6 +5,10 @@
import json
import logging
import re
+from typing import (
+ List,
+ Optional,
+)
from fastapi import (
Body,
@@ -33,6 +37,7 @@
from galaxy.model import (
User,
UserAddress,
+ UserQuotaUsage,
)
from galaxy.schema import APIKeyModel
from galaxy.schema.fields import DecodedDatabaseIdField
@@ -73,11 +78,18 @@
UserIdPathParam: DecodedDatabaseIdField = Path(..., title="User ID", description="The ID of the user to get.")
APIKeyPathParam: str = Path(..., title="API Key", description="The API key of the user.")
+FlexibleUserIdPathParam: str = Path(..., title="User ID", description="The ID of the user to get or __current__.")
+QuotaSourceLabelPathParam: str = Path(
+ ...,
+ title="Quota Source Label",
+ description="The label corresponding to the quota source to fetch usage information about.",
+)
@router.cbv
class FastAPIUsers:
service: UsersService = depends(UsersService)
+ user_serializer: users.UserSerializer = depends(users.UserSerializer)
@router.put(
"/api/users/recalculate_disk_usage",
@@ -140,6 +152,44 @@ def delete_api_key(
self.service.delete_api_key(trans, user_id)
return Response(status_code=status.HTTP_204_NO_CONTENT)
+ @router.get(
+ "/api/users/{user_id}/usage",
+ name="get_user_usage",
+ summary="Return the user's quota usage summary broken down by quota source",
+ )
+ def usage(
+ self,
+ trans: ProvidesUserContext = DependsOnTrans,
+ user_id: str = FlexibleUserIdPathParam,
+ ) -> List[UserQuotaUsage]:
+ user = get_user_full(trans, user_id, False)
+ if user:
+ rval = self.user_serializer.serialize_disk_usage(user)
+ return rval
+ else:
+ return []
+
+ @router.get(
+ "/api/users/{user_id}/usage/{label}",
+ name="get_user_usage_for_label",
+ summary="Return the user's quota usage summary for a given quota source label",
+ )
+ def usage_for(
+ self,
+ trans: ProvidesUserContext = DependsOnTrans,
+ user_id: str = FlexibleUserIdPathParam,
+ label: str = QuotaSourceLabelPathParam,
+ ) -> Optional[UserQuotaUsage]:
+ user = get_user_full(trans, user_id, False)
+ effective_label: Optional[str] = label
+ if label == "__null__":
+ effective_label = None
+ if user:
+ rval = self.user_serializer.serialize_disk_usage_for(user, effective_label)
+ return rval
+ else:
+ return None
+
@router.get(
"/api/users/{user_id}/beacon",
summary="Returns information about beacon share settings",
@@ -286,34 +336,7 @@ def _get_user_full(self, trans, user_id, **kwd):
"""Return referenced user or None if anonymous user is referenced."""
deleted = kwd.get("deleted", "False")
deleted = util.string_as_bool(deleted)
- try:
- # user is requesting data about themselves
- if user_id == "current":
- # ...and is anonymous - return usage and quota (if any)
- if not trans.user:
- return None
-
- # ...and is logged in - return full
- else:
- user = trans.user
- else:
- return managers_base.get_object(
- trans,
- user_id,
- "User",
- deleted=deleted,
- )
- # check that the user is requesting themselves (and they aren't del'd) unless admin
- if not trans.user_is_admin:
- if trans.user != user or user.deleted:
- raise exceptions.InsufficientPermissionsException(
- "You are not allowed to perform action on that user", id=user_id
- )
- return user
- except exceptions.MessageException:
- raise
- except Exception:
- raise exceptions.RequestParameterInvalidException("Invalid user id specified", id=user_id)
+ return get_user_full(trans, user_id, deleted)
@expose_api
def create(self, trans: GalaxyWebTransaction, payload: dict, **kwd):
@@ -414,7 +437,7 @@ def anon_user_api_value(self, trans):
if not trans.user and not trans.history:
# Can't return info about this user, may not have a history yet.
return {}
- usage = trans.app.quota_agent.get_usage(trans)
+ usage = trans.app.quota_agent.get_usage(trans, history=trans.history)
percent = trans.app.quota_agent.get_percent(trans=trans, usage=usage)
return {
"total_disk_usage": int(usage),
@@ -1121,3 +1144,34 @@ def _get_user(self, trans, id):
if user != trans.user and not trans.user_is_admin:
raise exceptions.InsufficientPermissionsException("Access denied.")
return user
+
+
+def get_user_full(trans: ProvidesUserContext, user_id: str, deleted: bool) -> Optional[User]:
+ try:
+ # user is requesting data about themselves
+ if user_id == "current":
+ # ...and is anonymous - return usage and quota (if any)
+ if not trans.user:
+ return None
+
+ # ...and is logged in - return full
+ else:
+ user = trans.user
+ else:
+ return managers_base.get_object(
+ trans,
+ user_id,
+ "User",
+ deleted=deleted,
+ )
+ # check that the user is requesting themselves (and they aren't del'd) unless admin
+ if not trans.user_is_admin:
+ if trans.user != user or user.deleted:
+ raise exceptions.InsufficientPermissionsException(
+ "You are not allowed to perform action on that user", id=user_id
+ )
+ return user
+ except exceptions.MessageException:
+ raise
+ except Exception:
+ raise exceptions.RequestParameterInvalidException("Invalid user id specified", id=user_id)
diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py
index 43257341d665..542b2cc6c531 100644
--- a/lib/galaxy/webapps/galaxy/controllers/admin.py
+++ b/lib/galaxy/webapps/galaxy/controllers/admin.py
@@ -698,6 +698,9 @@ def create_quota(self, trans, payload=None, **kwd):
if trans.request.method == "GET":
all_users = []
all_groups = []
+ labels = trans.app.object_store.get_quota_source_map().get_quota_source_labels()
+ label_options = [("Default Quota", None)]
+ label_options.extend([(label, label) for label in labels])
for user in (
trans.sa_session.query(trans.app.model.User)
.filter(trans.app.model.User.table.c.deleted == false())
@@ -713,7 +716,7 @@ def create_quota(self, trans, payload=None, **kwd):
default_options = [("No", "no")]
for type_ in trans.app.model.DefaultQuotaAssociation.types:
default_options.append((f"Yes, {type_}", type_))
- return {
+ rval = {
"title": "Create Quota",
"inputs": [
{"name": "name", "label": "Name"},
@@ -730,10 +733,23 @@ def create_quota(self, trans, payload=None, **kwd):
"options": default_options,
"help": "Warning: Any users or groups associated with this quota will be disassociated.",
},
- build_select_input("in_groups", "Groups", all_groups, []),
- build_select_input("in_users", "Users", all_users, []),
],
}
+ if len(label_options) > 1:
+ rval["inputs"].append(
+ {
+ "name": "quota_source_label",
+ "label": "Apply quota to labeled object stores.",
+ "options": label_options,
+ }
+ )
+ rval["inputs"].extend(
+ [
+ build_select_input("in_groups", "Groups", all_groups, []),
+ build_select_input("in_users", "Users", all_users, []),
+ ]
+ )
+ return rval
else:
try:
quota, message = self.quota_manager.create_quota(payload, decode_id=trans.security.decode_id)
diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py
index fe1578907b93..e75ba2e1dab9 100644
--- a/lib/galaxy/webapps/galaxy/controllers/dataset.py
+++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py
@@ -327,7 +327,12 @@ def get_edit(self, trans, dataset_id=None, **kwd):
permission_disable = True
permission_inputs = list()
if trans.user:
- if data.dataset.actions:
+ if not data.dataset.shareable:
+ permission_message = "The dataset is stored on private storage to you and cannot be shared."
+ permission_inputs.append(
+ {"name": "not_shareable", "type": "hidden", "label": permission_message, "readonly": True}
+ )
+ elif data.dataset.actions:
in_roles = {}
for action, roles in trans.app.security_agent.get_permissions(data.dataset).items():
in_roles[action.action] = [trans.security.encode_id(role.id) for role in roles]
@@ -882,7 +887,7 @@ def _purge(self, trans, dataset_id):
hda.deleted = True
# HDA is purgeable
# Decrease disk usage first
- hda.purge_usage_from_quota(user)
+ hda.purge_usage_from_quota(user, hda.dataset.quota_source_info)
# Mark purged
hda.purged = True
trans.sa_session.add(hda)
diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py
index 5ef2faf41870..041c53a7573b 100644
--- a/lib/galaxy/webapps/galaxy/controllers/history.py
+++ b/lib/galaxy/webapps/galaxy/controllers/history.py
@@ -609,7 +609,7 @@ def purge_deleted_datasets(self, trans):
for hda in trans.history.datasets:
if not hda.deleted or hda.purged:
continue
- hda.purge_usage_from_quota(trans.user)
+ hda.purge_usage_from_quota(trans.user, hda.dataset.quota_source_info)
hda.purged = True
trans.sa_session.add(hda)
trans.log_event(f"HDA id {hda.id} has been purged")
diff --git a/lib/galaxy/webapps/galaxy/services/datasets.py b/lib/galaxy/webapps/galaxy/services/datasets.py
index 201820078164..e6a903253ab0 100644
--- a/lib/galaxy/webapps/galaxy/services/datasets.py
+++ b/lib/galaxy/webapps/galaxy/services/datasets.py
@@ -98,6 +98,15 @@ class RequestDataType(str, Enum):
in_use_state = "in_use_state"
+class ConcreteObjectStoreQuotaSourceDetails(Model):
+ source: Optional[str] = Field(
+ description="The quota source label corresponding to the object store the dataset is stored in (or would be stored in)"
+ )
+ enabled: bool = Field(
+ description="Whether the object store tracks quota on the data (independent of Galaxy's configuration)"
+ )
+
+
class DatasetStorageDetails(Model):
object_store_id: Optional[str] = Field(
description="The identifier of the destination ObjectStore for this dataset.",
@@ -116,6 +125,13 @@ class DatasetStorageDetails(Model):
)
hashes: List[dict] = Field(description="The file contents hashes associated with the supplied dataset instance.")
sources: List[dict] = Field(description="The file sources associated with the supplied dataset instance.")
+ shareable: bool = Field(
+ description="Is this dataset shareable.",
+ )
+ quota: dict = Field(description="Information about quota sources around dataset storage.")
+ badges: List[Dict[str, Any]] = Field(
+ description="A mapping of object store labels to badges describing object store properties."
+ )
class DatasetInheritanceChainEntry(Model):
@@ -364,6 +380,7 @@ def show_storage(
object_store_id = dataset.object_store_id
name = object_store.get_concrete_store_name(dataset)
description = object_store.get_concrete_store_description_markdown(dataset)
+ badges = object_store.get_concrete_store_badges(dataset)
# not really working (existing problem)
try:
percent_used = object_store.get_store_usage_percent()
@@ -373,17 +390,27 @@ def show_storage(
except FileNotFoundError:
# uninitalized directory (emtpy) disk object store can cause this...
percent_used = None
+
+ quota_source = dataset.quota_source_info
+ quota = ConcreteObjectStoreQuotaSourceDetails(
+ source=quota_source.label,
+ enabled=quota_source.use,
+ )
+
dataset_state = dataset.state
hashes = [h.to_dict() for h in dataset.hashes]
sources = [s.to_dict() for s in dataset.sources]
return DatasetStorageDetails(
object_store_id=object_store_id,
+ shareable=dataset.shareable,
name=name,
description=description,
percent_used=percent_used,
dataset_state=dataset_state,
hashes=hashes,
sources=sources,
+ quota=quota,
+ badges=badges,
)
def show_inheritance_chain(
diff --git a/lib/galaxy/webapps/galaxy/services/history_contents.py b/lib/galaxy/webapps/galaxy/services/history_contents.py
index c388bae518cf..8c6952cda05c 100644
--- a/lib/galaxy/webapps/galaxy/services/history_contents.py
+++ b/lib/galaxy/webapps/galaxy/services/history_contents.py
@@ -61,6 +61,7 @@
User,
)
from galaxy.model.security import GalaxyRBACAgent
+from galaxy.objectstore import BaseObjectStore
from galaxy.schema import (
FilterQueryParams,
SerializationParams,
@@ -142,6 +143,11 @@ class LegacyHistoryContentsIndexParams(Model):
dataset_details: Optional[DatasetDetailsType]
deleted: Optional[bool]
visible: Optional[bool]
+ shareable: Optional[bool] = Field(
+ default=None,
+ title="Sharable",
+ description="Whether to return only shareable or not shareable datasets. Leave unset for both.",
+ )
class HistoryContentsIndexJobsSummaryParams(Model):
@@ -252,6 +258,7 @@ class HistoriesContentsService(ServiceBase, ServesExportStores, ConsumesModelSto
def __init__(
self,
security: IdEncodingHelper,
+ object_store: BaseObjectStore,
history_manager: histories.HistoryManager,
history_contents_manager: HistoryContentsManager,
hda_manager: hdas.HDAManager,
@@ -281,6 +288,7 @@ def __init__(
self.item_operator = HistoryItemOperator(self.hda_manager, self.hdca_manager, self.dataset_collection_manager)
self.short_term_storage_allocator = short_term_storage_allocator
self.genomes_manager = genomes_manager
+ self.object_store = object_store
def index(
self,
@@ -918,6 +926,13 @@ def __index_legacy(
ids = legacy_params_dict.get("ids")
if ids:
legacy_params_dict["ids"] = self.decode_ids(ids)
+
+ object_store_ids = None
+ shareable = legacy_params.shareable
+ if shareable is not None:
+ object_store_ids = self.object_store.object_store_ids(private=not shareable)
+ if object_store_ids:
+ legacy_params_dict["object_store_ids"] = object_store_ids
contents = history.contents_iter(**legacy_params_dict)
items = [
self._serialize_legacy_content_item(trans, content, legacy_params_dict.get("dataset_details"))
diff --git a/lib/galaxy/webapps/galaxy/services/tools.py b/lib/galaxy/webapps/galaxy/services/tools.py
index a95e6357ab7b..80d699a3b454 100644
--- a/lib/galaxy/webapps/galaxy/services/tools.py
+++ b/lib/galaxy/webapps/galaxy/services/tools.py
@@ -161,12 +161,17 @@ def _create(self, trans: ProvidesHistoryContext, payload, **kwd):
use_cached_job = payload.get("use_cached_job", False) or util.string_as_bool(
inputs.get("use_cached_job", "false")
)
-
+ preferred_object_store_id = payload.get("preferred_object_store_id")
input_format = str(payload.get("input_format", "legacy"))
if "data_manager_mode" in payload:
incoming["__data_manager_mode"] = payload["data_manager_mode"]
vars = tool.handle_input(
- trans, incoming, history=target_history, use_cached_job=use_cached_job, input_format=input_format
+ trans,
+ incoming,
+ history=target_history,
+ use_cached_job=use_cached_job,
+ input_format=input_format,
+ preferred_object_store_id=preferred_object_store_id,
)
new_pja_flush = False
diff --git a/lib/galaxy/workflow/run_request.py b/lib/galaxy/workflow/run_request.py
index 65146353e7f6..5e23cdd61fc6 100644
--- a/lib/galaxy/workflow/run_request.py
+++ b/lib/galaxy/workflow/run_request.py
@@ -11,6 +11,7 @@
from galaxy import exceptions
from galaxy.model import (
+ EffectiveOutput,
History,
HistoryDatasetAssociation,
LibraryDataset,
@@ -72,6 +73,10 @@ def __init__(
copy_inputs_to_history: bool = False,
use_cached_job: bool = False,
resource_params: Optional[Dict[int, Any]] = None,
+ preferred_object_store_id: Optional[str] = None,
+ preferred_outputs_object_store_id: Optional[str] = None,
+ preferred_intermediate_object_store_id: Optional[str] = None,
+ effective_outputs: Optional[List[EffectiveOutput]] = None,
) -> None:
self.target_history = target_history
self.replacement_dict = replacement_dict or {}
@@ -81,6 +86,10 @@ def __init__(
self.resource_params = resource_params or {}
self.allow_tool_state_corrections = allow_tool_state_corrections
self.use_cached_job = use_cached_job
+ self.preferred_object_store_id = preferred_object_store_id
+ self.preferred_outputs_object_store_id = preferred_outputs_object_store_id
+ self.preferred_intermediate_object_store_id = preferred_intermediate_object_store_id
+ self.effective_outputs = effective_outputs
def _normalize_inputs(
@@ -431,6 +440,20 @@ def build_workflow_run_configs(
f"Invalid value for parameter '{name}' found."
)
history.add_pending_items()
+ preferred_object_store_id = payload.get("preferred_object_store_id")
+ preferred_outputs_object_store_id = payload.get("preferred_outputs_object_store_id")
+ preferred_intermediate_object_store_id = payload.get("preferred_intermediate_object_store_id")
+ if payload.get("effective_outputs"):
+ raise exceptions.RequestParameterInvalidException(
+ "Cannot declare effective outputs on invocation in this fashion."
+ )
+ split_object_store_config = bool(
+ preferred_outputs_object_store_id is not None or preferred_intermediate_object_store_id is not None
+ )
+ if split_object_store_config and preferred_object_store_id:
+ raise exceptions.RequestParameterInvalidException(
+ "May specified either 'preferred_object_store_id' or one/both of 'preferred_outputs_object_store_id' and 'preferred_intermediate_object_store_id' but not both"
+ )
run_configs.append(
WorkflowRunConfig(
target_history=history,
@@ -440,6 +463,9 @@ def build_workflow_run_configs(
allow_tool_state_corrections=allow_tool_state_corrections,
use_cached_job=use_cached_job,
resource_params=resource_params,
+ preferred_object_store_id=preferred_object_store_id,
+ preferred_outputs_object_store_id=preferred_outputs_object_store_id,
+ preferred_intermediate_object_store_id=preferred_intermediate_object_store_id,
)
)
@@ -476,6 +502,20 @@ def add_parameter(name: str, value: str, type: WorkflowRequestInputParameter.typ
workflow_invocation.step_states.append(step_state)
if step.type == "subworkflow":
+ step.workflow_outputs
+ assert step.subworkflow
+ subworkflow: Workflow = step.subworkflow
+ effective_outputs: Optional[List[EffectiveOutput]] = None
+ if run_config.preferred_intermediate_object_store_id or run_config.preferred_outputs_object_store_id:
+ step_outputs = step.workflow_outputs
+ effective_outputs = []
+ for step_output in step_outputs:
+ subworkflow_output = subworkflow.workflow_output_for(step_output.output_name)
+ if subworkflow_output is not None:
+ output_dict = EffectiveOutput(
+ output_name=subworkflow_output.output_name, step_id=subworkflow_output.workflow_step_id
+ )
+ effective_outputs.append(output_dict)
subworkflow_run_config = WorkflowRunConfig(
target_history=run_config.target_history,
replacement_dict=run_config.replacement_dict,
@@ -485,12 +525,15 @@ def add_parameter(name: str, value: str, type: WorkflowRequestInputParameter.typ
param_map=run_config.param_map.get(step.order_index),
allow_tool_state_corrections=run_config.allow_tool_state_corrections,
resource_params=run_config.resource_params,
+ preferred_object_store_id=run_config.preferred_object_store_id,
+ preferred_intermediate_object_store_id=run_config.preferred_intermediate_object_store_id,
+ preferred_outputs_object_store_id=run_config.preferred_outputs_object_store_id,
+ effective_outputs=effective_outputs,
)
- assert step.subworkflow
subworkflow_invocation = workflow_run_config_to_request(
trans,
subworkflow_run_config,
- step.subworkflow,
+ subworkflow,
)
workflow_invocation.attach_subworkflow_invocation_for_step(
step,
@@ -520,6 +563,18 @@ def add_parameter(name: str, value: str, type: WorkflowRequestInputParameter.typ
"copy_inputs_to_history", "true" if run_config.copy_inputs_to_history else "false", param_types.META_PARAMETERS
)
add_parameter("use_cached_job", "true" if run_config.use_cached_job else "false", param_types.META_PARAMETERS)
+ for param in [
+ "preferred_object_store_id",
+ "preferred_outputs_object_store_id",
+ "preferred_intermediate_object_store_id",
+ ]:
+ value = getattr(run_config, param)
+ if value:
+ add_parameter(param, value, param_types.META_PARAMETERS)
+ if run_config.effective_outputs is not None:
+ # empty list needs to come through here...
+ add_parameter("effective_outputs", json.dumps(run_config.effective_outputs), param_types.META_PARAMETERS)
+
return workflow_invocation
@@ -533,6 +588,11 @@ def workflow_request_to_run_config(
param_map = {}
resource_params = {}
copy_inputs_to_history = None
+ # Preferred object store IDs - either split or join.
+ preferred_object_store_id = None
+ preferred_outputs_object_store_id = None
+ preferred_intermediate_object_store_id = None
+ effective_outputs = None
for parameter in workflow_invocation.input_parameters:
parameter_type = parameter.type
@@ -543,6 +603,14 @@ def workflow_request_to_run_config(
copy_inputs_to_history = parameter.value == "true"
if parameter.name == "use_cached_job":
use_cached_job = parameter.value == "true"
+ if parameter.name == "preferred_object_store_id":
+ preferred_object_store_id = parameter.value
+ if parameter.name == "preferred_outputs_object_store_id":
+ preferred_outputs_object_store_id = parameter.value
+ if parameter.name == "preferred_intermediate_object_store_id":
+ preferred_intermediate_object_store_id = parameter.value
+ if parameter.name == "effective_outputs":
+ effective_outputs = json.loads(parameter.value)
elif parameter_type == param_types.RESOURCE_PARAMETERS:
resource_params[parameter.name] = parameter.value
elif parameter_type == param_types.STEP_PARAMETERS:
@@ -569,5 +637,9 @@ def workflow_request_to_run_config(
copy_inputs_to_history=copy_inputs_to_history,
use_cached_job=use_cached_job,
resource_params=resource_params,
+ preferred_object_store_id=preferred_object_store_id,
+ preferred_outputs_object_store_id=preferred_outputs_object_store_id,
+ preferred_intermediate_object_store_id=preferred_intermediate_object_store_id,
+ effective_outputs=effective_outputs,
)
return workflow_run_config
diff --git a/lib/galaxy_test/api/test_dataset_collections.py b/lib/galaxy_test/api/test_dataset_collections.py
index db93e5d2702a..168713509f57 100644
--- a/lib/galaxy_test/api/test_dataset_collections.py
+++ b/lib/galaxy_test/api/test_dataset_collections.py
@@ -185,7 +185,7 @@ def test_list_list_list_download(self):
@requires_new_user
def test_hda_security(self):
with self.dataset_populator.test_history(require_new=False) as history_id:
- element_identifiers = self.dataset_collection_populator.pair_identifiers(history_id)
+ element_identifiers = self.dataset_collection_populator.pair_identifiers(history_id, wait=True)
self.dataset_populator.make_private(history_id, element_identifiers[0]["id"])
with self._different_user():
history_id = self.dataset_populator.new_history()
diff --git a/lib/galaxy_test/api/test_libraries.py b/lib/galaxy_test/api/test_libraries.py
index ffc680204c38..9fb218ee338c 100644
--- a/lib/galaxy_test/api/test_libraries.py
+++ b/lib/galaxy_test/api/test_libraries.py
@@ -599,4 +599,5 @@ def _create_dataset_in_folder_in_library(self, library_name, content="1 2 3", wa
hda_id = self.dataset_populator.new_dataset(history_id, content=content, wait=wait)["id"]
payload = {"from_hda_id": hda_id, "create_type": "file", "folder_id": folder_id}
ld = self._post(f"libraries/{folder_id}/contents", payload)
+ ld.raise_for_status()
return ld
diff --git a/lib/galaxy_test/base/api_asserts.py b/lib/galaxy_test/base/api_asserts.py
index 37f2aff59c5f..60e14c7b338f 100644
--- a/lib/galaxy_test/base/api_asserts.py
+++ b/lib/galaxy_test/base/api_asserts.py
@@ -85,7 +85,7 @@ def assert_error_message_contains(response: Union[Response, dict], expected_cont
as_dict = _as_dict(response)
assert_has_keys(as_dict, "err_msg")
err_msg = as_dict["err_msg"]
- assert expected_contains in err_msg
+ assert expected_contains in err_msg, f"Expected error message [{err_msg}] to contain [{expected_contains}]."
def _as_dict(response: Union[Response, dict]) -> Dict[str, Any]:
diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py
index a73be6f9085f..71cdc5390c9a 100644
--- a/lib/galaxy_test/base/populators.py
+++ b/lib/galaxy_test/base/populators.py
@@ -409,7 +409,10 @@ def new_dataset_request(
run_response = self.tools_post(payload)
else:
payload = self.fetch_payload(history_id, content=content, **kwds)
- run_response = self.fetch(payload, wait=wait)
+ fetch_kwds = dict(wait=wait)
+ if "assert_ok" in kwds:
+ fetch_kwds["assert_ok"] = kwds["assert_ok"]
+ run_response = self.fetch(payload, **fetch_kwds)
if wait:
self.wait_for_tool_run(history_id, run_response, assert_ok=kwds.get("assert_ok", True))
return run_response
@@ -649,6 +652,10 @@ def history_jobs(self, history_id: str) -> List[Dict[str, Any]]:
assert jobs_response.status_code == 200
return jobs_response.json()
+ def history_jobs_for_tool(self, history_id: str, tool_id: str) -> List[Dict[str, Any]]:
+ jobs = self.history_jobs(history_id)
+ return [j for j in jobs if j["tool_id"] == tool_id]
+
def invocation_jobs(self, invocation_id: str) -> List[Dict[str, Any]]:
query_params = {"invocation_id": invocation_id, "order_by": "create_time"}
jobs_response = self._get("jobs", query_params)
@@ -1074,6 +1081,26 @@ def user_private_role_id(self) -> str:
assert "id" in role, role
return role["id"]
+ def get_usage(self) -> List[Dict[str, Any]]:
+ usage_response = self.galaxy_interactor.get("users/current/usage")
+ usage_response.raise_for_status()
+ return usage_response.json()
+
+ def get_usage_for(self, label: Optional[str]) -> Dict[str, Any]:
+ label_as_str = label if label is not None else "__null__"
+ usage_response = self.galaxy_interactor.get(f"users/current/usage/{label_as_str}")
+ usage_response.raise_for_status()
+ return usage_response.json()
+
+ def update_user(self, properties: Dict[str, Any]) -> Dict[str, Any]:
+ update_response = self.update_user_raw(properties)
+ update_response.raise_for_status()
+ return update_response.json()
+
+ def update_user_raw(self, properties: Dict[str, Any]) -> Response:
+ update_response = self.galaxy_interactor.put("users/current", properties, json=True)
+ return update_response
+
def create_role(self, user_ids: list, description: Optional[str] = None) -> dict:
using_requirement("admin")
payload = {
@@ -1087,14 +1114,14 @@ def create_role(self, user_ids: list, description: Optional[str] = None) -> dict
def create_quota(self, quota_payload: dict) -> dict:
using_requirement("admin")
- quota_response = self._post("quotas", data=quota_payload, admin=True)
- quota_response.raise_for_status()
+ quota_response = self._post("quotas", data=quota_payload, admin=True, json=True)
+ api_asserts.assert_status_code_is_ok(quota_response)
return quota_response.json()
def get_quotas(self) -> list:
using_requirement("admin")
quota_response = self._get("quotas", admin=True)
- quota_response.raise_for_status()
+ api_asserts.assert_status_code_is_ok(quota_response)
return quota_response.json()
def make_private(self, history_id: str, dataset_id: str) -> dict:
@@ -1105,14 +1132,28 @@ def make_private(self, history_id: str, dataset_id: str) -> dict:
"access": [role_id],
"manage": [role_id],
}
+ response = self.update_permissions_raw(history_id, dataset_id, payload)
+ api_asserts.assert_status_code_is_ok(response)
+ return response.json()
+
+ def make_dataset_public_raw(self, history_id: str, dataset_id: str) -> Response:
+ role_id = self.user_private_role_id()
+ payload = {
+ "access": [],
+ "manage": [role_id],
+ }
+ response = self.update_permissions_raw(history_id, dataset_id, payload)
+ return response
+
+ def update_permissions_raw(self, history_id: str, dataset_id: str, payload: dict) -> Response:
url = f"histories/{history_id}/contents/{dataset_id}/permissions"
update_response = self._put(url, payload, admin=True, json=True)
- assert update_response.status_code == 200, update_response.content
- return update_response.json()
+ return update_response
def make_public(self, history_id: str) -> dict:
using_requirement("new_published_objects")
sharing_response = self._put(f"histories/{history_id}/publish")
+ api_asserts.assert_status_code_is_ok(sharing_response)
assert sharing_response.status_code == 200
return sharing_response.json()
@@ -1217,9 +1258,12 @@ def import_history_and_wait_for_name(self, import_data, history_name):
def history_names(self) -> Dict[str, Dict]:
return {h["name"]: h for h in self.get_histories()}
- def rename_history(self, history_id, new_name):
+ def rename_history(self, history_id: str, new_name: str):
+ self.update_history(history_id, {"name": new_name})
+
+ def update_history(self, history_id: str, payload: Dict[str, Any]) -> Response:
update_url = f"histories/{history_id}"
- put_response = self._put(update_url, {"name": new_name}, json=True)
+ put_response = self._put(update_url, payload, json=True)
return put_response
def get_histories(self):
@@ -1765,6 +1809,7 @@ def run_workflow(
expected_response: int = 200,
assert_ok: bool = True,
client_convert: Optional[bool] = None,
+ extra_invocation_kwds: Optional[Dict[str, Any]] = None,
round_trip_format_conversion: bool = False,
invocations: int = 1,
raw_yaml: bool = False,
@@ -1815,6 +1860,8 @@ def run_workflow(
workflow_request["parameters_normalized"] = True
if replacement_parameters:
workflow_request["replacement_params"] = json.dumps(replacement_parameters)
+ if extra_invocation_kwds is not None:
+ workflow_request.update(extra_invocation_kwds)
if has_uploads:
self.dataset_populator.wait_for_history(history_id, assert_ok=True)
assert invocations > 0
@@ -1984,6 +2031,9 @@ class RunJobsSummary(NamedTuple):
invocation: dict
workflow_request: dict
+ def jobs_for_tool(self, tool_id):
+ return [j for j in self.jobs if j["tool_id"] == tool_id]
+
class WorkflowPopulator(GalaxyInteractorHttpMixin, BaseWorkflowPopulator, ImporterGalaxyInterface):
def __init__(self, galaxy_interactor):
@@ -2697,8 +2747,8 @@ def __create_payload_collection(self, history_id: str, identifiers_func, collect
payload = dict(history_id=history_id, collection_type=collection_type, **kwds)
return payload
- def pair_identifiers(self, history_id: str, contents=None):
- hda1, hda2 = self.__datasets(history_id, count=2, contents=contents)
+ def pair_identifiers(self, history_id: str, contents=None, wait: bool = False):
+ hda1, hda2 = self.__datasets(history_id, count=2, contents=contents, wait=wait)
element_identifiers = [
dict(name="forward", src="hda", id=hda1["id"]),
@@ -2734,10 +2784,12 @@ def __create(self, payload, wait=False):
else:
return self.dataset_populator.fetch(payload, wait=wait)
- def __datasets(self, history_id: str, count: int, contents=None):
+ def __datasets(self, history_id: str, count: int, contents=None, wait: bool = False):
datasets = []
for i in range(count):
- new_kwds = {}
+ new_kwds = {
+ "wait": wait,
+ }
if contents:
new_kwds["content"] = contents[i]
datasets.append(self.dataset_populator.new_dataset(history_id, **new_kwds))
diff --git a/lib/galaxy_test/base/workflow_fixtures.py b/lib/galaxy_test/base/workflow_fixtures.py
index 0f164c087b0d..a01d39ce52c3 100644
--- a/lib/galaxy_test/base/workflow_fixtures.py
+++ b/lib/galaxy_test/base/workflow_fixtures.py
@@ -368,6 +368,102 @@
queries_0|input2: nested_workflow/workflow_output
"""
+# WORKFLOW_NESTED_SIMPLE with a nested workflow output marked as an
+# output on the outer workflow.
+WORKFLOW_NESTED_OUTPUT = """
+class: GalaxyWorkflow
+inputs:
+ outer_input: data
+outputs:
+ outer_output:
+ outputSource: second_cat/out_file1
+ nested_output:
+ outputSource: nested_workflow/workflow_output
+steps:
+ first_cat:
+ tool_id: cat1
+ in:
+ input1: outer_input
+ nested_workflow:
+ run:
+ class: GalaxyWorkflow
+ inputs:
+ inner_input: data
+ outputs:
+ workflow_output:
+ outputSource: random_lines/out_file1
+ steps:
+ random_lines:
+ tool_id: random_lines1
+ state:
+ num_lines: 1
+ input:
+ $link: inner_input
+ seed_source:
+ seed_source_selector: set_seed
+ seed: asdf
+ in:
+ inner_input: first_cat/out_file1
+ second_cat:
+ tool_id: cat1
+ in:
+ input1: nested_workflow/workflow_output
+ queries_0|input2: nested_workflow/workflow_output
+"""
+
+# WORKFLOW_NESTED_OUTPUT with two levels of nesting for the workflow
+# output
+WORKFLOW_NESTED_TWICE_OUTPUT = """
+class: GalaxyWorkflow
+inputs:
+ outer_input: data
+outputs:
+ outer_output:
+ outputSource: second_cat/out_file1
+ nested_output:
+ outputSource: nested_workflow/workflow_output
+steps:
+ first_cat:
+ tool_id: cat1
+ in:
+ input1: outer_input
+ nested_workflow:
+ run:
+ class: GalaxyWorkflow
+ inputs:
+ inner_input: data
+ outputs:
+ workflow_output:
+ outputSource: inner_nested_workflow/inner_workflow_output
+ steps:
+ inner_nested_workflow:
+ run:
+ class: GalaxyWorkflow
+ inputs:
+ really_inner_input: data
+ outputs:
+ inner_workflow_output:
+ outputSource: random_lines/out_file1
+ steps:
+ random_lines:
+ tool_id: random_lines1
+ state:
+ num_lines: 1
+ input:
+ $link: really_inner_input
+ seed_source:
+ seed_source_selector: set_seed
+ seed: asdf
+ in:
+ really_inner_input: inner_input
+ in:
+ inner_input: first_cat/out_file1
+ second_cat:
+ tool_id: cat1
+ in:
+ input1: nested_workflow/workflow_output
+ queries_0|input2: nested_workflow/workflow_output
+"""
WORKFLOW_NESTED_RUNTIME_PARAMETER = """
class: GalaxyWorkflow
diff --git a/scripts/cleanup_datasets/pgcleanup.py b/scripts/cleanup_datasets/pgcleanup.py
index 5b28ecfea5ac..5894bc438e7a 100755
--- a/scripts/cleanup_datasets/pgcleanup.py
+++ b/scripts/cleanup_datasets/pgcleanup.py
@@ -10,6 +10,7 @@
import inspect
import logging
import os
+import re
import string
import sys
import time
@@ -26,6 +27,7 @@
import galaxy.config
from galaxy.exceptions import ObjectNotFound
+from galaxy.model import calculate_user_disk_usage_statements
from galaxy.objectstore import build_object_store_from_config
from galaxy.util.script import (
app_properties_from_args,
@@ -76,6 +78,7 @@ class Action:
directly.)
"""
+ requires_objectstore = True
update_time_sql = ", update_time = NOW() AT TIME ZONE 'utc'"
force_retry_sql = " AND NOT purged"
primary_key = None
@@ -116,6 +119,9 @@ def __init__(self, app):
self.__row_methods = []
self.__post_methods = []
self.__exit_methods = []
+ if self.requires_objectstore:
+ self.object_store = build_object_store_from_config(self._config)
+ self._register_exit_method(self.object_store.shutdown)
self._init()
def __enter__(self):
@@ -248,13 +254,14 @@ def _init(self):
class RemovesObjects:
"""Base class for mixins that remove objects from object stores."""
+ requires_objectstore = True
+
def _init(self):
+ super()._init()
self.objects_to_remove = set()
log.info("Initializing object store for action %s", self.name)
- self.object_store = build_object_store_from_config(self._config)
self._register_row_method(self.collect_removed_object_info)
self._register_post_method(self.remove_objects)
- self._register_exit_method(self.object_store.shutdown)
def collect_removed_object_info(self, row):
object_id = getattr(row, self.id_column, None)
@@ -361,7 +368,10 @@ class RequiresDiskUsageRecalculation:
To use, ensure your query returns a ``recalculate_disk_usage_user_id`` column.
"""
+ requires_objectstore = True
+
def _init(self):
+ super()._init()
self.__recalculate_disk_usage_user_ids = set()
self._register_row_method(self.collect_recalculate_disk_usage_user_id)
self._register_post_method(self.recalculate_disk_usage)
@@ -381,30 +391,19 @@ def recalculate_disk_usage(self):
"""
log.info("Recalculating disk usage for users whose data were purged")
for user_id in sorted(self.__recalculate_disk_usage_user_ids):
- # TODO: h.purged = false should be unnecessary once all hdas in purged histories are purged.
- sql = """
- UPDATE galaxy_user
- SET disk_usage = (
- SELECT COALESCE(SUM(total_size), 0)
- FROM ( SELECT d.total_size
- FROM history_dataset_association hda
- JOIN history h ON h.id = hda.history_id
- JOIN dataset d ON hda.dataset_id = d.id
- WHERE h.user_id = %(user_id)s
- AND h.purged = false
- AND hda.purged = false
- AND d.purged = false
- AND d.id NOT IN (SELECT dataset_id
- FROM library_dataset_dataset_association)
- GROUP BY d.id) AS sizes)
- WHERE id = %(user_id)s
- RETURNING disk_usage;
- """
- args = {"user_id": user_id}
- cur = self._update(sql, args, add_event=False)
- for row in cur:
- # disk_usage might be None (e.g. user has purged all data)
- self.log.info("recalculate_disk_usage user_id %i to %s bytes" % (user_id, row.disk_usage))
+ quota_source_map = self.object_store.get_quota_source_map()
+ statements = calculate_user_disk_usage_statements(user_id, quota_source_map)
+
+ for sql, args in statements:
+ sql, _ = re.subn(r"\:([\w]+)", r"%(\1)s", sql)
+ new_args = {}
+ for key, val in args.items():
+ if isinstance(val, list):
+ val = tuple(val)
+ new_args[key] = val
+ self._update(sql, new_args, add_event=False)
+
+ self.log.info("recalculate_disk_usage user_id %i" % user_id)
class RemovesMetadataFiles(RemovesObjects):
diff --git a/scripts/set_user_disk_usage.py b/scripts/set_user_disk_usage.py
index 7f0a2ac3e12c..b23b159b0ca8 100755
--- a/scripts/set_user_disk_usage.py
+++ b/scripts/set_user_disk_usage.py
@@ -44,18 +44,18 @@ def init():
return init_models_from_config(config, object_store=object_store), object_store, engine
-def quotacheck(sa_session, users, engine):
+def quotacheck(sa_session, users, engine, object_store):
sa_session.refresh(user)
current = user.get_disk_usage()
print(user.username, "<" + user.email + ">:", end=" ")
if not args.dryrun:
# Apply new disk usage
- user.calculate_and_set_disk_usage()
+ user.calculate_and_set_disk_usage(object_store)
# And fetch
new = user.get_disk_usage()
else:
- new = user.calculate_disk_usage()
+ new = user.calculate_disk_usage_default_source(object_store)
print("old usage:", nice_size(current), "change:", end=" ")
if new in (current, None):
@@ -77,7 +77,7 @@ def quotacheck(sa_session, users, engine):
print("Processing %i users..." % user_count)
for i, user in enumerate(sa_session.query(model.User).enable_eagerloads(False).yield_per(1000)):
print("%3i%%" % int(float(i) / user_count * 100), end=" ")
- quotacheck(sa_session, user, engine)
+ quotacheck(sa_session, user, engine, object_store)
print("100% complete")
object_store.shutdown()
sys.exit(0)
@@ -88,5 +88,5 @@ def quotacheck(sa_session, users, engine):
if not user:
print("User not found")
sys.exit(1)
+ quotacheck(sa_session, user, engine, object_store)
object_store.shutdown()
- quotacheck(sa_session, user, engine)
diff --git a/test/integration/objectstore/test_private_handling.py b/test/integration/objectstore/test_private_handling.py
new file mode 100644
index 000000000000..f89d23d72cea
--- /dev/null
+++ b/test/integration/objectstore/test_private_handling.py
@@ -0,0 +1,52 @@
+"""Integration tests for mixing store_by."""
+
+import string
+
+from galaxy_test.base import api_asserts
+from ._base import BaseObjectStoreIntegrationTestCase
+
+PRIVATE_OBJECT_STORE_CONFIG_TEMPLATE = string.Template(
+ """
+
+
+
+
+
+"""
+)
+
+TEST_INPUT_FILES_CONTENT = "1 2 3"
+
+
+class TestPrivatePreventsSharingObjectStoreIntegration(BaseObjectStoreIntegrationTestCase):
+ @classmethod
+ def handle_galaxy_config_kwds(cls, config):
+ config["new_user_dataset_access_role_default_private"] = True
+ cls._configure_object_store(PRIVATE_OBJECT_STORE_CONFIG_TEMPLATE, config)
+
+ def test_both_types(self):
+ """Test each object store configures files correctly."""
+ with self.dataset_populator.test_history() as history_id:
+ hda = self.dataset_populator.new_dataset(history_id, content=TEST_INPUT_FILES_CONTENT, wait=True)
+ content = self.dataset_populator.get_history_dataset_content(history_id, hda["id"])
+ assert content.startswith(TEST_INPUT_FILES_CONTENT)
+ response = self.dataset_populator.make_dataset_public_raw(history_id, hda["id"])
+ api_asserts.assert_status_code_is(response, 400)
+ api_asserts.assert_error_code_is(response, 400008)
+ api_asserts.assert_error_message_contains(response, "Attempting to share a non-shareable dataset.")
+
+
+class TestPrivateCannotWritePublicDataObjectStoreIntegration(BaseObjectStoreIntegrationTestCase):
+ @classmethod
+ def handle_galaxy_config_kwds(cls, config):
+ config["new_user_dataset_access_role_default_private"] = False
+ cls._configure_object_store(PRIVATE_OBJECT_STORE_CONFIG_TEMPLATE, config)
+
+ def test_both_types(self):
+ with self.dataset_populator.test_history() as history_id:
+ response = self.dataset_populator.new_dataset_request(
+ history_id, content=TEST_INPUT_FILES_CONTENT, wait=True, assert_ok=False
+ )
+ job = response.json()["jobs"][0]
+ final_state = self.dataset_populator.wait_for_job(job["id"])
+ assert final_state == "error"
diff --git a/test/integration/objectstore/test_quota_limit.py b/test/integration/objectstore/test_quota_limit.py
new file mode 100644
index 000000000000..ad2cd6b1ef38
--- /dev/null
+++ b/test/integration/objectstore/test_quota_limit.py
@@ -0,0 +1,71 @@
+from ._base import BaseObjectStoreIntegrationTestCase
+from .test_selection_with_resource_parameters import (
+ DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE,
+ JOB_CONFIG_FILE,
+ JOB_RESOURCE_PARAMETERS_CONFIG_FILE,
+)
+
+
+class TestQuotaIntegration(BaseObjectStoreIntegrationTestCase):
+ @classmethod
+ def handle_galaxy_config_kwds(cls, config):
+ cls._configure_object_store(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, config)
+ config["job_config_file"] = JOB_CONFIG_FILE
+ config["job_resource_params_file"] = JOB_RESOURCE_PARAMETERS_CONFIG_FILE
+ config["enable_quotas"] = True
+
+ def test_selection_limit(self):
+ with self.dataset_populator.test_history() as history_id:
+ hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3\n4 5 6\n7 8 9\n")
+ self.dataset_populator.wait_for_history(history_id)
+ hda1_input = {"src": "hda", "id": hda1["id"]}
+
+ quotas = self.dataset_populator.get_quotas()
+ assert len(quotas) == 0
+
+ payload = {
+ "name": "defaultquota1",
+ "description": "first default quota",
+ "amount": "1 bytes",
+ "operation": "=",
+ "default": "registered",
+ }
+ self.dataset_populator.create_quota(payload)
+
+ payload = {
+ "name": "ebsquota1",
+ "description": "first ebs quota",
+ "amount": "100 MB",
+ "operation": "=",
+ "default": "registered",
+ "quota_source_label": "ebs",
+ }
+ self.dataset_populator.create_quota(payload)
+
+ quotas = self.dataset_populator.get_quotas()
+ assert len(quotas) == 2
+
+ hda2 = self.dataset_populator.new_dataset(history_id, content="1 2 3\n4 5 6\n7 8 9\n")
+ self.dataset_populator.wait_for_history(history_id)
+
+ hda2_now = self.dataset_populator.get_history_dataset_details(history_id, dataset=hda2, wait=False)
+ assert hda2_now["state"] == "paused"
+
+ create_10_inputs = {
+ "input1": hda1_input,
+ "input2": hda1_input,
+ "__job_resource|__job_resource__select": "yes",
+ "__job_resource|how_store": "slow",
+ }
+ create10_response = self.dataset_populator.run_tool(
+ "create_10",
+ create_10_inputs,
+ history_id,
+ assert_ok=False,
+ )
+ job_id = create10_response["jobs"][0]["id"]
+ self.dataset_populator.wait_for_job(job_id)
+ job_details = self.dataset_populator.get_job_details(job_id).json()
+ # This job isn't paused, it goes through because we used a different
+ # objectstore using job parameters.
+ assert job_details["state"] == "ok"
diff --git a/test/integration/objectstore/test_selection.py b/test/integration/objectstore/test_selection_with_resource_parameters.py
similarity index 80%
rename from test/integration/objectstore/test_selection.py
rename to test/integration/objectstore/test_selection_with_resource_parameters.py
index bb008229d8bb..63a5c1880245 100644
--- a/test/integration/objectstore/test_selection.py
+++ b/test/integration/objectstore/test_selection_with_resource_parameters.py
@@ -1,4 +1,4 @@
-"""Integration tests for object stores."""
+"""Test selecting an object store with resource parameters configured in job configuration."""
import os
import string
@@ -33,11 +33,13 @@
+
+
@@ -48,7 +50,7 @@
)
-class TestObjectStoreSelectionIntegration(BaseObjectStoreIntegrationTestCase):
+class TestObjectStoreSelectionWithResourceParameterIntegration(BaseObjectStoreIntegrationTestCase):
# populated by config_object_store
files_default_path: str
files_static_path: str
@@ -63,7 +65,8 @@ def handle_galaxy_config_kwds(cls, config):
config["job_config_file"] = JOB_CONFIG_FILE
config["job_resource_params_file"] = JOB_RESOURCE_PARAMETERS_CONFIG_FILE
config["object_store_store_by"] = "uuid"
- config["metadata_strategy"] = "celery_extended"
+ # Broken in dev https://github.com/galaxyproject/galaxy/pull/14055
+ # config["metadata_strategy"] = "celery_extended"
config["outputs_to_working_directory"] = True
def _object_store_counts(self):
@@ -87,7 +90,7 @@ def _assert_no_external_filename(self):
for external_filename_tuple in self._app.model.session.query(Dataset.external_filename).all():
assert external_filename_tuple[0] is None
- def test_tool_simple_constructs(self):
+ def test_objectstore_selection(self):
with self.dataset_populator.test_history() as history_id:
def _run_tool(tool_id, inputs):
@@ -109,11 +112,23 @@ def _run_tool(tool_id, inputs):
# One file uploaded, added to default object store ID.
self._assert_file_counts(1, 0, 0, 0)
+ usage_list = self.dataset_populator.get_usage()
+ # assert len(usage_list) == 1
+ assert usage_list[0]["quota_source_label"] is None
+ assert usage_list[0]["total_disk_usage"] == 6
+
+ usage = self.dataset_populator.get_usage_for(None)
+ assert usage["quota_source_label"] is None
+ assert usage["total_disk_usage"] == 6
# should create two files in static object store.
_run_tool("multi_data_param", {"f1": hda1_input, "f2": hda1_input})
self._assert_file_counts(1, 2, 0, 0)
+ usage = self.dataset_populator.get_usage_for(None)
+ assert usage["quota_source_label"] is None
+ assert usage["total_disk_usage"] == 18
+
# should create two files in ebs object store.
create_10_inputs_1 = {
"input1": hda1_input,
@@ -122,6 +137,18 @@ def _run_tool(tool_id, inputs):
_run_tool("create_10", create_10_inputs_1)
self._assert_file_counts(1, 2, 10, 0)
+ usage = self.dataset_populator.get_usage_for("ebs")
+ assert usage["quota_source_label"] == "ebs"
+ assert usage["total_disk_usage"] == 21
+
+ usage_list = self.dataset_populator.get_usage()
+ # assert len(usage_list) == 2
+ assert usage_list[0]["quota_source_label"] is None
+ assert usage_list[0]["total_disk_usage"] == 18
+ ebs_usage = [u for u in usage_list if u["quota_source_label"] == "ebs"][0]
+ assert ebs_usage["quota_source_label"] == "ebs"
+ assert ebs_usage["total_disk_usage"] == 21, str(usage_list)
+
# should create 10 files in S3 object store.
create_10_inputs_2 = {
"__job_resource|__job_resource__select": "yes",
diff --git a/test/integration/objectstore/test_selection_with_user_preferred_object_store.py b/test/integration/objectstore/test_selection_with_user_preferred_object_store.py
new file mode 100644
index 000000000000..0b6383f93b05
--- /dev/null
+++ b/test/integration/objectstore/test_selection_with_user_preferred_object_store.py
@@ -0,0 +1,489 @@
+"""Test selecting an object store with user's preferred object store."""
+
+import os
+import string
+from typing import (
+ Any,
+ Dict,
+ Optional,
+)
+
+from galaxy.model import Dataset
+from galaxy_test.base.populators import WorkflowPopulator
+from galaxy_test.base.workflow_fixtures import (
+ WORKFLOW_NESTED_OUTPUT,
+ WORKFLOW_NESTED_SIMPLE,
+ WORKFLOW_NESTED_TWICE_OUTPUT,
+)
+from ._base import BaseObjectStoreIntegrationTestCase
+
+SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
+
+DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE = string.Template(
+ """
+
+
+
+ This is my description of the default store with *markdown*.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+)
+
+
+TEST_WORKFLOW = """
+class: GalaxyWorkflow
+inputs:
+ input1: data
+outputs:
+ wf_output_1:
+ outputSource: second_cat/out_file1
+steps:
+ first_cat:
+ tool_id: cat
+ in:
+ input1: input1
+ second_cat:
+ tool_id: cat
+ in:
+ input1: first_cat/out_file1
+"""
+
+TEST_WORKFLOW_TEST_DATA = """
+input1:
+ value: 1.fasta
+ type: File
+ name: fasta1
+"""
+
+TEST_NESTED_WORKFLOW_TEST_DATA = """
+outer_input:
+ value: 1.fasta
+ type: File
+ name: fasta1
+"""
+
+# simple output collections
+WORKFLOW_WITH_COLLECTIONS_1 = """
+class: GalaxyWorkflow
+inputs:
+ text_input1: data
+outputs:
+ wf_output_1:
+ outputSource: collection_creates_list/list_output
+steps:
+ cat_inputs:
+ tool_id: cat1
+ in:
+ input1: text_input1
+ queries_0|input2: text_input1
+ split_up:
+ tool_id: collection_split_on_column
+ in:
+ input1: cat_inputs/out_file1
+ collection_creates_list:
+ tool_id: collection_creates_list
+ in:
+ input1: split_up/split_output
+"""
+
+
+# a collection with a dynamic output
+WORKFLOW_WITH_COLLECTIONS_2 = """
+class: GalaxyWorkflow
+inputs:
+ text_input1: data
+outputs:
+ wf_output_1:
+ outputSource: collection_creates_list/list_output
+ wf_output_2:
+ outputSource: split_up/split_output
+steps:
+ cat_inputs:
+ tool_id: cat1
+ in:
+ input1: text_input1
+ queries_0|input2: text_input1
+ split_up:
+ tool_id: collection_split_on_column
+ in:
+ input1: cat_inputs/out_file1
+ collection_creates_list:
+ tool_id: collection_creates_list
+ in:
+ input1: split_up/split_output
+"""
+
+
+WORKFLOW_WITH_COLLECTIONS_1_TEST_DATA = """
+text_input1: |
+ samp1\t10.0
+ samp2\t20.0
+"""
+
+
+def assert_storage_name_is(storage_dict: Dict[str, Any], name: str):
+ storage_name = storage_dict["name"]
+ assert name == storage_name, f"Found incorrect storage name {storage_name}, expected {name} in {storage_dict}"
+
+
+class TestObjectStoreSelectionWithPreferredObjectStoresIntegration(BaseObjectStoreIntegrationTestCase):
+ framework_tool_and_types = True
+
+ # populated by config_object_store
+ files_default_path: str
+ files_static_path: str
+ files_dynamic_path: str
+ files_dynamic_ebs_path: str
+ files_dynamic_s3_path: str
+
+ @classmethod
+ def handle_galaxy_config_kwds(cls, config):
+ super().handle_galaxy_config_kwds(config)
+ cls._configure_object_store(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, config)
+ config["object_store_store_by"] = "uuid"
+ config["outputs_to_working_directory"] = True
+
+ def setUp(self):
+ super().setUp()
+ self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
+
+ def test_setting_unselectable_object_store_id_not_allowed(self):
+ response = self.dataset_populator.update_user_raw({"preferred_object_store_id": "dynamic_s3"})
+ assert response.status_code == 400
+
+ def test_index_query(self):
+ selectable_object_stores_response = self._get("object_store?selectable=true")
+ selectable_object_stores_response.raise_for_status()
+ selectable_object_stores = selectable_object_stores_response.json()
+ selectable_object_store_ids = [s["object_store_id"] for s in selectable_object_stores]
+ assert "default" in selectable_object_store_ids
+ assert "static" in selectable_object_store_ids
+ assert "dynamic_s3" not in selectable_object_store_ids
+
+ def test_objectstore_selection(self):
+ with self.dataset_populator.test_history() as history_id:
+
+ def _run_tool(tool_id, inputs, preferred_object_store_id=None):
+ response = self.dataset_populator.run_tool(
+ tool_id,
+ inputs,
+ history_id,
+ preferred_object_store_id=preferred_object_store_id,
+ )
+ self.dataset_populator.wait_for_history(history_id)
+ return response
+
+ self._set_user_preferred_object_store_id("static")
+
+ storage_info, hda1 = self._create_hda_get_storage_info(history_id)
+ assert_storage_name_is(storage_info, "Static Storage")
+
+ self._reset_user_preferred_object_store_id()
+
+ storage_info, _ = self._create_hda_get_storage_info(history_id)
+ assert_storage_name_is(storage_info, "Default Store")
+
+ self.dataset_populator.update_history(history_id, {"preferred_object_store_id": "static"})
+ storage_info, _ = self._create_hda_get_storage_info(history_id)
+ assert_storage_name_is(storage_info, "Static Storage")
+
+ hda1_input = {"src": "hda", "id": hda1["id"]}
+ response = _run_tool("multi_data_param", {"f1": hda1_input, "f2": hda1_input})
+ storage_info = self._storage_info_for_job_output(response)
+ assert_storage_name_is(storage_info, "Static Storage")
+
+ hda1_input = {"src": "hda", "id": hda1["id"]}
+ response = _run_tool(
+ "multi_data_param", {"f1": hda1_input, "f2": hda1_input}, preferred_object_store_id="default"
+ )
+ storage_info = self._storage_info_for_job_output(response)
+ assert_storage_name_is(storage_info, "Default Store")
+
+ self._reset_user_preferred_object_store_id()
+
+ def test_objectstore_selection_dynamic_output_tools(self):
+ with self.dataset_populator.test_history() as history_id:
+
+ def _run_tool(tool_id, inputs, preferred_object_store_id=None):
+ response = self.dataset_populator.run_tool(
+ tool_id,
+ inputs,
+ history_id,
+ preferred_object_store_id=preferred_object_store_id,
+ )
+ return response
+
+ self._set_user_preferred_object_store_id("static")
+ response = _run_tool("collection_creates_dynamic_list_of_pairs", {"foo": "bar"})
+ self.dataset_populator.wait_for_job(response["jobs"][0]["id"], assert_ok=True)
+ some_dataset = self.dataset_populator.get_history_dataset_details(history_id)
+ storage_dict = self._storage_info(some_dataset)
+ assert_storage_name_is(storage_dict, "Static Storage")
+ self._reset_user_preferred_object_store_id()
+
+ def test_workflow_objectstore_selection(self):
+ with self.dataset_populator.test_history() as history_id:
+ output_dict, intermediate_dict = self._run_workflow_get_output_storage_info_dicts(history_id)
+ assert_storage_name_is(output_dict, "Default Store")
+ assert_storage_name_is(intermediate_dict, "Default Store")
+
+ output_dict, intermediate_dict = self._run_workflow_get_output_storage_info_dicts(
+ history_id, {"preferred_object_store_id": "static"}
+ )
+ assert_storage_name_is(output_dict, "Static Storage")
+ assert_storage_name_is(intermediate_dict, "Static Storage")
+
+ output_dict, intermediate_dict = self._run_workflow_get_output_storage_info_dicts(
+ history_id,
+ {
+ "preferred_outputs_object_store_id": "static",
+ "preferred_intermediate_object_store_id": "dynamic_ebs",
+ },
+ )
+ assert_storage_name_is(output_dict, "Static Storage")
+ assert_storage_name_is(intermediate_dict, "Dynamic EBS")
+
+ def test_simple_subworkflow_objectstore_selection(self):
+ with self.dataset_populator.test_history() as history_id:
+ output_dict, intermediate_dict = self._run_simple_nested_workflow_get_output_storage_info_dicts(
+ history_id,
+ )
+ assert_storage_name_is(output_dict, "Default Store")
+ assert_storage_name_is(intermediate_dict, "Default Store")
+
+ with self.dataset_populator.test_history() as history_id:
+ output_dict, intermediate_dict = self._run_simple_nested_workflow_get_output_storage_info_dicts(
+ history_id, {"preferred_object_store_id": "static"}
+ )
+ assert_storage_name_is(output_dict, "Static Storage")
+ assert_storage_name_is(intermediate_dict, "Static Storage")
+
+ def test_non_effective_subworkflow_outputs_ignored(self):
+ with self.dataset_populator.test_history() as history_id:
+ output_dict, intermediate_dict = self._run_simple_nested_workflow_get_output_storage_info_dicts(
+ history_id,
+ {
+ "preferred_outputs_object_store_id": "static",
+ "preferred_intermediate_object_store_id": "dynamic_ebs",
+ },
+ )
+ assert_storage_name_is(output_dict, "Static Storage")
+ assert_storage_name_is(intermediate_dict, "Dynamic EBS")
+
+ def test_effective_subworkflow_outputs(self):
+ with self.dataset_populator.test_history() as history_id:
+ (
+ output_dict,
+ intermediate_dict,
+ ) = self._run_nested_workflow_with_effective_output_get_output_storage_info_dicts(
+ history_id,
+ {
+ "preferred_outputs_object_store_id": "static",
+ "preferred_intermediate_object_store_id": "dynamic_ebs",
+ },
+ )
+ assert_storage_name_is(output_dict, "Static Storage")
+ assert_storage_name_is(intermediate_dict, "Dynamic EBS")
+
+ def test_effective_subworkflow_outputs_twice_nested(self):
+ with self.dataset_populator.test_history() as history_id:
+ (
+ output_dict,
+ intermediate_dict,
+ ) = self._run_nested_workflow_with_effective_output_get_output_storage_info_dicts(
+ history_id,
+ {
+ "preferred_outputs_object_store_id": "static",
+ "preferred_intermediate_object_store_id": "dynamic_ebs",
+ },
+ twice_nested=True,
+ )
+ assert_storage_name_is(output_dict, "Static Storage")
+ assert_storage_name_is(intermediate_dict, "Dynamic EBS")
+
+ def test_workflow_collection(self):
+ with self.dataset_populator.test_history() as history_id:
+ intermediate_dict, output_info = self._run_workflow_with_collections_1(history_id)
+ assert_storage_name_is(intermediate_dict, "Default Store")
+ assert_storage_name_is(output_info, "Default Store")
+
+ with self.dataset_populator.test_history() as history_id:
+ intermediate_dict, output_info = self._run_workflow_with_collections_1(
+ history_id, {"preferred_object_store_id": "static"}
+ )
+ assert_storage_name_is(intermediate_dict, "Static Storage")
+ assert_storage_name_is(output_info, "Static Storage")
+
+ def test_workflow_collection_mixed(self):
+ with self.dataset_populator.test_history() as history_id:
+ intermediate_dict, output_info = self._run_workflow_with_collections_1(
+ history_id,
+ {
+ "preferred_outputs_object_store_id": "static",
+ "preferred_intermediate_object_store_id": "dynamic_ebs",
+ },
+ )
+ assert_storage_name_is(output_info, "Static Storage")
+ assert_storage_name_is(intermediate_dict, "Dynamic EBS")
+
+ def test_workflow_collection_dynamic_output(self):
+ with self.dataset_populator.test_history() as history_id:
+ intermediate_dict, output_info = self._run_workflow_with_collections_2(
+ history_id,
+ {
+ "preferred_outputs_object_store_id": "static",
+ "preferred_intermediate_object_store_id": "dynamic_ebs",
+ },
+ )
+ assert_storage_name_is(output_info, "Static Storage")
+ assert_storage_name_is(intermediate_dict, "Dynamic EBS")
+
+ def _run_workflow_with_collections_1(self, history_id: str, extra_invocation_kwds: Optional[Dict[str, Any]] = None):
+ wf_run = self.workflow_populator.run_workflow(
+ WORKFLOW_WITH_COLLECTIONS_1,
+ test_data=WORKFLOW_WITH_COLLECTIONS_1_TEST_DATA,
+ history_id=history_id,
+ extra_invocation_kwds=extra_invocation_kwds,
+ )
+ jobs = wf_run.jobs_for_tool("cat1")
+ intermediate_info = self._storage_info_for_job_id(jobs[0]["id"])
+ list_jobs = wf_run.jobs_for_tool("collection_creates_list")
+ assert len(list_jobs) == 1
+ output_list_dict = self.dataset_populator.get_job_details(list_jobs[0]["id"], full=True).json()
+ output_collections = output_list_dict["output_collections"]
+ output_collection = output_collections["list_output"]
+ hdca = self.dataset_populator.get_history_collection_details(history_id, content_id=output_collection["id"])
+ objects = [e["object"] for e in hdca["elements"]]
+ output_info = self._storage_info(objects[0])
+ return intermediate_info, output_info
+
+ def _run_workflow_with_collections_2(self, history_id: str, extra_invocation_kwds: Optional[Dict[str, Any]] = None):
+ wf_run = self.workflow_populator.run_workflow(
+ WORKFLOW_WITH_COLLECTIONS_2,
+ test_data=WORKFLOW_WITH_COLLECTIONS_1_TEST_DATA,
+ history_id=history_id,
+ extra_invocation_kwds=extra_invocation_kwds,
+ )
+ jobs = wf_run.jobs_for_tool("cat1")
+ intermediate_info = self._storage_info_for_job_id(jobs[0]["id"])
+ list_jobs = wf_run.jobs_for_tool("collection_split_on_column")
+ assert len(list_jobs) == 1
+ output_list_dict = self.dataset_populator.get_job_details(list_jobs[0]["id"], full=True).json()
+ output_collections = output_list_dict["output_collections"]
+ output_collection = output_collections["split_output"]
+ hdca = self.dataset_populator.get_history_collection_details(history_id, content_id=output_collection["id"])
+ objects = [e["object"] for e in hdca["elements"]]
+ output_info = self._storage_info(objects[0])
+ return intermediate_info, output_info
+
+ def _run_simple_nested_workflow_get_output_storage_info_dicts(
+ self, history_id: str, extra_invocation_kwds: Optional[Dict[str, Any]] = None
+ ):
+ wf_run = self.workflow_populator.run_workflow(
+ WORKFLOW_NESTED_SIMPLE,
+ test_data=TEST_NESTED_WORKFLOW_TEST_DATA,
+ history_id=history_id,
+ extra_invocation_kwds=extra_invocation_kwds,
+ )
+ jobs = wf_run.jobs_for_tool("cat1")
+ print(jobs)
+ assert len(jobs) == 2
+
+ output_info = self._storage_info_for_job_id(jobs[0]["id"])
+ # nested workflow step... a non-output
+ randomlines_jobs = self.dataset_populator.history_jobs_for_tool(history_id, "random_lines1")
+ assert len(randomlines_jobs) == 1
+ intermediate_info = self._storage_info_for_job_id(randomlines_jobs[0]["id"])
+ return output_info, intermediate_info
+
+ def _run_nested_workflow_with_effective_output_get_output_storage_info_dicts(
+ self, history_id: str, extra_invocation_kwds: Optional[Dict[str, Any]] = None, twice_nested=False
+ ):
+ worklfow_data = WORKFLOW_NESTED_OUTPUT if not twice_nested else WORKFLOW_NESTED_TWICE_OUTPUT
+ wf_run = self.workflow_populator.run_workflow(
+ worklfow_data,
+ test_data=TEST_NESTED_WORKFLOW_TEST_DATA,
+ history_id=history_id,
+ extra_invocation_kwds=extra_invocation_kwds,
+ )
+ jobs = wf_run.jobs_for_tool("cat1")
+ print(jobs)
+ assert len(jobs) == 2
+
+ intermediate_info = self._storage_info_for_job_id(jobs[1]["id"])
+ # nested workflow step... a non-output
+ randomlines_jobs = self.dataset_populator.history_jobs_for_tool(history_id, "random_lines1")
+ assert len(randomlines_jobs) == 1
+ output_info = self._storage_info_for_job_id(randomlines_jobs[0]["id"])
+ return output_info, intermediate_info
+
+ def _run_workflow_get_output_storage_info_dicts(
+ self, history_id: str, extra_invocation_kwds: Optional[Dict[str, Any]] = None
+ ):
+ wf_run = self.workflow_populator.run_workflow(
+ TEST_WORKFLOW,
+ test_data=TEST_WORKFLOW_TEST_DATA,
+ history_id=history_id,
+ extra_invocation_kwds=extra_invocation_kwds,
+ )
+ jobs = wf_run.jobs_for_tool("cat")
+ print(jobs)
+ assert len(jobs) == 2
+ output_info = self._storage_info_for_job_id(jobs[0]["id"])
+ intermediate_info = self._storage_info_for_job_id(jobs[1]["id"])
+ return output_info, intermediate_info
+
+ def _storage_info_for_job_id(self, job_id: str) -> Dict[str, Any]:
+ job_dict = self.dataset_populator.get_job_details(job_id, full=True).json()
+ return self._storage_info_for_job_output(job_dict)
+
+ def _storage_info_for_job_output(self, job_dict) -> Dict[str, Any]:
+ outputs = job_dict["outputs"] # could be a list or dictionary depending on source
+ try:
+ output = outputs[0]
+ except KeyError:
+ output = list(outputs.values())[0]
+ storage_info = self._storage_info(output)
+ return storage_info
+
+ def _storage_info(self, hda):
+ return self.dataset_populator.dataset_storage_info(hda["id"])
+
+ def _set_user_preferred_object_store_id(self, store_id: Optional[str]):
+ user_properties = self.dataset_populator.update_user({"preferred_object_store_id": store_id})
+ assert user_properties["preferred_object_store_id"] == store_id
+
+ def _reset_user_preferred_object_store_id(self):
+ self._set_user_preferred_object_store_id(None)
+
+ def _create_hda_get_storage_info(self, history_id: str):
+ hda1 = self.dataset_populator.new_dataset(history_id, content="1 2 3")
+ self.dataset_populator.wait_for_history(history_id)
+ return self._storage_info(hda1), hda1
+
+ @property
+ def _latest_dataset(self):
+ latest_dataset = self._app.model.session.query(Dataset).order_by(Dataset.table.c.id.desc()).first()
+ return latest_dataset
diff --git a/test/integration/test_quota.py b/test/integration/test_quota.py
index 26ec8965d5ec..fa9a31919796 100644
--- a/test/integration/test_quota.py
+++ b/test/integration/test_quota.py
@@ -3,6 +3,7 @@
class TestQuotaIntegration(integration_util.IntegrationTestCase):
+ dataset_populator: DatasetPopulator
require_admin_user = True
@classmethod
@@ -163,6 +164,26 @@ def test_400_when_invalid_amount(self):
create_response = self._post("quotas", data=payload, json=True)
self._assert_status_code_is(create_response, 400)
+ def test_quota_source_label_basics(self):
+ quotas = self.dataset_populator.get_quotas()
+ prior_quotas_len = len(quotas)
+
+ payload = {
+ "name": "defaultmylabeledquota1",
+ "description": "first default quota that is labeled",
+ "amount": "120MB",
+ "operation": "=",
+ "default": "registered",
+ "quota_source_label": "mylabel",
+ }
+ self.dataset_populator.create_quota(payload)
+
+ quotas = self.dataset_populator.get_quotas()
+ assert len(quotas) == prior_quotas_len + 1
+
+ labels = [q["quota_source_label"] for q in quotas]
+ assert "mylabel" in labels
+
def _create_quota_with_name(self, quota_name: str, is_default: bool = False):
payload = self._build_quota_payload_with_name(quota_name, is_default)
create_response = self._post("quotas", data=payload, json=True)
diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py
index 8ea27b19b04e..3b57d9bbc0cc 100644
--- a/test/unit/data/test_galaxy_mapping.py
+++ b/test/unit/data/test_galaxy_mapping.py
@@ -22,6 +22,7 @@
get_object_session,
)
from galaxy.model.security import GalaxyRBACAgent
+from galaxy.objectstore import QuotaSourceMap
from galaxy.util.unittest import TestCase
datatypes_registry = galaxy.datatypes.registry.Registry()
@@ -35,6 +36,7 @@
not os.environ.get("GALAXY_TEST_UNIT_MAPPING_URI_POSTGRES_BASE"),
reason="GALAXY_TEST_UNIT_MAPPING_URI_POSTGRES_BASE not set",
)
+PRIVATE_OBJECT_STORE_ID = "my_private_data"
class BaseModelTestCase(TestCase):
@@ -153,8 +155,12 @@ def assert_display_name_converts_to_unicode(item, name):
assert history.get_display_name() == "Hello₩◎ґʟⅾ"
def test_hda_to_library_dataset_dataset_association(self):
- u = model.User(email="mary@example.com", password="password")
- hda = model.HistoryDatasetAssociation(name="hda_name")
+ model = self.model
+ u = self.model.User(email="mary@example.com", password="password")
+ h1 = model.History(name="History 1", user=u)
+ hda = model.HistoryDatasetAssociation(
+ name="hda_name", create_dataset=True, history=h1, sa_session=model.session
+ )
self.persist(hda)
trans = collections.namedtuple("trans", "user")
target_folder = model.LibraryFolder(name="library_folder")
@@ -180,6 +186,24 @@ def test_hda_to_library_dataset_dataset_association(self):
assert new_ldda.library_dataset.expired_datasets[0] == ldda
assert target_folder.item_count == 1
+ def test_hda_to_library_dataset_dataset_association_fails_if_private(self):
+ model = self.model
+ u = model.User(email="mary2@example.com", password="password")
+ h1 = model.History(name="History 1", user=u)
+ hda = model.HistoryDatasetAssociation(
+ name="hda_name", create_dataset=True, history=h1, sa_session=model.session
+ )
+ hda.dataset.object_store_id = PRIVATE_OBJECT_STORE_ID
+ self.persist(hda)
+ trans = collections.namedtuple("trans", "user")
+ target_folder = model.LibraryFolder(name="library_folder")
+ with pytest.raises(Exception) as exec_info:
+ hda.to_library_dataset_dataset_association(
+ trans=trans(user=u),
+ target_folder=target_folder,
+ )
+ assert galaxy.model.CANNOT_SHARE_PRIVATE_DATASET_MESSAGE in str(exec_info.value)
+
def test_tags(self):
TAG_NAME = "Test Tag"
my_tag = model.Tag(name=TAG_NAME)
@@ -474,7 +498,7 @@ def test_populated_optimized_list_list_not_populated(self):
def test_default_disk_usage(self):
u = model.User(email="disk_default@test.com", password="password")
self.persist(u)
- u.adjust_total_disk_usage(1)
+ u.adjust_total_disk_usage(1, None)
u_id = u.id
self.expunge()
user_reload = self.model.session.query(model.User).get(u_id)
@@ -588,8 +612,8 @@ def test_history_contents(self):
self.persist(u, h1, expunge=False)
d1 = self.new_hda(h1, name="1")
- d2 = self.new_hda(h1, name="2", visible=False)
- d3 = self.new_hda(h1, name="3", deleted=True)
+ d2 = self.new_hda(h1, name="2", visible=False, object_store_id="foobar")
+ d3 = self.new_hda(h1, name="3", deleted=True, object_store_id="three_store")
d4 = self.new_hda(h1, name="4", visible=False, deleted=True)
self.session().flush()
@@ -603,8 +627,11 @@ def contents_iter_names(**kwds):
assert contents_iter_names() == ["1", "2", "3", "4"]
assert contents_iter_names(deleted=False) == ["1", "2"]
assert contents_iter_names(visible=True) == ["1", "3"]
+ assert contents_iter_names(visible=True, object_store_ids=["three_store"]) == ["3"]
assert contents_iter_names(visible=False) == ["2", "4"]
assert contents_iter_names(deleted=True, visible=False) == ["4"]
+ assert contents_iter_names(deleted=False, object_store_ids=["foobar"]) == ["2"]
+ assert contents_iter_names(deleted=False, object_store_ids=["foobar2"]) == []
assert contents_iter_names(ids=[d1.id, d2.id, d3.id, d4.id]) == ["1", "2", "3", "4"]
assert contents_iter_names(ids=[d1.id, d2.id, d3.id, d4.id], max_in_filter_length=1) == ["1", "2", "3", "4"]
@@ -960,6 +987,77 @@ def test_next_hid(self):
h._next_hid(n=3)
assert h.hid_counter == 5
+ def test_cannot_make_private_objectstore_dataset_public(self):
+ security_agent = GalaxyRBACAgent(self.model)
+ u_from, u_to, _ = self._three_users("cannot_make_private_public")
+
+ h = self.model.History(name="History for Prevent Sharing", user=u_from)
+ d1 = self.model.HistoryDatasetAssociation(
+ extension="txt", history=h, create_dataset=True, sa_session=self.model.session
+ )
+ self.persist(h, d1)
+
+ d1.dataset.object_store_id = PRIVATE_OBJECT_STORE_ID
+ self._make_private(security_agent, u_from, d1)
+
+ with pytest.raises(Exception) as exec_info:
+ self._make_owned(security_agent, u_from, d1)
+ assert galaxy.model.CANNOT_SHARE_PRIVATE_DATASET_MESSAGE in str(exec_info.value)
+
+ def test_cannot_make_private_objectstore_dataset_shared(self):
+ security_agent = GalaxyRBACAgent(self.model)
+ u_from, u_to, _ = self._three_users("cannot_make_private_shared")
+
+ h = self.model.History(name="History for Prevent Sharing", user=u_from)
+ d1 = self.model.HistoryDatasetAssociation(
+ extension="txt", history=h, create_dataset=True, sa_session=self.model.session
+ )
+ self.persist(h, d1)
+
+ d1.dataset.object_store_id = PRIVATE_OBJECT_STORE_ID
+ self._make_private(security_agent, u_from, d1)
+
+ with pytest.raises(Exception) as exec_info:
+ security_agent.privately_share_dataset(d1.dataset, [u_to])
+ assert galaxy.model.CANNOT_SHARE_PRIVATE_DATASET_MESSAGE in str(exec_info.value)
+
+ def test_cannot_set_dataset_permisson_on_private(self):
+ security_agent = GalaxyRBACAgent(self.model)
+ u_from, u_to, _ = self._three_users("cannot_set_permissions_on_private")
+
+ h = self.model.History(name="History for Prevent Sharing", user=u_from)
+ d1 = self.model.HistoryDatasetAssociation(
+ extension="txt", history=h, create_dataset=True, sa_session=self.model.session
+ )
+ self.persist(h, d1)
+
+ d1.dataset.object_store_id = PRIVATE_OBJECT_STORE_ID
+ self._make_private(security_agent, u_from, d1)
+
+ role = security_agent.get_private_user_role(u_to, auto_create=True)
+ access_action = security_agent.permitted_actions.DATASET_ACCESS.action
+
+ with pytest.raises(Exception) as exec_info:
+ security_agent.set_dataset_permission(d1.dataset, {access_action: [role]})
+ assert galaxy.model.CANNOT_SHARE_PRIVATE_DATASET_MESSAGE in str(exec_info.value)
+
+ def test_cannot_make_private_dataset_public(self):
+ security_agent = GalaxyRBACAgent(self.model)
+ u_from, u_to, u_other = self._three_users("cannot_make_private_dataset_public")
+
+ h = self.model.History(name="History for Annotation", user=u_from)
+ d1 = self.model.HistoryDatasetAssociation(
+ extension="txt", history=h, create_dataset=True, sa_session=self.model.session
+ )
+ self.persist(h, d1)
+
+ d1.dataset.object_store_id = PRIVATE_OBJECT_STORE_ID
+ self._make_private(security_agent, u_from, d1)
+
+ with pytest.raises(Exception) as exec_info:
+ security_agent.make_dataset_public(d1.dataset)
+ assert galaxy.model.CANNOT_SHARE_PRIVATE_DATASET_MESSAGE in str(exec_info.value)
+
def _three_users(self, suffix):
email_from = f"user_{suffix}e1@example.com"
email_to = f"user_{suffix}e2@example.com"
@@ -976,18 +1074,26 @@ def _make_private(self, security_agent, user, hda):
access_action = security_agent.permitted_actions.DATASET_ACCESS.action
manage_action = security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action
permissions = {access_action: [role], manage_action: [role]}
- security_agent.set_all_dataset_permissions(hda.dataset, permissions)
+ self._set_permissions(security_agent, hda.dataset, permissions)
def _make_owned(self, security_agent, user, hda):
role = security_agent.get_private_user_role(user, auto_create=True)
manage_action = security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS.action
permissions = {manage_action: [role]}
- security_agent.set_all_dataset_permissions(hda.dataset, permissions)
+ self._set_permissions(security_agent, hda.dataset, permissions)
+
+ def _set_permissions(self, security_agent, dataset, permissions):
+ # TODO: refactor set_all_dataset_permissions to actually throw an exception :|
+ error = security_agent.set_all_dataset_permissions(dataset, permissions)
+ if error:
+ raise Exception(error)
def new_hda(self, history, **kwds):
- return history.add_dataset(
- model.HistoryDatasetAssociation(create_dataset=True, sa_session=self.model.session, **kwds)
- )
+ object_store_id = kwds.pop("object_store_id", None)
+ hda = self.model.HistoryDatasetAssociation(create_dataset=True, sa_session=self.model.session, **kwds)
+ if object_store_id is not None:
+ hda.dataset.object_store_id = object_store_id
+ return history.add_dataset(hda)
@skip_if_not_postgres_base
@@ -1027,8 +1133,11 @@ def _workflow_from_steps(user, steps):
class MockObjectStore:
- def __init__(self):
- pass
+ def __init__(self, quota_source_map=None):
+ self._quota_source_map = quota_source_map or QuotaSourceMap()
+
+ def get_quota_source_map(self):
+ return self._quota_source_map
def size(self, dataset):
return 42
@@ -1044,3 +1153,9 @@ def get_store_by(self, *args, **kwds):
def update_from_file(self, *arg, **kwds):
pass
+
+ def is_private(self, object):
+ if object.object_store_id == PRIVATE_OBJECT_STORE_ID:
+ return True
+ else:
+ return False
diff --git a/test/unit/data/test_quota.py b/test/unit/data/test_quota.py
index 002253109dae..508476c47c12 100644
--- a/test/unit/data/test_quota.py
+++ b/test/unit/data/test_quota.py
@@ -1,26 +1,103 @@
+import uuid
+
from galaxy import model
+from galaxy.objectstore import (
+ QuotaSourceInfo,
+ QuotaSourceMap,
+)
from galaxy.quota import DatabaseQuotaAgent
-from .test_galaxy_mapping import BaseModelTestCase
+from .test_galaxy_mapping import (
+ BaseModelTestCase,
+ MockObjectStore,
+)
-class TestCalculateUsage(BaseModelTestCase):
- def test_calculate_usage(self):
- u = model.User(email="calc_usage@example.com", password="password")
+class TestPurgeUsage(BaseModelTestCase):
+ def setUp(self):
+ super().setUp()
+ model = self.model
+ u = model.User(email="purge_usage@example.com", password="password")
+ u.disk_usage = 25
self.persist(u)
- h = model.History(name="History for Usage", user=u)
+ h = model.History(name="History for Purging", user=u)
self.persist(h)
+ self.u = u
+ self.h = h
- d1 = model.HistoryDatasetAssociation(
- extension="txt", history=h, create_dataset=True, sa_session=self.model.session
+ def _setup_dataset(self):
+ d1 = self.model.HistoryDatasetAssociation(
+ extension="txt", history=self.h, create_dataset=True, sa_session=self.model.session
)
d1.dataset.total_size = 10
self.persist(d1)
+ return d1
+
+ def test_calculate_usage(self):
+ d1 = self._setup_dataset()
+ quota_source_info = QuotaSourceInfo(None, True)
+ d1.purge_usage_from_quota(self.u, quota_source_info)
+ self.persist(self.u)
+ assert int(self.u.disk_usage) == 15
+
+ def test_calculate_usage_untracked(self):
+ # test quota tracking off on the objectstore
+ d1 = self._setup_dataset()
+ quota_source_info = QuotaSourceInfo(None, False)
+ d1.purge_usage_from_quota(self.u, quota_source_info)
+ self.persist(self.u)
+ assert int(self.u.disk_usage) == 25
+
+ def test_calculate_usage_per_source(self):
+ self.u.adjust_total_disk_usage(124, "myquotalabel")
+
+ # test quota tracking with a non-default quota label
+ d1 = self._setup_dataset()
+ quota_source_info = QuotaSourceInfo("myquotalabel", True)
+ d1.purge_usage_from_quota(self.u, quota_source_info)
+ self.persist(self.u)
+ assert int(self.u.disk_usage) == 25
- assert u.calculate_disk_usage() == 10
+ usages = self.u.dictify_usage()
+ assert len(usages) == 2
+ assert usages[1].quota_source_label == "myquotalabel"
+ assert usages[1].total_disk_usage == 114
+
+
+class TestCalculateUsage(BaseModelTestCase):
+ def setUp(self):
+ model = self.model
+ u = model.User(email="calc_usage%s@example.com" % str(uuid.uuid1()), password="password")
+ self.persist(u)
+ h = model.History(name="History for Calculated Usage", user=u)
+ self.persist(h)
+ self.u = u
+ self.h = h
+
+ def _add_dataset(self, total_size, object_store_id=None):
+ model = self.model
+ d1 = model.HistoryDatasetAssociation(
+ extension="txt", history=self.h, create_dataset=True, sa_session=self.model.session
+ )
+ d1.dataset.total_size = total_size
+ d1.dataset.object_store_id = object_store_id
+ self.persist(d1)
+ return d1
+
+ def test_calculate_usage(self):
+ model = self.model
+ u = self.u
+ h = self.h
+
+ d1 = self._add_dataset(10)
+
+ object_store = MockObjectStore()
+ assert u.calculate_disk_usage_default_source(object_store) == 10
assert u.disk_usage is None
- u.calculate_and_set_disk_usage()
- assert u.disk_usage == 10
+ u.calculate_and_set_disk_usage(object_store)
+ assert u.calculate_disk_usage_default_source(object_store) == 10
+ # method no longer updates user object
+ # assert u.disk_usage == 10
# Test dataset being in another history doesn't duplicate usage cost.
h2 = model.History(name="Second usage history", user=u)
@@ -32,7 +109,138 @@ def test_calculate_usage(self):
d3 = model.HistoryDatasetAssociation(extension="txt", history=h, dataset=d1.dataset)
self.persist(d3)
- assert u.calculate_disk_usage() == 10
+ assert u.calculate_disk_usage_default_source(object_store) == 10
+
+ def test_calculate_usage_disabled_quota(self):
+ u = self.u
+
+ self._add_dataset(10, "not_tracked")
+ self._add_dataset(15, "tracked")
+
+ quota_source_map = QuotaSourceMap()
+ not_tracked = QuotaSourceMap()
+ not_tracked.default_quota_enabled = False
+ quota_source_map.backends["not_tracked"] = not_tracked
+
+ object_store = MockObjectStore(quota_source_map)
+
+ assert u.calculate_disk_usage_default_source(object_store) == 15
+
+ def test_calculate_usage_alt_quota(self):
+ model = self.model
+ u = self.u
+
+ self._add_dataset(10)
+ self._add_dataset(15, "alt_source_store")
+
+ quota_source_map = QuotaSourceMap()
+ alt_source = QuotaSourceMap()
+ alt_source.default_quota_source = "alt_source"
+ quota_source_map.backends["alt_source_store"] = alt_source
+
+ object_store = MockObjectStore(quota_source_map)
+
+ u.calculate_and_set_disk_usage(object_store)
+ model.context.refresh(u)
+ usages = u.dictify_usage(object_store)
+ assert len(usages) == 2
+ assert usages[0].quota_source_label is None
+ assert usages[0].total_disk_usage == 10
+
+ assert usages[1].quota_source_label == "alt_source"
+ assert usages[1].total_disk_usage == 15
+
+ usage = u.dictify_usage_for(None)
+ assert usage.quota_source_label is None
+ assert usage.total_disk_usage == 10
+
+ usage = u.dictify_usage_for("alt_source")
+ assert usage.quota_source_label == "alt_source"
+ assert usage.total_disk_usage == 15
+
+ usage = u.dictify_usage_for("unused_source")
+ assert usage.quota_source_label == "unused_source"
+ assert usage.total_disk_usage == 0
+
+ def test_calculate_usage_removes_unused_quota_labels(self):
+ model = self.model
+ u = self.u
+
+ self._add_dataset(10)
+ self._add_dataset(15, "alt_source_store")
+
+ quota_source_map = QuotaSourceMap()
+ alt_source = QuotaSourceMap()
+ alt_source.default_quota_source = "alt_source"
+ quota_source_map.backends["alt_source_store"] = alt_source
+
+ object_store = MockObjectStore(quota_source_map)
+
+ u.calculate_and_set_disk_usage(object_store)
+ model.context.refresh(u)
+ usages = u.dictify_usage()
+ assert len(usages) == 2
+ assert usages[0].quota_source_label is None
+ assert usages[0].total_disk_usage == 10
+
+ assert usages[1].quota_source_label == "alt_source"
+ assert usages[1].total_disk_usage == 15
+
+ alt_source.default_quota_source = "new_alt_source"
+ u.calculate_and_set_disk_usage(object_store)
+ model.context.refresh(u)
+ usages = u.dictify_usage()
+ assert len(usages) == 2
+ assert usages[0].quota_source_label is None
+ assert usages[0].total_disk_usage == 10
+
+ assert usages[1].quota_source_label == "new_alt_source"
+ assert usages[1].total_disk_usage == 15
+
+ def test_dictify_usage_unused_quota_labels(self):
+ model = self.model
+ u = self.u
+
+ self._add_dataset(10)
+ self._add_dataset(15, "alt_source_store")
+
+ quota_source_map = QuotaSourceMap()
+ alt_source = QuotaSourceMap()
+ alt_source.default_quota_source = "alt_source"
+ quota_source_map.backends["alt_source_store"] = alt_source
+
+ unused_source = QuotaSourceMap()
+ unused_source.default_quota_source = "unused_source"
+ quota_source_map.backends["unused_source_store"] = unused_source
+
+ object_store = MockObjectStore(quota_source_map)
+ u.calculate_and_set_disk_usage(object_store)
+ model.context.refresh(u)
+ usages = u.dictify_usage(object_store)
+ assert len(usages) == 3
+
+ def test_calculate_usage_default_storage_disabled(self):
+ model = self.model
+ u = self.u
+
+ self._add_dataset(10)
+ self._add_dataset(15, "alt_source_store")
+
+ quota_source_map = QuotaSourceMap(None, False)
+ alt_source = QuotaSourceMap("alt_source", True)
+ quota_source_map.backends["alt_source_store"] = alt_source
+
+ object_store = MockObjectStore(quota_source_map)
+
+ u.calculate_and_set_disk_usage(object_store)
+ model.context.refresh(u)
+ usages = u.dictify_usage(object_store)
+ assert len(usages) == 2
+ assert usages[0].quota_source_label is None
+ assert usages[0].total_disk_usage == 0
+
+ assert usages[1].quota_source_label == "alt_source"
+ assert usages[1].total_disk_usage == 15
class TestQuota(BaseModelTestCase):
@@ -86,6 +294,27 @@ def test_quota(self):
self._add_group_quota(u, quota)
self._assert_user_quota_is(u, None)
+ def test_labeled_quota(self):
+ model = self.model
+ u = model.User(email="labeled_quota@example.com", password="password")
+ self.persist(u)
+
+ label1 = "coollabel1"
+ self._assert_user_quota_is(u, None, label1)
+
+ quota = model.Quota(name="default registered labeled", amount=21, quota_source_label=label1)
+ self.quota_agent.set_default_quota(
+ model.DefaultQuotaAssociation.types.REGISTERED,
+ quota,
+ )
+
+ self._assert_user_quota_is(u, 21, label1)
+
+ quota = model.Quota(name="user quota add labeled", amount=31, operation="+", quota_source_label=label1)
+ self._add_user_quota(u, quota)
+
+ self._assert_user_quota_is(u, 52, label1)
+
def _add_group_quota(self, user, quota):
group = model.Group()
uga = model.UserGroupAssociation(user, group)
@@ -97,17 +326,56 @@ def _add_user_quota(self, user, quota):
user.quotas.append(uqa)
self.persist(quota, uqa, user)
- def _assert_user_quota_is(self, user, amount):
- assert amount == self.quota_agent.get_quota(user)
- if amount is None:
- user.total_disk_usage = 1000
- job = model.Job()
- job.user = user
- assert not self.quota_agent.is_over_quota(None, job, None)
- else:
- job = model.Job()
- job.user = user
- user.total_disk_usage = amount - 1
- assert not self.quota_agent.is_over_quota(None, job, None)
- user.total_disk_usage = amount + 1
- assert self.quota_agent.is_over_quota(None, job, None)
+ def _assert_user_quota_is(self, user, amount, quota_source_label=None):
+ actual_quota = self.quota_agent.get_quota(user, quota_source_label=quota_source_label)
+ assert amount == actual_quota, f"Expected quota [{amount}], got [{actual_quota}]"
+ if quota_source_label is None:
+ if amount is None:
+ user.total_disk_usage = 1000
+ job = self.model.Job()
+ job.user = user
+ assert not self.quota_agent.is_over_quota(None, job, None)
+ else:
+ job = self.model.Job()
+ job.user = user
+ user.total_disk_usage = amount - 1
+ assert not self.quota_agent.is_over_quota(None, job, None)
+ user.total_disk_usage = amount + 1
+ assert self.quota_agent.is_over_quota(None, job, None)
+
+
+class TestUsage(BaseModelTestCase):
+ def test_usage(self):
+ model = self.model
+ u = model.User(email="usage@example.com", password="password")
+ self.persist(u)
+
+ u.adjust_total_disk_usage(123, None)
+ self.persist(u)
+
+ assert u.get_disk_usage() == 123
+
+ def test_labeled_usage(self):
+ model = self.model
+ u = model.User(email="labeled.usage@example.com", password="password")
+ self.persist(u)
+ assert len(u.quota_source_usages) == 0
+
+ u.adjust_total_disk_usage(123, "foobar")
+ usages = u.dictify_usage()
+ assert len(usages) == 1
+
+ assert u.get_disk_usage() == 0
+ assert u.get_disk_usage(quota_source_label="foobar") == 123
+ self.model.context.refresh(u)
+
+ usages = u.dictify_usage()
+ assert len(usages) == 2
+
+ u.adjust_total_disk_usage(124, "foobar")
+ self.model.context.refresh(u)
+
+ usages = u.dictify_usage()
+ assert len(usages) == 2
+ assert usages[1].quota_source_label == "foobar"
+ assert usages[1].total_disk_usage == 247
diff --git a/test/unit/objectstore/test_objectstore.py b/test/unit/objectstore/test_objectstore.py
index fa6ed55a1b5f..568e6630b3aa 100644
--- a/test/unit/objectstore/test_objectstore.py
+++ b/test/unit/objectstore/test_objectstore.py
@@ -307,8 +307,26 @@ def test_concrete_name_without_objectstore_id():
assert files1_name is None
+MIXED_STORE_BY_DISTRIBUTED_TEST_CONFIG = """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+
MIXED_STORE_BY_HIERARCHICAL_TEST_CONFIG = """
-
+
@@ -326,21 +344,156 @@ def test_concrete_name_without_objectstore_id():
def test_mixed_store_by():
+ with TestConfig(MIXED_STORE_BY_DISTRIBUTED_TEST_CONFIG) as (directory, object_store):
+ as_dict = object_store.to_dict()
+ assert as_dict["backends"][0]["store_by"] == "id"
+ assert as_dict["backends"][1]["store_by"] == "uuid"
+
with TestConfig(MIXED_STORE_BY_HIERARCHICAL_TEST_CONFIG) as (directory, object_store):
as_dict = object_store.to_dict()
assert as_dict["backends"][0]["store_by"] == "id"
assert as_dict["backends"][1]["store_by"] == "uuid"
+def test_mixed_private():
+ # Distributed object store can combine private and non-private concrete objectstores
+ with TestConfig(MIXED_STORE_BY_DISTRIBUTED_TEST_CONFIG) as (directory, object_store):
+ ids = object_store.object_store_ids()
+ print(ids)
+ assert len(ids) == 2
+
+ ids = object_store.object_store_ids(private=True)
+ assert len(ids) == 1
+ assert ids[0] == "files2"
+
+ ids = object_store.object_store_ids(private=False)
+ assert len(ids) == 1
+ assert ids[0] == "files1"
+
+ as_dict = object_store.to_dict()
+ assert not as_dict["backends"][0]["private"]
+ assert as_dict["backends"][1]["private"]
+
+ with TestConfig(MIXED_STORE_BY_HIERARCHICAL_TEST_CONFIG) as (directory, object_store):
+ as_dict = object_store.to_dict()
+ assert as_dict["backends"][0]["private"]
+ assert as_dict["backends"][1]["private"]
+
+ assert object_store.private
+ assert as_dict["private"] is True
+
+
+BADGES_TEST_1_CONFIG_XML = """
+
+
+
+
+
+
+ Fast interconnects.
+
+
+ Storage is backed up to tape nightly.
+
+
+"""
+
+
+BADGES_TEST_1_CONFIG_YAML = """
+type: disk
+files_dir: "${temp_directory}/files1"
+store_by: uuid
+extra_dirs:
+ - type: temp
+ path: "${temp_directory}/tmp1"
+ - type: job_work
+ path: "${temp_directory}/job_working_directory1"
+badges:
+ - type: short_term
+ - type: faster
+ message: Fast interconnects.
+ - type: less_stable
+ - type: more_secure
+ - type: backed_up
+ message: Storage is backed up to tape nightly.
+"""
+
+
+def test_badges_parsing():
+ for config_str in [BADGES_TEST_1_CONFIG_XML, BADGES_TEST_1_CONFIG_YAML]:
+ with TestConfig(config_str) as (directory, object_store):
+ badges = object_store.to_dict()["badges"]
+ assert len(badges) == 6
+ badge_1 = badges[0]
+ assert badge_1["type"] == "short_term"
+ assert badge_1["message"] is None
+
+ badge_2 = badges[1]
+ assert badge_2["type"] == "faster"
+ assert badge_2["message"] == "Fast interconnects."
+
+ badge_3 = badges[2]
+ assert badge_3["type"] == "less_stable"
+ assert badge_3["message"] is None
+
+ badge_4 = badges[3]
+ assert badge_4["type"] == "more_secure"
+ assert badge_4["message"] is None
+
+
+BADGES_TEST_CONFLICTS_1_CONFIG_YAML = """
+type: disk
+files_dir: "${temp_directory}/files1"
+badges:
+ - type: slower
+ - type: faster
+"""
+
+
+BADGES_TEST_CONFLICTS_2_CONFIG_YAML = """
+type: disk
+files_dir: "${temp_directory}/files1"
+badges:
+ - type: more_secure
+ - type: less_secure
+"""
+
+
+def test_badges_parsing_conflicts():
+ for config_str in [BADGES_TEST_CONFLICTS_1_CONFIG_YAML]:
+ exception_raised = False
+ try:
+ with TestConfig(config_str) as (directory, object_store):
+ pass
+ except Exception as e:
+ assert "faster" in str(e)
+ assert "slower" in str(e)
+ exception_raised = True
+ assert exception_raised
+
+ for config_str in [BADGES_TEST_CONFLICTS_2_CONFIG_YAML]:
+ exception_raised = False
+ try:
+ with TestConfig(config_str) as (directory, object_store):
+ pass
+ except Exception as e:
+ assert "more_secure" in str(e)
+ assert "less_secure" in str(e)
+ exception_raised = True
+ assert exception_raised
+
+
DISTRIBUTED_TEST_CONFIG = """
+
+
@@ -354,6 +507,8 @@ def test_mixed_store_by():
type: distributed
backends:
- id: files1
+ quota:
+ source: 1files
type: disk
weight: 2
files_dir: "${temp_directory}/files1"
@@ -363,6 +518,8 @@ def test_mixed_store_by():
- type: job_work
path: "${temp_directory}/job_working_directory1"
- id: files2
+ quota:
+ source: 2files
type: disk
weight: 1
files_dir: "${temp_directory}/files2"
@@ -395,10 +552,45 @@ def test_distributed_store():
_assert_has_keys(as_dict, ["backends", "extra_dirs", "type"])
_assert_key_has_value(as_dict, "type", "distributed")
+ backends = as_dict["backends"]
+ assert len(backends)
+ assert backends[0]["quota"]["source"] == "1files"
+ assert backends[1]["quota"]["source"] == "2files"
+
extra_dirs = as_dict["extra_dirs"]
assert len(extra_dirs) == 2
+HIERARCHICAL_MUST_HAVE_UNIFIED_QUOTA_SOURCE = """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+
+def test_hiercachical_backend_must_share_quota_source():
+ the_exception = None
+ for config_str in [HIERARCHICAL_MUST_HAVE_UNIFIED_QUOTA_SOURCE]:
+ try:
+ with TestConfig(config_str) as (directory, object_store):
+ pass
+ except Exception as e:
+ the_exception = e
+ assert the_exception is not None
+
+
# Unit testing the cloud and advanced infrastructure object stores is difficult, but
# we can at least stub out initializing and test the configuration of these things from
# XML and dicts.
@@ -486,7 +678,7 @@ def test_config_parse_pithos():
assert len(extra_dirs) == 2
-S3_TEST_CONFIG = """
+S3_TEST_CONFIG = """
@@ -498,6 +690,7 @@ def test_config_parse_pithos():
S3_TEST_CONFIG_YAML = """
type: s3
+private: true
auth:
access_key: access_moo
secret_key: secret_cow
@@ -521,6 +714,7 @@ def test_config_parse_pithos():
def test_config_parse_s3():
for config_str in [S3_TEST_CONFIG, S3_TEST_CONFIG_YAML]:
with TestConfig(config_str, clazz=UnitializeS3ObjectStore) as (directory, object_store):
+ assert object_store.private
assert object_store.access_key == "access_moo"
assert object_store.secret_key == "secret_cow"