Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Strip galaxy filename annotation on upload #18561

Open
wants to merge 7 commits into
base: dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions client/src/components/ConfirmDialog.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,22 @@
export default {
methods: {
async confirm(message, options = {}) {
// To allow for HTML content in the message, we need to convert it to a VNode.
// https://bootstrap-vue.org/docs/components/modal#message-box-notes
if (options.html) {
message = this.htmlToVNode(message);
}

return this.$bvModal.msgBoxConfirm(message, {
title: "Please Confirm",
titleClass: "h-md",
hideHeaderClose: false,
...options,
});
},
htmlToVNode(html) {
return [this.$createElement("div", { domProps: { innerHTML: html } })];
},
},
render() {
return {};
Expand Down
5 changes: 3 additions & 2 deletions client/src/components/Upload/CompositeBox.vue
Original file line number Diff line number Diff line change
Expand Up @@ -100,14 +100,15 @@ function eventReset() {
}

/** Start upload process */
function eventStart() {
async function eventStart() {
uploadValues.value.forEach((model) => {
model.dbKey = dbKey.value;
model.extension = extension.value;
});
try {
const data = await uploadPayload(uploadValues.value, props.historyId, true);
uploadSubmit({
data: uploadPayload(uploadValues.value, props.historyId, true),
data,
error: eventError,
progress: eventProgress,
success: eventSuccess,
Expand Down
4 changes: 2 additions & 2 deletions client/src/components/Upload/UploadContainer.vue
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,8 @@ function immediateUpload(files) {
}
}

function toData(items, history_id, composite = false) {
return uploadPayload(items, history_id, composite);
async function toData(items, history_id, composite = false) {
return await uploadPayload(items, history_id, composite);
}

async function loadExtensions() {
Expand Down
15 changes: 11 additions & 4 deletions client/src/utils/data.js
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,16 @@ function _mountSelectionDialog(clazz, options) {
* Creates a history dataset by submitting an upload request
* TODO: This should live somewhere else.
*/
export function create(options) {
export async function create(options) {
async function getHistory() {
if (!options.history_id) {
return getCurrentGalaxyHistory();
}
return options.history_id;
}
getHistory().then((history_id) => {
try {
const history_id = await getHistory();
const payload = await uploadPayload([options], history_id);
uploadSubmit({
success: (response) => {
refreshContentsWrapper();
Expand All @@ -105,10 +107,15 @@ export function create(options) {
},
error: options.error,
data: {
payload: uploadPayload([options], history_id),
payload,
},
});
});
} catch (err) {
console.error("Error creating history dataset via upload:", err);
if (options.error) {
options.error(err);
}
}
}

export function refreshContentsWrapper() {
Expand Down
59 changes: 50 additions & 9 deletions client/src/utils/upload-payload.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { useConfirmDialog } from "@/composables/confirmDialog";

export const DEFAULT_FILE_NAME = "New File";
export const URI_PREFIXES = [
"http://",
Expand All @@ -18,16 +20,53 @@ export function isUrl(content) {
return URI_PREFIXES.some((prefix) => content.startsWith(prefix));
}

export function uploadPayload(items, historyId, composite = false) {
export function isGalaxyFile(content) {
if (content === undefined || content === null) {
return false;
}
const galaxyRegexPattern = /Galaxy\d+-\[(.*?)\](\..+)/;
const match = content.match(galaxyRegexPattern);
if (match) {
return true;
} else {
return false;
}
}

export async function uploadPayload(items, historyId, composite = false) {
const files = [];
const elements = items
.map((item) => {
const elements = await Promise.all(
items.map(async (item) => {
if (!item.optional || item.fileSize > 0) {
// avoid submitting default file name, server will set file name
let fileName = item.fileName;
if (fileName === DEFAULT_FILE_NAME) {
fileName = null;
}

if (!composite && isGalaxyFile(item.fileName)) {
const modifiedFileName = item.fileName.replace(/Galaxy\d+-\[(.*?)\](\..+)/, "$1");

const { confirm: confirmFromDialog } = useConfirmDialog();
const keepModifiedName = await confirmFromDialog(
`This looks like a previous Galaxy file. We have renamed it.<br /><br />
<b>Original Name:</b> <i>${item.fileName}</i><br />
<b>Modified Name:</b> <i>${modifiedFileName}</i><br /><br />
Do you want to keep the modified name?`,
{
title: "Rename potential Galaxy file",
okTitle: "Rename",
okVariant: "primary",
cancelTitle: "Keep Original",
html: true,
}
);
if (keepModifiedName) {
item.fileName = modifiedFileName;
fileName = modifiedFileName;
}
}

// consolidate exclusive file content attributes
const urlContent = (item.fileUri || item.filePath || item.fileContent || "").trim();
const hasFileData = item.fileData && item.fileData.size > 0;
Expand Down Expand Up @@ -91,21 +130,23 @@ export function uploadPayload(items, historyId, composite = false) {
}
}
})
.filter((item) => item)
.flat();
if (elements.length === 0) {
);

const flattenedElements = elements.filter((item) => item).flat();
if (flattenedElements.length === 0) {
throw new Error("No valid items provided.");
}

const target = {
destination: { type: "hdas" },
elements: elements,
elements: flattenedElements,
};
if (composite) {
const compositeItems = [
{
src: "composite",
dbkey: elements[0].dbkey,
ext: elements[0].ext,
dbkey: flattenedElements[0].dbkey,
ext: flattenedElements[0].ext,
composite: {
items: target.elements,
},
Expand Down
73 changes: 61 additions & 12 deletions client/src/utils/upload-payload.test.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,21 @@
import { isUrl, uploadPayload } from "./upload-payload.js";

// confirm dialog to ask user if they want to auto rename a potential Galaxy file
jest.mock("@/composables/confirmDialog", () => ({
useConfirmDialog: jest.fn(() => ({
// we assume here the user always wants to auto rename the Galaxy file
confirm: jest.fn(() => Promise.resolve(true)),
})),
}));

describe("uploadPayload", () => {
test("basic validation", () => {
expect(() => uploadPayload([], "historyId")).toThrow("No valid items provided.");
expect(() => uploadPayload([{}], "historyId")).toThrow("Content not available.");
expect(() => uploadPayload([{ fileContent: "fileContent" }], "historyId")).toThrow(
test("basic validation", async () => {
await expect(uploadPayload([], "historyId")).rejects.toThrow("No valid items provided.");
await expect(uploadPayload([{}], "historyId")).rejects.toThrow("Content not available.");
await expect(uploadPayload([{ fileContent: "fileContent" }], "historyId")).rejects.toThrow(
"Unknown file mode: undefined."
);
expect(() =>
await expect(
uploadPayload(
[
{
Expand All @@ -23,7 +31,7 @@ describe("uploadPayload", () => {
],
"historyId"
)
).toThrow("Invalid url: xyz://test.me.1");
).rejects.toThrow("Invalid url: xyz://test.me.1");
});

test("url detection", () => {
Expand All @@ -32,8 +40,8 @@ describe("uploadPayload", () => {
expect(isUrl("http://")).toBeTruthy();
});

test("regular payload", () => {
const p = uploadPayload(
test("regular payload", async () => {
const p = await uploadPayload(
[
{ fileContent: " fileContent ", fileMode: "new", fileName: "1" },
{
Expand Down Expand Up @@ -66,12 +74,22 @@ describe("uploadPayload", () => {
spaceToTab: false,
toPosixLines: false,
},
{
dbKey: "dbKey5",
deferred: true,
extension: "extension5",
fileData: { size: 1 },
fileMode: "local",
fileName: "Galaxy5-[PreviousGalaxyFile].bed",
spaceToTab: true,
toPosixLines: true,
},
],
"historyId"
);
expect(p).toEqual({
auto_decompress: true,
files: [{ size: 1 }],
files: [{ size: 1 }, { size: 1 }],
history_id: "historyId",
targets: [
{
Expand Down Expand Up @@ -126,14 +144,24 @@ describe("uploadPayload", () => {
to_posix_lines: false,
url: "http://test.me",
},
// confirmDialog for auto renaming, assume user wants to auto rename
{
dbkey: "dbKey5",
deferred: true,
ext: "extension5",
name: "PreviousGalaxyFile",
space_to_tab: true,
src: "files",
to_posix_lines: true,
},
],
},
],
});
});

test("composite payload", () => {
const p = uploadPayload(
test("composite payload", async () => {
const p = await uploadPayload(
[
{ fileContent: "fileContent", fileMode: "new", fileName: "1" },
{
Expand All @@ -147,13 +175,24 @@ describe("uploadPayload", () => {
spaceToTab: true,
toPosixLines: true,
},
{
dbKey: "dbKey2",
deferred: true,
extension: "extension2",
fileContent: "fileContent",
fileData: "fileData",
fileMode: "local",
fileName: "Galaxy2-[PreviousGalaxyFile].bed",
spaceToTab: true,
toPosixLines: true,
},
],
"historyId",
true
);
expect(p).toEqual({
auto_decompress: true,
files: ["fileData"],
files: ["fileData", "fileData"],
history_id: "historyId",
targets: [
{
Expand Down Expand Up @@ -181,6 +220,16 @@ describe("uploadPayload", () => {
src: "files",
to_posix_lines: true,
},
// in composite mode, we do not have a confirmDialog for auto renaming
{
dbkey: "dbKey2",
deferred: true,
ext: "extension2",
name: "Galaxy2-[PreviousGalaxyFile].bed",
space_to_tab: true,
src: "files",
to_posix_lines: true,
},
],
},
dbkey: "?",
Expand Down
16 changes: 8 additions & 8 deletions client/src/utils/upload-queue.js
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,11 @@ export class UploadQueue {
}

// Initiate upload process
start() {
async start() {
if (!this.isRunning) {
this.isRunning = true;
this._processUrls();
this._process();
await this._processUrls();
await this._process();
}
}

Expand All @@ -81,7 +81,7 @@ export class UploadQueue {
}

// Process an upload, recursive
_process() {
async _process() {
if (this.size === 0 || this.isPaused) {
this.isRunning = false;
this.isPaused = false;
Expand All @@ -100,19 +100,19 @@ export class UploadQueue {
if (!item.targetHistoryId) {
throw new Error(`Missing target history for upload item [${index}] ${item.fileName}`);
}
const data = uploadPayload([item], item.targetHistoryId);
const data = await uploadPayload([item], item.targetHistoryId);
// Initiate upload request
this._processSubmit(index, data);
} catch (e) {
// Parse error message for failed upload item
this.opts.error(index, String(e));
// Continue queue
this._process();
await this._process();
}
}

// Submit remote files as single batch request per target history
_processUrls() {
async _processUrls() {
const batchByHistory = {};
for (const index of this.queue.keys()) {
const model = this.opts.get(index);
Expand All @@ -131,7 +131,7 @@ export class UploadQueue {
for (const historyId in batchByHistory) {
const list = batchByHistory[historyId];
try {
const data = uploadPayload(list, historyId);
const data = await uploadPayload(list, historyId);
sendPayload(data, {
success: (message) => {
list.forEach((model) => {
Expand Down
Loading
Loading