From 1d88d356c85bbd54332042c88b13224991cf90e0 Mon Sep 17 00:00:00 2001 From: Julien Vignoud Date: Wed, 23 Oct 2024 13:36:31 +0200 Subject: [PATCH] debug github actions --- .github/workflows/lint-test-build.yml | 13 ++ discojs/src/validation/validator.ts | 7 + webapp/cypress.config.ts | 5 + webapp/cypress/e2e/datasetInput.cy.ts | 216 ++++++++++---------- webapp/cypress/e2e/tasks.cy.ts | 62 +++--- webapp/cypress/e2e/testing.cy.ts | 107 +++++----- webapp/cypress/e2e/training.cy.ts | 180 ++++++++-------- webapp/cypress/support/e2e.ts | 2 + webapp/src/components/testing/TestSteps.vue | 4 +- 9 files changed, 319 insertions(+), 277 deletions(-) diff --git a/.github/workflows/lint-test-build.yml b/.github/workflows/lint-test-build.yml index 30371dc10..765488adf 100644 --- a/.github/workflows/lint-test-build.yml +++ b/.github/workflows/lint-test-build.yml @@ -293,6 +293,13 @@ jobs: test-webapp: needs: [build-lib, build-lib-web, download-datasets] runs-on: ubuntu-latest + # Runs tests in parallel with matrix strategy https://docs.cypress.io/guides/guides/parallelization + # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs + # Also see warning here https://github.com/cypress-io/github-action#parallel + strategy: + fail-fast: false # https://github.com/cypress-io/github-action/issues/48 + matrix: + containers: [1, 2] # Uses 2 parallel instances steps: - uses: actions/checkout@v4 with: @@ -314,8 +321,14 @@ jobs: working-directory: webapp install: false start: npm start + wait-on: 'http://localhost:8081' # Waits for above + # Records to Cypress Cloud + # https://docs.cypress.io/guides/cloud/projects#Set-up-a-project-to-record + record: true + parallel: true # Runs test in parallel using settings above env: VITE_SERVER_URL: http://server + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} test-cli: needs: [build-lib, build-lib-node, build-server, download-datasets] diff --git a/discojs/src/validation/validator.ts b/discojs/src/validation/validator.ts index 437cbf73a..64eda99bb 100644 --- a/discojs/src/validation/validator.ts +++ b/discojs/src/validation/validator.ts @@ -94,6 +94,13 @@ export class Validator { ) throw new Error("unexpected shape of dataset"); + // TODO: implement WebWorker to remove this wait + // https://github.com/epfml/disco/issues/758 + // When running on cpu the inference hogs the main thread + // and freezes the UI + if (tf.getBackend() === "cpu") { + await new Promise((resolve) => setTimeout(resolve, 100)); + } const prediction = await this.#model.predict(row.xs); tf.dispose(row); let predictions: number[]; diff --git a/webapp/cypress.config.ts b/webapp/cypress.config.ts index 1031a48be..b7598aec4 100644 --- a/webapp/cypress.config.ts +++ b/webapp/cypress.config.ts @@ -5,10 +5,15 @@ import * as fs from "node:fs/promises"; export default defineConfig({ e2e: { baseUrl: "http://localhost:8081/", + projectId: "aps8et", setupNodeEvents(on) { on("task", { readdir: async (p: string) => (await fs.readdir(p)).map((filename) => path.join(p, filename)), + log: (message) => { + console.log(message) + return null + }, }); }, }, diff --git a/webapp/cypress/e2e/datasetInput.cy.ts b/webapp/cypress/e2e/datasetInput.cy.ts index 51f6d11cc..1a4cbf8b5 100644 --- a/webapp/cypress/e2e/datasetInput.cy.ts +++ b/webapp/cypress/e2e/datasetInput.cy.ts @@ -1,108 +1,108 @@ -import { basicTask, setupServerWith } from "../support/e2e"; - -// TODO move to components testing -// upstream doesn't yet allow that vuejs/test-utils#2468 - -describe("image dataset input by group", () => { - it("shows passed labels", () => { - setupServerWith( - basicTask({ - dataType: "image", - LABEL_LIST: ["first", "second", "third"], - IMAGE_H: 100, - IMAGE_W: 100, - }), - ); - - cy.visit("/#/list"); - cy.get("button").contains("participate").click(); - - cy.get("button").contains("next").click(); - - cy.get("button").contains("group").click(); - - cy.contains("Group label: first").should("exist"); - cy.contains("Group label: second").should("exist"); - cy.contains("Group label: third").should("exist"); - }); - - it("allows to input images", () => { - setupServerWith( - basicTask({ - dataType: "image", - LABEL_LIST: ["label"], - IMAGE_H: 100, - IMAGE_W: 100, - }), - ); - - cy.visit("/#/list"); - cy.get("button").contains("participate").click(); - - cy.get("button").contains("next").click(); - - cy.get("button").contains("group").click(); - cy.contains("select images").selectFile([ - { fileName: "first.png", contents: new Uint8Array() }, - { fileName: "second.png", contents: new Uint8Array() }, - { fileName: "third.png", contents: new Uint8Array() }, - ]); - - cy.contains("Number of selected files: 3").should("exist"); - }); -}); - -describe("image dataset input by csv", () => { - it("allows to input CSV then images", () => { - setupServerWith( - basicTask({ - dataType: "image", - LABEL_LIST: ["label"], - IMAGE_H: 100, - IMAGE_W: 100, - }), - ); - - cy.visit("/#/list"); - cy.get("button").contains("participate").click(); - - cy.get("button").contains("next").click(); - - cy.get("button").contains("csv").click(); - cy.contains("select CSV").selectFile({ - fileName: "csv", - contents: new TextEncoder().encode( - "filename,label\n" + - "first,first\n" + - "second,second\n" + - "third,third\n", - ), - }); - - cy.contains("select images").selectFile([ - { fileName: "first.png", contents: new Uint8Array() }, - { fileName: "second.png", contents: new Uint8Array() }, - { fileName: "third.png", contents: new Uint8Array() }, - ]); - - cy.contains("Number of selected files: 3").should("exist"); - }); -}); - -describe("tabular dataset input", () => { - it("allows to input CSV", () => { - setupServerWith(basicTask({ dataType: "tabular" })); - - cy.visit("/#/list"); - cy.get("button").contains("participate").click(); - - cy.get("button").contains("next").click(); - - cy.contains("select CSV").selectFile({ - fileName: "filename", - contents: new TextEncoder().encode("a,b,c\n1,2,3\n"), - }); - - cy.contains("filename").should("exist"); - }); -}); +// import { basicTask, setupServerWith } from "../support/e2e"; + +// // TODO move to components testing +// // upstream doesn't yet allow that vuejs/test-utils#2468 + +// describe("image dataset input by group", () => { +// it("shows passed labels", () => { +// setupServerWith( +// basicTask({ +// dataType: "image", +// LABEL_LIST: ["first", "second", "third"], +// IMAGE_H: 100, +// IMAGE_W: 100, +// }), +// ); + +// cy.visit("/#/list"); +// cy.get("button").contains("participate").click(); + +// cy.get("button").contains("next").click(); + +// cy.get("button").contains("group").click(); + +// cy.contains("Group label: first").should("exist"); +// cy.contains("Group label: second").should("exist"); +// cy.contains("Group label: third").should("exist"); +// }); + +// it("allows to input images", () => { +// setupServerWith( +// basicTask({ +// dataType: "image", +// LABEL_LIST: ["label"], +// IMAGE_H: 100, +// IMAGE_W: 100, +// }), +// ); + +// cy.visit("/#/list"); +// cy.get("button").contains("participate").click(); + +// cy.get("button").contains("next").click(); + +// cy.get("button").contains("group").click(); +// cy.contains("select images").selectFile([ +// { fileName: "first.png", contents: new Uint8Array() }, +// { fileName: "second.png", contents: new Uint8Array() }, +// { fileName: "third.png", contents: new Uint8Array() }, +// ]); + +// cy.contains("Number of selected files: 3").should("exist"); +// }); +// }); + +// describe("image dataset input by csv", () => { +// it("allows to input CSV then images", () => { +// setupServerWith( +// basicTask({ +// dataType: "image", +// LABEL_LIST: ["label"], +// IMAGE_H: 100, +// IMAGE_W: 100, +// }), +// ); + +// cy.visit("/#/list"); +// cy.get("button").contains("participate").click(); + +// cy.get("button").contains("next").click(); + +// cy.get("button").contains("csv").click(); +// cy.contains("select CSV").selectFile({ +// fileName: "csv", +// contents: new TextEncoder().encode( +// "filename,label\n" + +// "first,first\n" + +// "second,second\n" + +// "third,third\n", +// ), +// }); + +// cy.contains("select images").selectFile([ +// { fileName: "first.png", contents: new Uint8Array() }, +// { fileName: "second.png", contents: new Uint8Array() }, +// { fileName: "third.png", contents: new Uint8Array() }, +// ]); + +// cy.contains("Number of selected files: 3").should("exist"); +// }); +// }); + +// describe("tabular dataset input", () => { +// it("allows to input CSV", () => { +// setupServerWith(basicTask({ dataType: "tabular" })); + +// cy.visit("/#/list"); +// cy.get("button").contains("participate").click(); + +// cy.get("button").contains("next").click(); + +// cy.contains("select CSV").selectFile({ +// fileName: "filename", +// contents: new TextEncoder().encode("a,b,c\n1,2,3\n"), +// }); + +// cy.contains("filename").should("exist"); +// }); +// }); diff --git a/webapp/cypress/e2e/tasks.cy.ts b/webapp/cypress/e2e/tasks.cy.ts index f181c88bc..a920af47d 100644 --- a/webapp/cypress/e2e/tasks.cy.ts +++ b/webapp/cypress/e2e/tasks.cy.ts @@ -1,41 +1,41 @@ -import { defaultTasks } from "@epfml/discojs"; +// import { defaultTasks } from "@epfml/discojs"; -import { setupServerWith } from "../support/e2e.ts"; +// import { setupServerWith } from "../support/e2e.ts"; -describe("tasks page", () => { - it("displays tasks", () => { - setupServerWith( - defaultTasks.titanic, - defaultTasks.mnist, - defaultTasks.lusCovid, - defaultTasks.wikitext, - ); +// describe("tasks page", () => { +// it("displays tasks", () => { +// setupServerWith( +// defaultTasks.titanic, +// defaultTasks.mnist, +// defaultTasks.lusCovid, +// defaultTasks.wikitext, +// ); - cy.visit("/#/list").contains("button", "participate"); +// cy.visit("/#/list").contains("button", "participate"); - // Length 5 = 4 tasks and 1 div for text description - cy.get('div[id="tasks"]').children().should("have.length", 5); - }); +// // Length 5 = 4 tasks and 1 div for text description +// cy.get('div[id="tasks"]').children().should("have.length", 5); +// }); - it("redirects to training", () => { - setupServerWith(defaultTasks.titanic); +// it("redirects to training", () => { +// setupServerWith(defaultTasks.titanic); - cy.visit("/#/list").contains("button", "participate"); +// cy.visit("/#/list").contains("button", "participate"); - cy.get(`div[id="titanic"]`).find("button").click(); - cy.url().should("eq", `${Cypress.config().baseUrl}#/titanic`); +// cy.get(`div[id="titanic"]`).find("button").click(); +// cy.url().should("eq", `${Cypress.config().baseUrl}#/titanic`); - cy.contains("button", "previous").click(); - cy.url().should("eq", `${Cypress.config().baseUrl}#/list`); - }); +// cy.contains("button", "previous").click(); +// cy.url().should("eq", `${Cypress.config().baseUrl}#/list`); +// }); - it("displays error message", () => { - cy.intercept( - { hostname: "server", pathname: "tasks" }, - { statusCode: 404 }, - ); +// it("displays error message", () => { +// cy.intercept( +// { hostname: "server", pathname: "tasks" }, +// { statusCode: 404 }, +// ); - cy.visit("/#/list"); - cy.contains("button", "reload page"); - }); -}); +// cy.visit("/#/list"); +// cy.contains("button", "reload page"); +// }); +// }); diff --git a/webapp/cypress/e2e/testing.cy.ts b/webapp/cypress/e2e/testing.cy.ts index 56b7060c8..c2adf6c45 100644 --- a/webapp/cypress/e2e/testing.cy.ts +++ b/webapp/cypress/e2e/testing.cy.ts @@ -2,77 +2,92 @@ import { defaultTasks } from "@epfml/discojs"; import { setupServerWith } from "../support/e2e"; -it("can test titanic", () => { - setupServerWith(defaultTasks.titanic); +// it("can test titanic", () => { +// setupServerWith(defaultTasks.titanic); - cy.visit("/#/evaluate"); - cy.contains("button", "download").click(); - cy.contains("button", "test").click(); +// cy.visit("/#/evaluate"); +// cy.contains("button", "download").click(); +// cy.contains("button", "test").click(); - cy.contains("label", "select CSV").selectFile( - "../datasets/titanic_train.csv", - ); - cy.contains("button", "next").click(); +// cy.contains("label", "select CSV").selectFile( +// "../datasets/titanic_train.csv", +// ); +// cy.contains("button", "next").click(); - cy.contains("Test & validate") - .parent() - .parent() - .contains("button", "test") - .click(); +// cy.contains("Test & validate") +// .parent() +// .parent() +// .contains("button", "test") +// .click(); - cy.contains("button", "download as csv"); -}); +// cy.contains("button", "download as csv"); +// }); -it("can test lus_covid", () => { - setupServerWith(defaultTasks.lusCovid); +// it("can test lus_covid", () => { +// setupServerWith(defaultTasks.lusCovid); - cy.visit("/#/evaluate"); - cy.contains("button", "download").click(); - cy.contains("button", "test").click(); +// cy.visit("/#/evaluate"); +// cy.contains("button", "download").click(); +// cy.contains("button", "test").click(); - cy.task("readdir", "../datasets/lus_covid/COVID+/").then((files) => - cy.contains("label", "select images").selectFile(files), - ); - cy.contains("button", "next").click(); +// cy.task("readdir", "../datasets/lus_covid/COVID+/").then((files) => +// cy.contains("label", "select images").selectFile(files), +// ); +// cy.contains("button", "next").click(); - cy.contains("Test & validate") - .parent() - .parent() - .contains("button", "test") - .click(); +// cy.contains("Test & validate") +// .parent() +// .parent() +// .contains("button", "test") +// .click(); - cy.contains("button", "download as csv", { timeout: 10_000 }); -}); +// cy.contains("button", "download as csv", { timeout: 10_000 }); +// }); it("can start and stop testing of wikitext", () => { setupServerWith(defaultTasks.wikitext); - + cy.task("log", "starting wikitext test"); + + cy.task("log", "server setup done"); cy.visit("/#/evaluate"); cy.contains("button", "download").click(); cy.contains("button", "test").click(); - + + cy.task("log", "evaluate page loaded"); // input the dataset cy.contains("label", "select text").selectFile( "../datasets/wikitext/wiki.test.tokens", ); - + + cy.task("log", "dataset selected"); // NOTE: internet connection needed // wait for the tokenizer to load and the filename to display // otherwise the training starts before the dataset is ready cy.contains("Connect your data") - .parent() - .parent() - .contains("wiki.test.tokens", { timeout: 5_000 }); - - cy.contains("button", "next").click(); + .parent() + .parent() + .contains("wiki.test.tokens", { timeout: 20_000 }); + cy.task("log", "tokenizer loaded"); + cy.contains("button", "next").click(); + cy.task("log", "button clicked"); + // get the parent of the Test button // otherwise cypress doesn't find it - cy.contains("Test & validate") - .parent() - .parent() - .contains("button", "test") - .click(); + cy.get('[data-cy="start-test"]').click() + cy.task("log", "testing started"); + // cy.task("log", "testing started 2"); - cy.contains("button", "stop testing").click(); + // cy.contains("button", "download as csv", { timeout: 500 }); + // cy.task("log", "found download as csv button"); + + // cy.contains("Test & validate") + // .parent() + // .parent() + // .get('[data-cy="stop-test"]', { timeout: 1_000 }); + // cy.task("log", "found stop testing button"); + + cy.get('[data-cy="stop-test"]') + .click({ waitForAnimations: false }); + cy.task("log", "testing stopped"); }); diff --git a/webapp/cypress/e2e/training.cy.ts b/webapp/cypress/e2e/training.cy.ts index 350c763ed..9a65f272d 100644 --- a/webapp/cypress/e2e/training.cy.ts +++ b/webapp/cypress/e2e/training.cy.ts @@ -1,90 +1,90 @@ -import { defaultTasks } from "@epfml/discojs"; - -import { setupServerWith } from "../support/e2e"; - -describe("training page", () => { - it("is navigable", () => { - setupServerWith(defaultTasks.titanic); - - cy.visit("/"); - - cy.contains("button", "get started").click(); - cy.contains("button", "participate").click(); - cy.contains("button", "participate").click(); - - const navigationButtons = 3; - for (let i = 0; i < navigationButtons; i++) { - cy.contains("button", "next").click(); - } - for (let i = 0; i < navigationButtons + 1; i++) { - cy.contains("button", "previous").click(); - } - }); - - it("can train titanic", () => { - setupServerWith(defaultTasks.titanic); - - cy.visit("/"); - - cy.contains("button", "get started").click(); - cy.contains("button", "participate").click(); - cy.contains("button", "participate").click(); - cy.contains("button", "next").click(); - - cy.contains("label", "select CSV").selectFile( - "../datasets/titanic_train.csv", - ); - cy.contains("button", "next").click(); - - cy.contains("button", "locally").click(); - cy.contains("button", "start training").click(); - cy.contains("h6", "epochs") - .next({ timeout: 40_000 }) - .should("have.text", "10 / 10"); - cy.contains("button", "next").click(); - - cy.contains("button", "test model").click(); - - cy.contains("Titanic Prediction"); - }); - - it("can start and stop training of lus_covid", () => { - setupServerWith(defaultTasks.lusCovid); - - // throwing to stop training - cy.on("uncaught:exception", (e) => !e.message.includes("stop training")); - - cy.visit("/"); - - cy.contains("button", "get started").click(); - cy.contains("button", "participate").click(); - cy.contains("button", "participate").click(); - cy.contains("button", "next").click(); - - cy.task("readdir", "../datasets/lus_covid/COVID+/").then( - (files) => - cy - .contains("h4", "COVID-Positive") - .parents() - .contains("select images") - .selectFile(files), - ); - cy.task("readdir", "../datasets/lus_covid/COVID-/").then( - (files) => - cy - .contains("h4", "COVID-Negative") - .parents() - .contains("select images") - .selectFile(files), - ); - cy.contains("button", "next").click(); - - cy.contains("button", "locally").click(); - cy.contains("button", "start training").click(); - cy.contains("h6", "current batch") - .next({ timeout: 40_000 }) - .should("have.text", "2"); - - cy.contains("button", "stop training").click(); - }); -}); +// import { defaultTasks } from "@epfml/discojs"; + +// import { setupServerWith } from "../support/e2e"; + +// describe("training page", () => { + // it("is navigable", () => { + // setupServerWith(defaultTasks.titanic); + + // cy.visit("/"); + + // cy.contains("button", "get started").click(); + // cy.contains("button", "participate").click(); + // cy.contains("button", "participate").click(); + + // const navigationButtons = 3; + // for (let i = 0; i < navigationButtons; i++) { + // cy.contains("button", "next").click(); + // } + // for (let i = 0; i < navigationButtons + 1; i++) { + // cy.contains("button", "previous").click(); + // } + // }); + + // it("can train titanic", () => { + // setupServerWith(defaultTasks.titanic); + + // cy.visit("/"); + + // cy.contains("button", "get started").click(); + // cy.contains("button", "participate").click(); + // cy.contains("button", "participate").click(); + // cy.contains("button", "next").click(); + + // cy.contains("label", "select CSV").selectFile( + // "../datasets/titanic_train.csv", + // ); + // cy.contains("button", "next").click(); + + // cy.contains("button", "locally").click(); + // cy.contains("button", "start training").click(); + // cy.contains("h6", "epochs") + // .next({ timeout: 40_000 }) + // .should("have.text", "10 / 10"); + // cy.contains("button", "next").click(); + + // cy.contains("button", "test model").click(); + + // cy.contains("Titanic Prediction"); + // }); + + // it("can start and stop training of lus_covid", () => { + // setupServerWith(defaultTasks.lusCovid); + + // // throwing to stop training + // cy.on("uncaught:exception", (e) => !e.message.includes("stop training")); + + // cy.visit("/"); + + // cy.contains("button", "get started").click(); + // cy.contains("button", "participate").click(); + // cy.contains("button", "participate").click(); + // cy.contains("button", "next").click(); + + // cy.task("readdir", "../datasets/lus_covid/COVID+/").then( + // (files) => + // cy + // .contains("h4", "COVID-Positive") + // .parents() + // .contains("select images") + // .selectFile(files), + // ); + // cy.task("readdir", "../datasets/lus_covid/COVID-/").then( + // (files) => + // cy + // .contains("h4", "COVID-Negative") + // .parents() + // .contains("select images") + // .selectFile(files), + // ); + // cy.contains("button", "next").click(); + + // cy.contains("button", "locally").click(); + // cy.contains("button", "start training").click(); + // cy.contains("h6", "current batch") + // .next({ timeout: 40_000 }) + // .should("have.text", "2"); + + // cy.contains("button", "stop training").click(); + // }); +// }); diff --git a/webapp/cypress/support/e2e.ts b/webapp/cypress/support/e2e.ts index 1d8e97004..d42a4a47a 100644 --- a/webapp/cypress/support/e2e.ts +++ b/webapp/cypress/support/e2e.ts @@ -74,3 +74,5 @@ beforeEach( req.onsuccess = resolve; }), ); + +beforeEach(() => { localStorage.debug = "discojs*,webapp*" }); diff --git a/webapp/src/components/testing/TestSteps.vue b/webapp/src/components/testing/TestSteps.vue index f8b5e8002..f0f8ea8c9 100644 --- a/webapp/src/components/testing/TestSteps.vue +++ b/webapp/src/components/testing/TestSteps.vue @@ -40,12 +40,12 @@ against a chosen dataset of yours. Below, once you assessed the model, you can compare the ground truth and the predicted values
- test + test
- stop testing + stop testing