From c249182aaf276b029ee6c5b5ea9c0a60275380a9 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 21 Feb 2024 11:10:01 -0600 Subject: [PATCH 1/6] Switch to global test timeout (#621) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- tests/e2e.test.ts | 14 +++++++------- vite.config.js | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/e2e.test.ts b/tests/e2e.test.ts index 21b18880b..66ea5650b 100644 --- a/tests/e2e.test.ts +++ b/tests/e2e.test.ts @@ -141,7 +141,7 @@ describe('E2E Test', () => { expect(existsSync(outputLocation)).toBe(true) - }, 2 * 60 * 1000) // Allow two minutes to create dataset + }) test('Create new pipeline by specifying a name', async () => { @@ -185,7 +185,7 @@ describe('E2E Test', () => { // Advance to formats page await toNextPage('structure') - }, 10 * 1000) + }) test('Specify data formats', async () => { @@ -227,7 +227,7 @@ describe('E2E Test', () => { await toNextPage('locate') - }, 10 * 1000) + }) test('Locate all your source data programmatically', async () => { @@ -319,7 +319,7 @@ describe('E2E Test', () => { await toNextPage('sourcedata') - }, 10 * 1000) + }) test('Review source data information', async () => { @@ -342,7 +342,7 @@ describe('E2E Test', () => { await toNextPage('inspect') - }, 30 * 1000) // Wait for conversion preview to complete + }) // Wait for conversion preview to complete test('Review NWB Inspector output', async () => { @@ -358,7 +358,7 @@ describe('E2E Test', () => { if (skipUpload) await toHome() - }, 60 * 1000) // Wait for full conversion to complete + }) // Wait for full conversion to complete const uploadDescribe = skipUpload ? describe.skip: describe @@ -402,7 +402,7 @@ describe('E2E Test', () => { await toNextPage('review') - }, 3 * 60 * 1000) // Wait for upload to finish (~2min on M2) + }) // Wait for upload to finish (~2min on M2) test('Review upload results', async () => { diff --git a/vite.config.js b/vite.config.js index 13ae4ac38..747d09a47 100644 --- a/vite.config.js +++ b/vite.config.js @@ -5,5 +5,6 @@ export default defineConfig({ test: { environment: "jsdom", setupFiles: ["dotenv/config"], + testTimeout: 3 * 60 * 1000, }, }); From 45d1262012e26520dac04241a1bf48a0b1a8b502 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Wed, 21 Feb 2024 11:27:44 -0600 Subject: [PATCH 2/6] Update search strategy (#620) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- src/renderer/src/stories/Search.js | 36 +++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/src/renderer/src/stories/Search.js b/src/renderer/src/stories/Search.js index 50cd073aa..70e601896 100644 --- a/src/renderer/src/stories/Search.js +++ b/src/renderer/src/stories/Search.js @@ -256,23 +256,37 @@ export class Search extends LitElement { #sortedCategories = []; + getTokens = (input) => + input + .replaceAll(/[^\w\s]/g, "") + .split(" ") + .map((token) => token.trim().toLowerCase()) + .filter((token) => token); + #populate = (input = this.value ?? "") => { const toShow = []; - // Check if the input value matches the label - this.#options.forEach(({ option, label }, i) => { - if (label.toLowerCase().includes(input.toLowerCase()) && !toShow.includes(i)) toShow.push(i); - }); + const inputTokens = this.getTokens(input); - // Check if the input value matches any of the keywords - this.#options.forEach(({ option, keywords = [], structuredKeywords = {} }, i) => { - [...keywords, ...Object.values(structuredKeywords).flat()].forEach((keyword) => { - if (keyword.toLowerCase().includes(input.toLowerCase()) && !toShow.includes(i)) toShow.push(i); - }); + // Check if the input value matches the label or any of the keywords + this.#options.forEach(({ label, keywords, structuredKeywords }, i) => { + const labelTokens = this.getTokens(label); + const allKeywords = [...keywords, ...Object.values(structuredKeywords).flat()]; + const allKeywordTokens = allKeywords.map((keyword) => this.getTokens(keyword)).flat(); + const allTokens = [...labelTokens, ...allKeywordTokens]; + + const result = inputTokens.reduce((acc, token) => { + for (let subtoken of allTokens) { + if (subtoken.startsWith(token) && !toShow.includes(i)) return (acc += 1); + } + return acc; + }, 0); + + if (result === inputTokens.length) toShow.push(i); }); this.#options.forEach(({ option }, i) => { - if (toShow.includes(i)) { + if (toShow.includes(i) || !inputTokens.length) { option.removeAttribute("hidden"); } else { option.setAttribute("hidden", ""); @@ -285,7 +299,7 @@ export class Search extends LitElement { else element.removeAttribute("hidden"); }); - this.setAttribute("active", !!toShow.length); + this.setAttribute("active", !!toShow.length || !inputTokens.length); this.setAttribute("interacted", true); }; From 30fec800ce7518840adb7100bcb28b2dcda70ed2 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Tue, 27 Feb 2024 09:56:28 -0800 Subject: [PATCH 3/6] Skip DANDI upload to allow tests to pass with broken external services (#627) --- .github/workflows/testing-external.yml | 93 ++++++++++++++++++++++++++ .github/workflows/testing.yml | 6 -- 2 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/testing-external.yml diff --git a/.github/workflows/testing-external.yml b/.github/workflows/testing-external.yml new file mode 100644 index 000000000..711a81653 --- /dev/null +++ b/.github/workflows/testing-external.yml @@ -0,0 +1,93 @@ +name: External Tests +on: + schedule: + - cron: "0 16 * * *" # Daily at noon EST + pull_request: + +concurrency: # Cancel previous workflows on the same pull request + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CACHE_NUMBER: 2 # increase to reset cache manually + +jobs: + testing: + name: External tests on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + include: + - os: ubuntu-latest + label: environments/environment-Linux.yml + + - os: macos-latest + label: environments/environment-Mac.yml + + - os: windows-latest + label: environments/environment-Windows.yml + + + steps: + - uses: actions/checkout@v3 + - run: git fetch --prune --unshallow --tags + + # see https://github.com/conda-incubator/setup-miniconda#caching-environments + - name: Setup Mambaforge + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: nwb-guide + use-mamba: true + + - name: Set cache date + id: get-date + run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache Conda env + uses: actions/cache@v2 + with: + path: ${{ env.CONDA }}/envs + key: conda-${{ runner.os }}-${{ runner.arch }}-${{steps.get-date.outputs.today }}-${{ hashFiles(matrix.label) }}-${{ env.CACHE_NUMBER }} + id: cache + + - if: steps.cache.outputs.cache-hit != 'true' + name: Create and activate environment + run: mamba env update -n nwb-guide -f ${{ matrix.label }} + + - name: Use Node.js 18 + uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Install GUIDE + run: npm ci + + - name: Create env file + run: | + touch .env + echo DANDI_STAGING_API_KEY=${{ secrets.DANDI_STAGING_API_KEY }} >> .env + + - if: matrix.os != 'ubuntu-latest' + name: Run tests + run: npm run coverage:app + + - if: matrix.os == 'ubuntu-latest' + name: Run tests with xvfb + run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run coverage:app + + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + fail_ci_if_error: true diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 17fc67b48..6c6fbad4c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -71,12 +71,6 @@ jobs: - name: Install GUIDE run: npm ci - - - name: Create env file - run: | - touch .env - echo DANDI_STAGING_API_KEY=${{ secrets.DANDI_STAGING_API_KEY }} >> .env - - if: matrix.os != 'ubuntu-latest' name: Run tests run: npm run test:coverage From 374ed896a0da87e670a7d6c6b30c0f503648fcd3 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 27 Feb 2024 10:12:29 -0800 Subject: [PATCH 4/6] Update installation instructions (#626) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- docs/developer_guide.rst | 10 ++++++---- docs/installation.rst | 17 +++++++++++------ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/developer_guide.rst b/docs/developer_guide.rst index 972ccdb5f..1fabc5dd3 100644 --- a/docs/developer_guide.rst +++ b/docs/developer_guide.rst @@ -24,7 +24,6 @@ Start by cloning the repository Install Python Dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Install the appropriate Python dependencies for your operating system. **Windows** @@ -51,6 +50,10 @@ Install the appropriate Python dependencies for your operating system. conda env create -f ./environments/environment-Linux.yml + +Activate the Python Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Before starting NWB GUIDE, you'll need to ensure that the Python environment is activated. .. code-block:: bash @@ -58,8 +61,8 @@ Before starting NWB GUIDE, you'll need to ensure that the Python environment is conda activate nwb-guide -Installing JavaScript Dependencies -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Install JavaScript Dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Next, install all JavaScript dependencies based on the `package-lock.json` file. @@ -78,7 +81,6 @@ You can now run the following command to start the application using Electron. npm start - Repo Structure -------------- 1. **src/renderer/src** - Contains all the source code for the frontend diff --git a/docs/installation.rst b/docs/installation.rst index 147de87ad..563dc9e5f 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -5,22 +5,27 @@ Installation Windows ------- -Download and run the `NWB-GUIDE-Setup-vX.Y.Z.exe `_ file and follow all instruction prompts. +Download and run `NWB-GUIDE-x64.exe `_ and follow all instruction prompts. MacOS - Intel ------------- -Download the `NWB-GUIDE-vX.Y.Z.dmg `_ file, which should prompt you to move it into your 'Applications' folder in order to run. +Download `NWB-GUIDE-x64.dmg `_, which should prompt you to move it into your 'Applications' folder in order to run. MacOS - Apple Silicon --------------------- -Download the `NWB-GUIDE-vX.Y.Z-arm64.dmg `_ file, which should prompt you to move it into your 'Applications' folder in order to run. +Download `NWB-GUIDE-arm64.dmg `_, which should prompt you to move it into your 'Applications' folder in order to run. .. note:: Some data formats can have issues using this build of the application. If you encounter errors when using a particular interface, try following the advanced :ref:`Developer Installation instructions` instructions. -Ubuntu ------- +Linux +----- -Please clone the :linux-fix:`linux-fix <>` branch of the NWB GUIDE and follow the :ref:`Developer Installation instructions` on this documentation after the "Clone the Repo" step. +Please clone the :linux-fix:`linux-fix <>` branch of the NWB GUIDE and follow the :ref:`Developer Installation instructions` after the "Clone the Repo" step. + +.. code-block:: bash + + git clone --branch linux-fix https://github.com/NeurodataWithoutBorders/nwb-guide + cd nwb-guide From 99e5e5294f25f574357d7eb3a6ed32496e19b660 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Tue, 27 Feb 2024 10:29:43 -0800 Subject: [PATCH 5/6] Miscellaneous Small Fixes (#622) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- src/renderer/src/stories/InfoBox.js | 5 ++++- src/renderer/src/stories/JSONSchemaForm.js | 1 - .../stories/pages/guided-mode/GuidedStart.js | 3 +-- .../pages/guided-mode/data/GuidedMetadata.js | 22 +++++++++---------- .../stories/pages/guided-mode/data/utils.js | 4 ++-- .../pages/guided-mode/setup/GuidedSubjects.js | 1 - src/renderer/src/stories/table/Cell.ts | 2 -- 7 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/renderer/src/stories/InfoBox.js b/src/renderer/src/stories/InfoBox.js index c11b4c54b..3177dd2c5 100644 --- a/src/renderer/src/stories/InfoBox.js +++ b/src/renderer/src/stories/InfoBox.js @@ -1,5 +1,6 @@ import { LitElement, css, html } from "lit"; import { Chevron } from "./Chevron"; +import { unsafeHTML } from "lit/directives/unsafe-html.js"; export class InfoBox extends LitElement { static get styles() { @@ -120,7 +121,9 @@ export class InfoBox extends LitElement { ${new Chevron({ direction: "right" })}
- ${this.content} + ${typeof this.content === "string" ? unsafeHTML(this.content) : this.content}
`; } diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index c08b8c1ff..ed318d187 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -464,7 +464,6 @@ export class JSONSchemaForm extends LitElement { // if (!isValid && allErrors.length && nMissingRequired === allErrors.length) message = `${nMissingRequired} required inputs are not defined.`; - console.log(allErrors); // Check if all inputs are valid if (flaggedInputs.length) { flaggedInputs[0].focus(); diff --git a/src/renderer/src/stories/pages/guided-mode/GuidedStart.js b/src/renderer/src/stories/pages/guided-mode/GuidedStart.js index 666742215..43cb4f2cc 100644 --- a/src/renderer/src/stories/pages/guided-mode/GuidedStart.js +++ b/src/renderer/src/stories/pages/guided-mode/GuidedStart.js @@ -104,8 +104,7 @@ export class GuidedStartPage extends Page { Although not required to use the GUIDE, you can learn more about the NWB conversion process in the neuroconv documentation page + >neuroconv documentation page. `, })} diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 636b4faa9..53ddb4065 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -84,7 +84,6 @@ export class GuidedMetadataPage extends ManagedPage { } beforeSave = () => { - console.log(this.localState.results, this.info.globalState.results); merge(this.localState.results, this.info.globalState.results); }; @@ -195,22 +194,21 @@ export class GuidedMetadataPage extends ManagedPage { // Set most Ophys tables to have minItems / maxItems equal (i.e. no editing possible) drillSchemaProperties( resolvedSchema, - (path, schema, target, isPatternProperties) => { + (path, schema, target, isPatternProperties, parentSchema) => { if (path[0] === "Ophys") { const name = path.slice(-1)[0]; - if (isPatternProperties) { - schema.minItems = schema.maxItems = Object.values(resolveFromPath(path, results)).length; - return; - } + if (isPatternProperties) + return (schema.minItems = schema.maxItems = + Object.values(resolveFromPath(path, results)).length); if (schema.type === "array") { - if ( - name !== "Device" && - target && - name in target // Skip unresolved deep in pattern properties - ) { - schema.minItems = schema.maxItems = target[name].length; + if (name !== "Device" && target) { + if (name in target) + schema.minItems = schema.maxItems = target[name].length; // Skip unresolved deep in pattern properties) + // Remove Ophys requirements if left initially undefined + else if (parentSchema.required.includes(name)) + parentSchema.required = parentSchema.required.filter((n) => n !== name); } } } diff --git a/src/renderer/src/stories/pages/guided-mode/data/utils.js b/src/renderer/src/stories/pages/guided-mode/data/utils.js index dafaa89ca..6593aba50 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/utils.js +++ b/src/renderer/src/stories/pages/guided-mode/data/utils.js @@ -60,7 +60,7 @@ export function drillSchemaProperties(schema = {}, callback, target, path = [], const info = patternProperties[regexp]; const updatedPath = [...path, regexp]; callback(updatedPath, info, undefined, true); - drillSchemaProperties(info, callback, undefined, updatedPath, true); + drillSchemaProperties(info, callback, undefined, updatedPath, true, schema); } for (let name in properties) { @@ -70,7 +70,7 @@ export function drillSchemaProperties(schema = {}, callback, target, path = [], const updatedPath = [...path, name]; - callback(updatedPath, info, target); + callback(updatedPath, info, target, undefined, schema); drillSchemaProperties(info, callback, target?.[name], updatedPath, inPatternProperties); } diff --git a/src/renderer/src/stories/pages/guided-mode/setup/GuidedSubjects.js b/src/renderer/src/stories/pages/guided-mode/setup/GuidedSubjects.js index 82e4f8c7b..44a0dfbb0 100644 --- a/src/renderer/src/stories/pages/guided-mode/setup/GuidedSubjects.js +++ b/src/renderer/src/stories/pages/guided-mode/setup/GuidedSubjects.js @@ -134,7 +134,6 @@ export class GuidedSubjectsPage extends Page { this.notify(`${header(name)} has been overridden with a global value.`, "warning", 3000); }, onUpdate: () => { - console.log("UPDATED!"); this.unsavedUpdates = "conversions"; }, validateOnChange: (localPath, parent, v) => { diff --git a/src/renderer/src/stories/table/Cell.ts b/src/renderer/src/stories/table/Cell.ts index 86fe647a1..1cda87fa5 100644 --- a/src/renderer/src/stories/table/Cell.ts +++ b/src/renderer/src/stories/table/Cell.ts @@ -110,8 +110,6 @@ export class TableCell extends LitElement { } set value(value) { - if (!value) value = [] - if (this.input) this.input.set(renderValue(value, this.schema)) // Allow null to be set directly this.#value = this.input ? this.input.getValue() // Ensure all operations are undoable / value is coerced From 472d87008dcd6536f05375631cd034d0cef3abab Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Tue, 27 Feb 2024 12:38:37 -0800 Subject: [PATCH 6/6] Add README badges (#624) Co-authored-by: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> --- .github/workflows/Build-and-deploy-mac.yml | 2 +- .github/workflows/Build-and-deploy-win.yml | 2 +- .../pyflask-build-and-dist-tests.yml | 2 +- .github/workflows/testing-external.yml | 4 +-- .github/workflows/testing.yml | 4 +-- README.md | 27 +++++++++++++------ 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.github/workflows/Build-and-deploy-mac.yml b/.github/workflows/Build-and-deploy-mac.yml index de198a81a..3a4955d5b 100644 --- a/.github/workflows/Build-and-deploy-mac.yml +++ b/.github/workflows/Build-and-deploy-mac.yml @@ -1,4 +1,4 @@ -name: Build-and-deploy-mac +name: Mac Release run-name: ${{ github.actor }} is building a MAC release for NWB GUIDE on: diff --git a/.github/workflows/Build-and-deploy-win.yml b/.github/workflows/Build-and-deploy-win.yml index d51fa59ea..3d4dbb249 100644 --- a/.github/workflows/Build-and-deploy-win.yml +++ b/.github/workflows/Build-and-deploy-win.yml @@ -1,4 +1,4 @@ -name: Build-and-deploy-win +name: Windows Release run-name: ${{ github.actor }} is building a Windows release for NWB GUIDE on: diff --git a/.github/workflows/pyflask-build-and-dist-tests.yml b/.github/workflows/pyflask-build-and-dist-tests.yml index 82d487cbd..e967b7f78 100644 --- a/.github/workflows/pyflask-build-and-dist-tests.yml +++ b/.github/workflows/pyflask-build-and-dist-tests.yml @@ -1,4 +1,4 @@ -name: PyFlask build and distributable tests +name: Build Tests — Flask on: schedule: - cron: "0 16 * * *" # Daily at noon EST diff --git a/.github/workflows/testing-external.yml b/.github/workflows/testing-external.yml index 711a81653..2b44c9580 100644 --- a/.github/workflows/testing-external.yml +++ b/.github/workflows/testing-external.yml @@ -1,4 +1,4 @@ -name: External Tests +name: Dev Tests (Live Services) on: schedule: - cron: "0 16 * * *" # Daily at noon EST @@ -13,7 +13,7 @@ env: jobs: testing: - name: External tests on ${{ matrix.os }} + name: Dev tests with live services on ${{ matrix.os }} runs-on: ${{ matrix.os }} defaults: run: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 6c6fbad4c..203b55f5c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,4 +1,4 @@ -name: Back-end Tests +name: Dev Tests on: schedule: - cron: "0 16 * * *" # Daily at noon EST @@ -13,7 +13,7 @@ env: jobs: testing: - name: Back-end tests on ${{ matrix.os }} + name: Dev tests on ${{ matrix.os }} runs-on: ${{ matrix.os }} defaults: run: diff --git a/README.md b/README.md index 5dec8e785..c0eaf5f4e 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,28 @@

NeuroConv logo

NWB Graphical User Interface for Data Entry

+

+ Full Tests + Full Tests with External Dependencies + Build and Distributable Tests + codecov + Documentation + License: MIT +

+

+ Mac Build + Windows Build +

+

+ Python code style: black + JavaScript code style: prettier +

+

+ NWB Slack +

- NWB GUIDE is a desktop app that provides a no-code user interface for converting neurophysiology data to NWB. -

- - Watch the video - -

- - ## Installation See the installation instructions in our [documentation](https://nwb-guide.readthedocs.io/en/latest/installation.html).