diff --git a/.github/workflows/Build-and-deploy-mac.yml b/.github/workflows/Build-and-deploy-mac.yml index de198a81a..3a4955d5b 100644 --- a/.github/workflows/Build-and-deploy-mac.yml +++ b/.github/workflows/Build-and-deploy-mac.yml @@ -1,4 +1,4 @@ -name: Build-and-deploy-mac +name: Mac Release run-name: ${{ github.actor }} is building a MAC release for NWB GUIDE on: diff --git a/.github/workflows/Build-and-deploy-win.yml b/.github/workflows/Build-and-deploy-win.yml index d51fa59ea..3d4dbb249 100644 --- a/.github/workflows/Build-and-deploy-win.yml +++ b/.github/workflows/Build-and-deploy-win.yml @@ -1,4 +1,4 @@ -name: Build-and-deploy-win +name: Windows Release run-name: ${{ github.actor }} is building a Windows release for NWB GUIDE on: diff --git a/.github/workflows/pyflask-build-and-dist-tests.yml b/.github/workflows/pyflask-build-and-dist-tests.yml index 82d487cbd..e967b7f78 100644 --- a/.github/workflows/pyflask-build-and-dist-tests.yml +++ b/.github/workflows/pyflask-build-and-dist-tests.yml @@ -1,4 +1,4 @@ -name: PyFlask build and distributable tests +name: Build Tests — Flask on: schedule: - cron: "0 16 * * *" # Daily at noon EST diff --git a/.github/workflows/testing-external.yml b/.github/workflows/testing-external.yml new file mode 100644 index 000000000..2b44c9580 --- /dev/null +++ b/.github/workflows/testing-external.yml @@ -0,0 +1,93 @@ +name: Dev Tests (Live Services) +on: + schedule: + - cron: "0 16 * * *" # Daily at noon EST + pull_request: + +concurrency: # Cancel previous workflows on the same pull request + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CACHE_NUMBER: 2 # increase to reset cache manually + +jobs: + testing: + name: Dev tests with live services on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + include: + - os: ubuntu-latest + label: environments/environment-Linux.yml + + - os: macos-latest + label: environments/environment-Mac.yml + + - os: windows-latest + label: environments/environment-Windows.yml + + + steps: + - uses: actions/checkout@v3 + - run: git fetch --prune --unshallow --tags + + # see https://github.com/conda-incubator/setup-miniconda#caching-environments + - name: Setup Mambaforge + uses: conda-incubator/setup-miniconda@v2 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: nwb-guide + use-mamba: true + + - name: Set cache date + id: get-date + run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache Conda env + uses: actions/cache@v2 + with: + path: ${{ env.CONDA }}/envs + key: conda-${{ runner.os }}-${{ runner.arch }}-${{steps.get-date.outputs.today }}-${{ hashFiles(matrix.label) }}-${{ env.CACHE_NUMBER }} + id: cache + + - if: steps.cache.outputs.cache-hit != 'true' + name: Create and activate environment + run: mamba env update -n nwb-guide -f ${{ matrix.label }} + + - name: Use Node.js 18 + uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Install GUIDE + run: npm ci + + - name: Create env file + run: | + touch .env + echo DANDI_STAGING_API_KEY=${{ secrets.DANDI_STAGING_API_KEY }} >> .env + + - if: matrix.os != 'ubuntu-latest' + name: Run tests + run: npm run coverage:app + + - if: matrix.os == 'ubuntu-latest' + name: Run tests with xvfb + run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run coverage:app + + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + fail_ci_if_error: true diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 811308008..5cf41c2b7 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,4 +1,4 @@ -name: Back-end Tests +name: Dev Tests on: schedule: - cron: "0 16 * * *" # Daily at noon EST @@ -13,7 +13,7 @@ env: jobs: testing: - name: Back-end tests on ${{ matrix.os }} + name: Dev tests on ${{ matrix.os }} runs-on: ${{ matrix.os }} defaults: run: @@ -74,12 +74,6 @@ jobs: - name: Install GUIDE run: npm ci - - - name: Create env file - run: | - touch .env - echo DANDI_STAGING_API_KEY=${{ secrets.DANDI_STAGING_API_KEY }} >> .env - - if: matrix.os != 'ubuntu-latest' name: Run tests uses: nick-fields/retry@v3 diff --git a/README.md b/README.md index 5dec8e785..c0eaf5f4e 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,28 @@

NeuroConv logo

NWB Graphical User Interface for Data Entry

+

+ Full Tests + Full Tests with External Dependencies + Build and Distributable Tests + codecov + Documentation + License: MIT +

+

+ Mac Build + Windows Build +

+

+ Python code style: black + JavaScript code style: prettier +

+

+ NWB Slack +

- NWB GUIDE is a desktop app that provides a no-code user interface for converting neurophysiology data to NWB. -

- - Watch the video - -

- - ## Installation See the installation instructions in our [documentation](https://nwb-guide.readthedocs.io/en/latest/installation.html). diff --git a/docs/developer_guide.rst b/docs/developer_guide.rst index 972ccdb5f..1fabc5dd3 100644 --- a/docs/developer_guide.rst +++ b/docs/developer_guide.rst @@ -24,7 +24,6 @@ Start by cloning the repository Install Python Dependencies ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Install the appropriate Python dependencies for your operating system. **Windows** @@ -51,6 +50,10 @@ Install the appropriate Python dependencies for your operating system. conda env create -f ./environments/environment-Linux.yml + +Activate the Python Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Before starting NWB GUIDE, you'll need to ensure that the Python environment is activated. .. code-block:: bash @@ -58,8 +61,8 @@ Before starting NWB GUIDE, you'll need to ensure that the Python environment is conda activate nwb-guide -Installing JavaScript Dependencies -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Install JavaScript Dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Next, install all JavaScript dependencies based on the `package-lock.json` file. @@ -78,7 +81,6 @@ You can now run the following command to start the application using Electron. npm start - Repo Structure -------------- 1. **src/renderer/src** - Contains all the source code for the frontend diff --git a/docs/installation.rst b/docs/installation.rst index 147de87ad..563dc9e5f 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -5,22 +5,27 @@ Installation Windows ------- -Download and run the `NWB-GUIDE-Setup-vX.Y.Z.exe `_ file and follow all instruction prompts. +Download and run `NWB-GUIDE-x64.exe `_ and follow all instruction prompts. MacOS - Intel ------------- -Download the `NWB-GUIDE-vX.Y.Z.dmg `_ file, which should prompt you to move it into your 'Applications' folder in order to run. +Download `NWB-GUIDE-x64.dmg `_, which should prompt you to move it into your 'Applications' folder in order to run. MacOS - Apple Silicon --------------------- -Download the `NWB-GUIDE-vX.Y.Z-arm64.dmg `_ file, which should prompt you to move it into your 'Applications' folder in order to run. +Download `NWB-GUIDE-arm64.dmg `_, which should prompt you to move it into your 'Applications' folder in order to run. .. note:: Some data formats can have issues using this build of the application. If you encounter errors when using a particular interface, try following the advanced :ref:`Developer Installation instructions` instructions. -Ubuntu ------- +Linux +----- -Please clone the :linux-fix:`linux-fix <>` branch of the NWB GUIDE and follow the :ref:`Developer Installation instructions` on this documentation after the "Clone the Repo" step. +Please clone the :linux-fix:`linux-fix <>` branch of the NWB GUIDE and follow the :ref:`Developer Installation instructions` after the "Clone the Repo" step. + +.. code-block:: bash + + git clone --branch linux-fix https://github.com/NeurodataWithoutBorders/nwb-guide + cd nwb-guide diff --git a/src/renderer/src/stories/InfoBox.js b/src/renderer/src/stories/InfoBox.js index c11b4c54b..3177dd2c5 100644 --- a/src/renderer/src/stories/InfoBox.js +++ b/src/renderer/src/stories/InfoBox.js @@ -1,5 +1,6 @@ import { LitElement, css, html } from "lit"; import { Chevron } from "./Chevron"; +import { unsafeHTML } from "lit/directives/unsafe-html.js"; export class InfoBox extends LitElement { static get styles() { @@ -120,7 +121,9 @@ export class InfoBox extends LitElement { ${new Chevron({ direction: "right" })}
- ${this.content} + ${typeof this.content === "string" ? unsafeHTML(this.content) : this.content}
`; } diff --git a/src/renderer/src/stories/JSONSchemaForm.js b/src/renderer/src/stories/JSONSchemaForm.js index c08b8c1ff..ed318d187 100644 --- a/src/renderer/src/stories/JSONSchemaForm.js +++ b/src/renderer/src/stories/JSONSchemaForm.js @@ -464,7 +464,6 @@ export class JSONSchemaForm extends LitElement { // if (!isValid && allErrors.length && nMissingRequired === allErrors.length) message = `${nMissingRequired} required inputs are not defined.`; - console.log(allErrors); // Check if all inputs are valid if (flaggedInputs.length) { flaggedInputs[0].focus(); diff --git a/src/renderer/src/stories/Search.js b/src/renderer/src/stories/Search.js index 50cd073aa..70e601896 100644 --- a/src/renderer/src/stories/Search.js +++ b/src/renderer/src/stories/Search.js @@ -256,23 +256,37 @@ export class Search extends LitElement { #sortedCategories = []; + getTokens = (input) => + input + .replaceAll(/[^\w\s]/g, "") + .split(" ") + .map((token) => token.trim().toLowerCase()) + .filter((token) => token); + #populate = (input = this.value ?? "") => { const toShow = []; - // Check if the input value matches the label - this.#options.forEach(({ option, label }, i) => { - if (label.toLowerCase().includes(input.toLowerCase()) && !toShow.includes(i)) toShow.push(i); - }); + const inputTokens = this.getTokens(input); - // Check if the input value matches any of the keywords - this.#options.forEach(({ option, keywords = [], structuredKeywords = {} }, i) => { - [...keywords, ...Object.values(structuredKeywords).flat()].forEach((keyword) => { - if (keyword.toLowerCase().includes(input.toLowerCase()) && !toShow.includes(i)) toShow.push(i); - }); + // Check if the input value matches the label or any of the keywords + this.#options.forEach(({ label, keywords, structuredKeywords }, i) => { + const labelTokens = this.getTokens(label); + const allKeywords = [...keywords, ...Object.values(structuredKeywords).flat()]; + const allKeywordTokens = allKeywords.map((keyword) => this.getTokens(keyword)).flat(); + const allTokens = [...labelTokens, ...allKeywordTokens]; + + const result = inputTokens.reduce((acc, token) => { + for (let subtoken of allTokens) { + if (subtoken.startsWith(token) && !toShow.includes(i)) return (acc += 1); + } + return acc; + }, 0); + + if (result === inputTokens.length) toShow.push(i); }); this.#options.forEach(({ option }, i) => { - if (toShow.includes(i)) { + if (toShow.includes(i) || !inputTokens.length) { option.removeAttribute("hidden"); } else { option.setAttribute("hidden", ""); @@ -285,7 +299,7 @@ export class Search extends LitElement { else element.removeAttribute("hidden"); }); - this.setAttribute("active", !!toShow.length); + this.setAttribute("active", !!toShow.length || !inputTokens.length); this.setAttribute("interacted", true); }; diff --git a/src/renderer/src/stories/pages/guided-mode/GuidedStart.js b/src/renderer/src/stories/pages/guided-mode/GuidedStart.js index 666742215..43cb4f2cc 100644 --- a/src/renderer/src/stories/pages/guided-mode/GuidedStart.js +++ b/src/renderer/src/stories/pages/guided-mode/GuidedStart.js @@ -104,8 +104,7 @@ export class GuidedStartPage extends Page { Although not required to use the GUIDE, you can learn more about the NWB conversion process in the neuroconv documentation page + >neuroconv documentation page. `, })} diff --git a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js index 636b4faa9..53ddb4065 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js +++ b/src/renderer/src/stories/pages/guided-mode/data/GuidedMetadata.js @@ -84,7 +84,6 @@ export class GuidedMetadataPage extends ManagedPage { } beforeSave = () => { - console.log(this.localState.results, this.info.globalState.results); merge(this.localState.results, this.info.globalState.results); }; @@ -195,22 +194,21 @@ export class GuidedMetadataPage extends ManagedPage { // Set most Ophys tables to have minItems / maxItems equal (i.e. no editing possible) drillSchemaProperties( resolvedSchema, - (path, schema, target, isPatternProperties) => { + (path, schema, target, isPatternProperties, parentSchema) => { if (path[0] === "Ophys") { const name = path.slice(-1)[0]; - if (isPatternProperties) { - schema.minItems = schema.maxItems = Object.values(resolveFromPath(path, results)).length; - return; - } + if (isPatternProperties) + return (schema.minItems = schema.maxItems = + Object.values(resolveFromPath(path, results)).length); if (schema.type === "array") { - if ( - name !== "Device" && - target && - name in target // Skip unresolved deep in pattern properties - ) { - schema.minItems = schema.maxItems = target[name].length; + if (name !== "Device" && target) { + if (name in target) + schema.minItems = schema.maxItems = target[name].length; // Skip unresolved deep in pattern properties) + // Remove Ophys requirements if left initially undefined + else if (parentSchema.required.includes(name)) + parentSchema.required = parentSchema.required.filter((n) => n !== name); } } } diff --git a/src/renderer/src/stories/pages/guided-mode/data/utils.js b/src/renderer/src/stories/pages/guided-mode/data/utils.js index dafaa89ca..6593aba50 100644 --- a/src/renderer/src/stories/pages/guided-mode/data/utils.js +++ b/src/renderer/src/stories/pages/guided-mode/data/utils.js @@ -60,7 +60,7 @@ export function drillSchemaProperties(schema = {}, callback, target, path = [], const info = patternProperties[regexp]; const updatedPath = [...path, regexp]; callback(updatedPath, info, undefined, true); - drillSchemaProperties(info, callback, undefined, updatedPath, true); + drillSchemaProperties(info, callback, undefined, updatedPath, true, schema); } for (let name in properties) { @@ -70,7 +70,7 @@ export function drillSchemaProperties(schema = {}, callback, target, path = [], const updatedPath = [...path, name]; - callback(updatedPath, info, target); + callback(updatedPath, info, target, undefined, schema); drillSchemaProperties(info, callback, target?.[name], updatedPath, inPatternProperties); } diff --git a/src/renderer/src/stories/pages/guided-mode/setup/GuidedSubjects.js b/src/renderer/src/stories/pages/guided-mode/setup/GuidedSubjects.js index 82e4f8c7b..44a0dfbb0 100644 --- a/src/renderer/src/stories/pages/guided-mode/setup/GuidedSubjects.js +++ b/src/renderer/src/stories/pages/guided-mode/setup/GuidedSubjects.js @@ -134,7 +134,6 @@ export class GuidedSubjectsPage extends Page { this.notify(`${header(name)} has been overridden with a global value.`, "warning", 3000); }, onUpdate: () => { - console.log("UPDATED!"); this.unsavedUpdates = "conversions"; }, validateOnChange: (localPath, parent, v) => { diff --git a/src/renderer/src/stories/table/Cell.ts b/src/renderer/src/stories/table/Cell.ts index 86fe647a1..1cda87fa5 100644 --- a/src/renderer/src/stories/table/Cell.ts +++ b/src/renderer/src/stories/table/Cell.ts @@ -110,8 +110,6 @@ export class TableCell extends LitElement { } set value(value) { - if (!value) value = [] - if (this.input) this.input.set(renderValue(value, this.schema)) // Allow null to be set directly this.#value = this.input ? this.input.getValue() // Ensure all operations are undoable / value is coerced diff --git a/tests/e2e.test.ts b/tests/e2e.test.ts index 21b18880b..66ea5650b 100644 --- a/tests/e2e.test.ts +++ b/tests/e2e.test.ts @@ -141,7 +141,7 @@ describe('E2E Test', () => { expect(existsSync(outputLocation)).toBe(true) - }, 2 * 60 * 1000) // Allow two minutes to create dataset + }) test('Create new pipeline by specifying a name', async () => { @@ -185,7 +185,7 @@ describe('E2E Test', () => { // Advance to formats page await toNextPage('structure') - }, 10 * 1000) + }) test('Specify data formats', async () => { @@ -227,7 +227,7 @@ describe('E2E Test', () => { await toNextPage('locate') - }, 10 * 1000) + }) test('Locate all your source data programmatically', async () => { @@ -319,7 +319,7 @@ describe('E2E Test', () => { await toNextPage('sourcedata') - }, 10 * 1000) + }) test('Review source data information', async () => { @@ -342,7 +342,7 @@ describe('E2E Test', () => { await toNextPage('inspect') - }, 30 * 1000) // Wait for conversion preview to complete + }) // Wait for conversion preview to complete test('Review NWB Inspector output', async () => { @@ -358,7 +358,7 @@ describe('E2E Test', () => { if (skipUpload) await toHome() - }, 60 * 1000) // Wait for full conversion to complete + }) // Wait for full conversion to complete const uploadDescribe = skipUpload ? describe.skip: describe @@ -402,7 +402,7 @@ describe('E2E Test', () => { await toNextPage('review') - }, 3 * 60 * 1000) // Wait for upload to finish (~2min on M2) + }) // Wait for upload to finish (~2min on M2) test('Review upload results', async () => { diff --git a/vite.config.js b/vite.config.js index 13ae4ac38..747d09a47 100644 --- a/vite.config.js +++ b/vite.config.js @@ -5,5 +5,6 @@ export default defineConfig({ test: { environment: "jsdom", setupFiles: ["dotenv/config"], + testTimeout: 3 * 60 * 1000, }, });