diff --git a/.github/workflows/example_data_cache.yml b/.github/workflows/example_data_cache.yml index e27070078c..5a2b6d9b82 100644 --- a/.github/workflows/example_data_cache.yml +++ b/.github/workflows/example_data_cache.yml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: python-version: ["3.12"] - os: [ubuntu-latest, macos-latest, macos-13] #, windows-latest] + os: [ubuntu-latest, macos-latest, macos-13, windows-latest] steps: diff --git a/.github/workflows/testing_pipelines.yml b/.github/workflows/testing_pipelines.yml index 1a4ca41d6f..d2eb48e9d9 100644 --- a/.github/workflows/testing_pipelines.yml +++ b/.github/workflows/testing_pipelines.yml @@ -26,8 +26,8 @@ jobs: - os: macos-13 # Mac x64 runner label: environments/environment-MAC-intel.yml -# - os: windows-latest -# label: environments/environment-Windows.yml + - os: windows-latest + label: environments/environment-Windows.yml steps: @@ -95,33 +95,18 @@ jobs: path: ./behavior_testing_data key: behavior-datasets-${{ matrix.os }}-${{ steps.behavior.outputs.HASH_behavior_DATASET }} - - name: Save working directory to environment file - run: echo "GIN_DATA_DIR=$(pwd)" >> .env - if: runner.os != 'Windows' - - - name: Save working directory to environment file (Windows) - run: echo GIN_DATA_DIR=%cd% >> .env - shell: bash - if: runner.os == 'Windows' - - # Display environment file for debugging - - name: Print environment file - run: cat .env - if: runner.os != 'Windows' - - - name: Print environment file - run: type .env - shell: bash - if: runner.os == 'Windows' - # Run pipeline tests - if: matrix.os != 'ubuntu-latest' name: Run tests run: npm run test:pipelines + env: + GIN_DATA_DIRECTORY: ${{ github.workspace }} - if: matrix.os == 'ubuntu-latest' name: Run tests with xvfb run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test:pipelines + env: + GIN_DATA_DIRECTORY: ${{ github.workspace }} - name: Archive Pipeline Test Screenshots if: always() diff --git a/package.json b/package.json index 81aea6245c..64f15c4ec8 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,8 @@ "test:app": "vitest run --exclude \"**/pipelines.test.ts\"", "test:tutorial": "vitest tutorial", "test:pipelines": "vitest pipelines", + "test:progress": "vitest progress", + "test:metadata": "vitest metadata", "test:server": "pytest src/pyflask/tests/ -s -vv", "wait5s": "node -e \"setTimeout(() => process.exit(0),5000)\"", "test:executable": "concurrently -n EXE,TEST --kill-others --success first \"node tests/testPyinstallerExecutable.js --port 3434 --forever\" \"npm run wait5s && pytest src/pyflask/tests/ -s --target http://localhost:3434\"", diff --git a/src/electron/frontend/core/components/pages/guided-mode/data/GuidedPathExpansion.js b/src/electron/frontend/core/components/pages/guided-mode/data/GuidedPathExpansion.js index 20e4e07cc6..af89aea7da 100644 --- a/src/electron/frontend/core/components/pages/guided-mode/data/GuidedPathExpansion.js +++ b/src/electron/frontend/core/components/pages/guided-mode/data/GuidedPathExpansion.js @@ -381,76 +381,71 @@ export class GuidedPathExpansionPage extends Page { const interfaceName = parentPath.slice(-1)[0]; - if (fs) { - const baseDir = form.getFormElement([...parentPath, "base_directory"]); - if (name === "format_string_path") { - if (value && baseDir && !baseDir.value) { - return [ - { - message: html`A base directory must be provided to locate your files.`, - type: "error", - }, - ]; - } + const baseDir = form.getFormElement([...parentPath, "base_directory"]); + if (name === "format_string_path") { + if (value && baseDir && !baseDir.value) { + return [ + { + message: html`A base directory must be provided to locate your files.`, + type: "error", + }, + ]; + } - const base_directory = [...parentPath, "base_directory"].reduce( - (acc, key) => acc[key], - this.form.resolved - ); + const base_directory = [...parentPath, "base_directory"].reduce( + (acc, key) => acc[key], + this.form.resolved + ); - if (!base_directory) return true; // Do not calculate if base is not found + if (!base_directory) return true; // Do not calculate if base is not found - const entry = { base_directory }; + const entry = { base_directory }; - if (value.split(".").length > 1) entry.file_path = value; - else entry.folder_path = value; + if (value.split(".").length > 1) entry.file_path = value; + else entry.folder_path = value; - const results = await run( - `neuroconv/locate`, - { [interfaceName]: entry }, - { swal: false } - ).catch((error) => { + const results = await run(`neuroconv/locate`, { [interfaceName]: entry }, { swal: false }).catch( + (error) => { this.notify(error.message, "error"); throw error; - }); + } + ); - const resolved = []; + const resolved = []; - for (let sub in results) { - for (let ses in results[sub]) { - const source_data = results[sub][ses].source_data[interfaceName]; - const path = source_data.file_path ?? source_data.folder_path; - resolved.push(path.slice(base_directory.length + 1)); - } + for (let sub in results) { + for (let ses in results[sub]) { + const source_data = results[sub][ses].source_data[interfaceName]; + const path = source_data.file_path ?? source_data.folder_path; + resolved.push(path.slice(base_directory.length + 1)); } + } - if (resolved.length === 0) - return [ - { - message: html`No source files found using the provided information.`, - type: "warning", - }, - ]; - + if (resolved.length === 0) return [ { - message: html`

- ✅Source Files Found for - ${interfaceName} -

- ${base_directory} - ${new List({ - items: resolved.map((path) => { - return { value: path }; - }), - editable: false, - })}`, - type: "info", + message: html`No source files found using the provided information.`, + type: "warning", }, ]; - } + + return [ + { + message: html`

+ ✅Source Files Found for ${interfaceName} +

+ ${base_directory} + ${new List({ + items: resolved.map((path) => { + return { value: path }; + }), + editable: false, + })}`, + type: "info", + }, + ]; } }, })); diff --git a/src/electron/frontend/core/components/pages/settings/SettingsPage.js b/src/electron/frontend/core/components/pages/settings/SettingsPage.js index 732ba905de..d5ea234eab 100644 --- a/src/electron/frontend/core/components/pages/settings/SettingsPage.js +++ b/src/electron/frontend/core/components/pages/settings/SettingsPage.js @@ -15,15 +15,9 @@ import { merge, setUndefinedIfNotDeclared } from "../utils"; import { notyf } from "../../../dependencies.js"; import { homeDirectory, testDataFolderPath } from "../../../globals.js"; -import { - SERVER_FILE_PATH, - electron, - path, - port, - fs, - onUpdateAvailable, - onUpdateProgress, -} from "../../../../utils/electron.js"; +import { SERVER_FILE_PATH, electron, path, port, fs } from "../../../../utils/electron.js"; + +import { onUpdateAvailable, onUpdateProgress } from "../../../../utils/auto-update.js"; import saveSVG from "../../../../assets/icons/save.svg?raw"; import folderSVG from "../../../../assets/icons/folder_open.svg?raw"; diff --git a/src/electron/frontend/core/globals.js b/src/electron/frontend/core/globals.js index fa3fc0e26e..0ca85c1a23 100644 --- a/src/electron/frontend/core/globals.js +++ b/src/electron/frontend/core/globals.js @@ -1,9 +1,11 @@ -import { app, path, crypto, isElectron } from "../utils/electron.js"; +import { os, path, crypto, isElectron, isTestEnvironment } from "../utils/electron.js"; import paths from "../../../paths.config.json" assert { type: "json" }; import supportedInterfaces from "../../../supported_interfaces.json" assert { type: "json" }; +export { isTestEnvironment }; + export const joinPath = (...args) => (path ? path.join(...args) : args.filter((str) => str).join("/")); export let runOnLoad = (fn) => { @@ -17,24 +19,27 @@ export const reloadPageToHome = () => { }; // Clear all query params // Filesystem Management -const root = globalThis?.process?.env?.VITEST ? joinPath(paths.root, ".test") : paths.root; -export const homeDirectory = app?.getPath("home") ?? ""; -export const appDirectory = homeDirectory ? joinPath(homeDirectory, root) : ""; -export const guidedProgressFilePath = appDirectory ? joinPath(appDirectory, ...paths.subfolders.progress) : ""; +const root = isTestEnvironment ? joinPath(paths.root, ".test") : paths.root; + +export const homeDirectory = os ? os.homedir() : "/"; + +export const appDirectory = joinPath(homeDirectory, root); + +export const guidedProgressFilePath = joinPath(appDirectory, ...paths.subfolders.progress); -export const previewSaveFolderPath = appDirectory ? joinPath(appDirectory, ...paths.subfolders.preview) : ""; -export const conversionSaveFolderPath = appDirectory ? joinPath(appDirectory, ...paths.subfolders.conversions) : ""; +export const previewSaveFolderPath = joinPath(appDirectory, ...paths.subfolders.preview); +export const conversionSaveFolderPath = joinPath(appDirectory, ...paths.subfolders.conversions); -export const testDataFolderPath = appDirectory ? joinPath(appDirectory, ...paths.subfolders.testdata) : ""; +export const testDataFolderPath = joinPath(appDirectory, ...paths.subfolders.testdata); // Encryption const IV_LENGTH = 16; const KEY_LENGTH = 32; -export const ENCRYPTION_KEY = appDirectory +export const ENCRYPTION_KEY = isElectron ? Buffer.concat([Buffer.from(appDirectory), Buffer.alloc(KEY_LENGTH)], KEY_LENGTH) - : null; + : ""; -export const ENCRYPTION_IV = crypto ? crypto.randomBytes(IV_LENGTH) : null; +export const ENCRYPTION_IV = isElectron ? crypto.randomBytes(IV_LENGTH) : ""; // Storybook export const isStorybook = window.location.href.includes("iframe.html"); diff --git a/src/electron/frontend/core/index.ts b/src/electron/frontend/core/index.ts index 08415c3e10..c7d0a48973 100644 --- a/src/electron/frontend/core/index.ts +++ b/src/electron/frontend/core/index.ts @@ -1,5 +1,7 @@ import "./pages.js" import { isElectron, electron } from '../utils/electron.js' +import { isTestEnvironment } from './globals.js' + const { ipcRenderer } = electron; import { Dashboard } from './components/Dashboard.js' @@ -55,8 +57,6 @@ async function isOnline() { statusBar.items[1].status = true - const isTestEnvironment = globalThis?.process?.env?.VITEST - if (isTestEnvironment) return notyf.open({ diff --git a/src/electron/frontend/core/progress/index.js b/src/electron/frontend/core/progress/index.js index 761ab85348..aede8554b0 100644 --- a/src/electron/frontend/core/progress/index.js +++ b/src/electron/frontend/core/progress/index.js @@ -8,6 +8,7 @@ import { ENCRYPTION_KEY, ENCRYPTION_IV, } from "../globals.js"; + import { fs, crypto } from "../../utils/electron.js"; import { joinPath, runOnLoad } from "../globals"; @@ -87,8 +88,7 @@ class GlobalAppConfig { save() { const encoded = encodeObject(this.data); - if (fs) fs.writeFileSync(this.path, JSON.stringify(encoded, null, 2)); - else localStorage.setItem(this.path, JSON.stringify(encoded)); + fs.writeFileSync(this.path, JSON.stringify(encoded, null, 2)); } } @@ -115,7 +115,7 @@ export const save = (page, overrides = {}) => { }; export const getEntries = () => { - if (fs && !fs.existsSync(guidedProgressFilePath)) fs.mkdirSync(guidedProgressFilePath, { recursive: true }); //Check if progress folder exists. If not, create it. + if (!fs.existsSync(guidedProgressFilePath)) fs.mkdirSync(guidedProgressFilePath, { recursive: true }); //Check if progress folder exists. If not, create it. const progressFiles = fs ? fs.readdirSync(guidedProgressFilePath) : Object.keys(localStorage); return progressFiles.filter((path) => path.slice(-5) === ".json"); }; diff --git a/src/electron/frontend/core/progress/operations.js b/src/electron/frontend/core/progress/operations.js index 8d60e238ef..3154e349d8 100644 --- a/src/electron/frontend/core/progress/operations.js +++ b/src/electron/frontend/core/progress/operations.js @@ -7,17 +7,13 @@ export const remove = (name) => { const progressFilePathToDelete = joinPath(guidedProgressFilePath, name + ".json"); //delete the progress file - if (fs) { - if (fs.existsSync(progressFilePathToDelete)) fs.unlinkSync(progressFilePathToDelete); - } else localStorage.removeItem(progressFilePathToDelete); + if (fs.existsSync(progressFilePathToDelete)) fs.unlinkSync(progressFilePathToDelete); - if (fs) { - // delete default preview location - fs.rmSync(joinPath(previewSaveFolderPath, name), { recursive: true, force: true }); + // delete default preview location + fs.rmSync(joinPath(previewSaveFolderPath, name), { recursive: true, force: true }); - // delete default conversion location - fs.rmSync(joinPath(conversionSaveFolderPath, name), { recursive: true, force: true }); - } + // delete default conversion location + fs.rmSync(joinPath(conversionSaveFolderPath, name), { recursive: true, force: true }); return true; }; diff --git a/src/electron/frontend/core/progress/update.js b/src/electron/frontend/core/progress/update.js index c621427b67..f07ea80b19 100644 --- a/src/electron/frontend/core/progress/update.js +++ b/src/electron/frontend/core/progress/update.js @@ -15,11 +15,7 @@ export const rename = (newDatasetName, previousDatasetName) => { // update old progress file with new dataset name const oldProgressFilePath = `${guidedProgressFilePath}/${previousDatasetName}.json`; const newProgressFilePath = `${guidedProgressFilePath}/${newDatasetName}.json`; - if (fs) fs.renameSync(oldProgressFilePath, newProgressFilePath); - else { - localStorage.setItem(newProgressFilePath, localStorage.getItem(oldProgressFilePath)); - localStorage.removeItem(oldProgressFilePath); - } + fs.renameSync(oldProgressFilePath, newProgressFilePath); } else throw new Error("No previous project name provided"); }; @@ -56,9 +52,9 @@ export const updateFile = (projectName, callback) => { var guidedFilePath = joinPath(guidedProgressFilePath, projectName + ".json"); + console.log(guidedProgressFilePath); + // Save the file through the available mechanisms - if (fs) { - if (!fs.existsSync(guidedProgressFilePath)) fs.mkdirSync(guidedProgressFilePath, { recursive: true }); //create progress folder if one does not exist - fs.writeFileSync(guidedFilePath, JSON.stringify(data, null, 2)); - } else localStorage.setItem(guidedFilePath, JSON.stringify(data)); + if (!fs.existsSync(guidedProgressFilePath)) fs.mkdirSync(guidedProgressFilePath, { recursive: true }); //create progress folder if one does not exist + fs.writeFileSync(guidedFilePath, JSON.stringify(data, null, 2)); }; diff --git a/src/electron/frontend/core/server/index.ts b/src/electron/frontend/core/server/index.ts index e8b69b2292..aecc5b3f18 100644 --- a/src/electron/frontend/core/server/index.ts +++ b/src/electron/frontend/core/server/index.ts @@ -1,6 +1,8 @@ -import { isElectron, electron, app, port } from '../../utils/electron.js' +import { isElectron, electron, app } from '../../utils/electron.js' const { ipcRenderer } = electron; +import { isTestEnvironment } from '../globals.js' + import { notyf, } from '../dependencies.js' @@ -35,7 +37,6 @@ export async function pythonServerOpened() { if (openPythonStatusNotyf) notyf.dismiss(openPythonStatusNotyf) - const isTestEnvironment = globalThis?.process?.env?.VITEST if (isTestEnvironment) return openPythonStatusNotyf = notyf.open({ diff --git a/src/electron/frontend/utils/auto-update.js b/src/electron/frontend/utils/auto-update.js new file mode 100644 index 0000000000..4b4452ebde --- /dev/null +++ b/src/electron/frontend/utils/auto-update.js @@ -0,0 +1,25 @@ +let updateAvailable = false; +const updateAvailableCallbacks = []; +export const onUpdateAvailable = (callback) => { + if (updateAvailable) callback(updateAvailable); + else updateAvailableCallbacks.push(callback); +}; + +let updateProgress = null; + +const updateProgressCallbacks = []; +export const onUpdateProgress = (callback) => { + if (updateProgress) callback(updateProgress); + else updateProgressCallbacks.push(callback); +}; + +export const registerUpdateProgress = (info) => { + updateProgress = info; + updateProgressCallbacks.forEach((cb) => cb(info)); +}; + +export const registerUpdate = (info) => { + updateAvailable = info; + document.body.setAttribute("data-update-available", JSON.stringify(info)); + updateAvailableCallbacks.forEach((cb) => cb(info)); +}; diff --git a/src/electron/frontend/utils/electron.js b/src/electron/frontend/utils/electron.js index e8c6f2e4a0..8531cfdb0a 100644 --- a/src/electron/frontend/utils/electron.js +++ b/src/electron/frontend/utils/electron.js @@ -1,86 +1,52 @@ +import { registerUpdate, registerUpdateProgress } from "./auto-update.js"; import { updateURLParams } from "./url.js"; -var userAgent = navigator.userAgent.toLowerCase(); -export const isElectron = userAgent.indexOf(" electron/") > -1; - -export let port = 4242; -export let SERVER_FILE_PATH = ""; -export const electron = globalThis.electron ?? {}; // ipcRenderer, remote, shell, etc. -export let fs = null; -export let os = null; -export let remote = {}; -export let app = null; -export let path = null; -export let log = null; -export let crypto = null; +export const isTestEnvironment = globalThis?.process?.env?.VITEST; -let updateAvailable = false; -const updateAvailableCallbacks = []; -export const onUpdateAvailable = (callback) => { - if (updateAvailable) callback(updateAvailable); - else updateAvailableCallbacks.push(callback); -}; +const userAgent = navigator.userAgent.toLowerCase(); +export const isElectron = userAgent.indexOf(" electron/") > -1; -let updateProgress = null; +const hasNodeAccess = isElectron || isTestEnvironment; -const updateProgressCallbacks = []; -export const onUpdateProgress = (callback) => { - if (updateProgress) callback(updateProgress); - else updateProgressCallbacks.push(callback); -}; +export const electron = globalThis.electron ?? {}; // ipcRenderer, remote, shell, etc. -export const registerUpdateProgress = (info) => { - updateProgress = info; - updateProgressCallbacks.forEach((cb) => cb(info)); -}; +// Node Modules +export const fs = hasNodeAccess && require("fs-extra"); // File System +export const os = hasNodeAccess && require("os"); +export const crypto = hasNodeAccess && require("crypto"); +export const path = hasNodeAccess && require("path"); -const registerUpdate = (info) => { - updateAvailable = info; - document.body.setAttribute("data-update-available", JSON.stringify(info)); - updateAvailableCallbacks.forEach((cb) => cb(info)); -}; +// Remote Electron Modules +export const remote = isElectron ? require("@electron/remote") : {}; +export const app = remote.app; -// Used in tests -try { - crypto = require("crypto"); -} catch {} +// Electron Information +export const port = isElectron ? electron.ipcRenderer.sendSync("get-port") : 4242; +export const SERVER_FILE_PATH = isElectron ? electron.ipcRenderer.sendSync("get-server-file-path") : ""; +// Link the renderer to the main process if (isElectron) { - try { - fs = require("fs-extra"); // File System - os = require("os"); - crypto = require("crypto"); - remote = require("@electron/remote"); - app = remote.app; - - electron.ipcRenderer.on("fileOpened", (info, filepath) => { - updateURLParams({ file: filepath }); - const dashboard = document.querySelector("nwb-dashboard"); - const activePage = dashboard.getAttribute("activePage"); - if (activePage === "preview") dashboard.requestUpdate(); - else dashboard.setAttribute("activePage", "preview"); - }); - - ["log", "warn", "error"].forEach((method) => - electron.ipcRenderer.on(`console.${method}`, (_, ...args) => console[method](`[main-process]:`, ...args)) - ); - - electron.ipcRenderer.on(`checking-for-update`, (_, ...args) => console.log(`[Update]:`, ...args)); + electron.ipcRenderer.on("fileOpened", (info, filepath) => { + updateURLParams({ file: filepath }); + const dashboard = document.querySelector("nwb-dashboard"); + const activePage = dashboard.getAttribute("activePage"); + if (activePage === "preview") dashboard.requestUpdate(); + else dashboard.setAttribute("activePage", "preview"); + }); - electron.ipcRenderer.on(`update-available`, (_, info) => (info ? registerUpdate(info) : "")); + ["log", "warn", "error"].forEach((method) => + electron.ipcRenderer.on(`console.${method}`, (_, ...args) => console[method](`[main-process]:`, ...args)) + ); - electron.ipcRenderer.on(`update-progress`, (_, info) => registerUpdateProgress(info)); - electron.ipcRenderer.on(`update-complete`, (_, ...args) => console.log(`[Update]:`, ...args)); + console.log("User OS:", os.type(), os.platform(), "version:", os.release()); - electron.ipcRenderer.on(`update-error`, (_, ...args) => console.log(`[Update]:`, ...args)); + // Update Handling + electron.ipcRenderer.on(`checking-for-update`, (_, ...args) => console.log(`[Update]:`, ...args)); - port = electron.ipcRenderer.sendSync("get-port"); - console.log("User OS:", os.type(), os.platform(), "version:", os.release()); + electron.ipcRenderer.on(`update-available`, (_, info) => (info ? registerUpdate(info) : "")); - SERVER_FILE_PATH = electron.ipcRenderer.sendSync("get-server-file-path"); + electron.ipcRenderer.on(`update-progress`, (_, info) => registerUpdateProgress(info)); + electron.ipcRenderer.on(`update-complete`, (_, ...args) => console.log(`[Update]:`, ...args)); - path = require("path"); - } catch (error) { - console.error("Electron API access failed —", error); - } -} else console.warn("Electron API is blocked for web builds"); + electron.ipcRenderer.on(`update-error`, (_, ...args) => console.log(`[Update]:`, ...args)); +} diff --git a/tests/e2e/pipelines.test.ts b/tests/e2e/pipelines.test.ts index 3c0f41ea88..c775916bc4 100644 --- a/tests/e2e/pipelines.test.ts +++ b/tests/e2e/pipelines.test.ts @@ -15,7 +15,7 @@ import { sleep } from '../puppeteer'; // NOTE: We assume the user has put the GIN data in ~/NWB_GUIDE/test-data/GIN -const testGINPath = process.env.GIN_DATA_DIR ?? join(homedir(), paths.root, 'test-data', 'GIN') +const testGINPath = process.env.GIN_DATA_DIRECTORY ?? join(homedir(), paths.root, 'test-data', 'GIN') console.log('Using test GIN data at:', testGINPath) const pipelineDescribeFn = existsSync(testGINPath) ? describe : describe.skip diff --git a/tests/metadata.test.ts b/tests/metadata.test.ts index 0bb326f31a..4773b4d5ed 100644 --- a/tests/metadata.test.ts +++ b/tests/metadata.test.ts @@ -7,13 +7,12 @@ import baseMetadataSchema from '../src/schemas/base-metadata.schema' import { createMockGlobalState } from './utils' import { Validator } from 'jsonschema' -import { tempPropertyKey, textToArray } from '../src/electron/frontend/core/components/forms/utils' +import { textToArray } from '../src/electron/frontend/core/components/forms/utils' import { updateResultsFromSubjects } from '../src/electron/frontend/core/components/pages/guided-mode/setup/utils' import { JSONSchemaForm } from '../src/electron/frontend/core/components/JSONSchemaForm' import { validateOnChange } from "../src/electron/frontend/core/validation/index.js"; import { SimpleTable } from '../src/electron/frontend/core/components/SimpleTable' -import { JSONSchemaInput } from '../src/electron/frontend/core/components/JSONSchemaInput.js' function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms));