From fb9cad472c41b48173fa4bfd9c730dee7e4ce9b3 Mon Sep 17 00:00:00 2001 From: Alexander Jones Date: Tue, 30 Jul 2024 18:32:52 -0500 Subject: [PATCH 1/7] Bump hed-validator to v3.15.2 --- bids-validator/package.json | 2 +- package-lock.json | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/bids-validator/package.json b/bids-validator/package.json index 117999dc8..f1e96898e 100644 --- a/bids-validator/package.json +++ b/bids-validator/package.json @@ -45,7 +45,7 @@ "date-fns": "^3.6.0", "events": "^3.3.0", "exifreader": "^4.23.3", - "hed-validator": "^3.15.1", + "hed-validator": "^3.15.2", "ignore": "^5.3.1", "is-utf8": "^0.2.1", "jest": "^29.7.0", diff --git a/package-lock.json b/package-lock.json index 5ed2d6805..88e84d99a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -37,7 +37,7 @@ "date-fns": "^3.6.0", "events": "^3.3.0", "exifreader": "^4.23.3", - "hed-validator": "^3.15.1", + "hed-validator": "^3.15.2", "ignore": "^5.3.1", "is-utf8": "^0.2.1", "jest": "^29.7.0", @@ -51,7 +51,7 @@ "pako": "^1.0.6", "path": "^0.12.7", "pluralize": "^8.0.0", - "semver": "7.6.3", + "semver": "^7.6.3", "stream-browserify": "^3.0.0", "table": "^6.8.2", "util": "^0.12.5", @@ -70,8 +70,8 @@ "esbuild-runner": "^2.2.2", "eslint": "^8.57.0", "eslint-config-prettier": "^9.1.0", - "eslint-plugin-prettier": "5.2.1", - "husky": "9.1.1", + "eslint-plugin-prettier": "^5.2.1", + "husky": "^9.1.1", "lockfile": "^1.0.4", "sync-request": "6.1.0" }, @@ -10755,9 +10755,9 @@ } }, "node_modules/hed-validator": { - "version": "3.15.1", - "resolved": "https://registry.npmjs.org/hed-validator/-/hed-validator-3.15.1.tgz", - "integrity": "sha512-3UAzeVF6R+wuZaF11S0pvuDmF5vOzx5LylxthrbgRUOid/RvcAGTJpkJtkP8A7haIL/NI1CTA3ugL0cSGb0LKg==", + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/hed-validator/-/hed-validator-3.15.2.tgz", + "integrity": "sha512-2qpk2EC0ZeqfNabFkMWBsSAX+aEz2d2twt+hJpY8PsiN+7QUsfKm9P8EDumWtO30iTNHHSTsrJtHBkdX63YhsQ==", "dependencies": { "buffer": "^6.0.3", "cross-fetch": "^4.0.0", @@ -24372,11 +24372,11 @@ "esbuild-runner": "^2.2.2", "eslint": "^8.57.0", "eslint-config-prettier": "^9.1.0", - "eslint-plugin-prettier": "5.2.1", + "eslint-plugin-prettier": "^5.2.1", "events": "^3.3.0", "exifreader": "^4.23.3", - "hed-validator": "^3.15.1", - "husky": "9.1.1", + "hed-validator": "^3.15.2", + "husky": "^9.1.1", "ignore": "^5.3.1", "is-utf8": "^0.2.1", "jest": "^29.7.0", @@ -24391,7 +24391,7 @@ "pako": "^1.0.6", "path": "^0.12.7", "pluralize": "^8.0.0", - "semver": "7.6.3", + "semver": "^7.6.3", "stream-browserify": "^3.0.0", "sync-request": "6.1.0", "table": "^6.8.2", @@ -26976,9 +26976,9 @@ } }, "hed-validator": { - "version": "3.15.1", - "resolved": "https://registry.npmjs.org/hed-validator/-/hed-validator-3.15.1.tgz", - "integrity": "sha512-3UAzeVF6R+wuZaF11S0pvuDmF5vOzx5LylxthrbgRUOid/RvcAGTJpkJtkP8A7haIL/NI1CTA3ugL0cSGb0LKg==", + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/hed-validator/-/hed-validator-3.15.2.tgz", + "integrity": "sha512-2qpk2EC0ZeqfNabFkMWBsSAX+aEz2d2twt+hJpY8PsiN+7QUsfKm9P8EDumWtO30iTNHHSTsrJtHBkdX63YhsQ==", "requires": { "buffer": "^6.0.3", "cross-fetch": "^4.0.0", From 6db60f493aa7d3ef7ac5ad8027bb2750589fbc3d Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 30 Jul 2024 19:30:56 -0400 Subject: [PATCH 2/7] feat(files): Give files pointers to parent file trees --- bids-validator/src/files/browser.test.ts | 19 ++++++++++++++--- bids-validator/src/files/browser.ts | 6 ++++-- bids-validator/src/files/deno.ts | 5 +++-- bids-validator/src/files/ignore.ts | 2 +- bids-validator/src/files/nifti.ts | 2 +- .../src/issues/datasetIssues.test.ts | 2 +- bids-validator/src/schema/associations.ts | 3 +-- bids-validator/src/schema/context.ts | 3 +-- bids-validator/src/schema/fixtures.test.ts | 5 +++++ bids-validator/src/tests/simple-dataset.ts | 5 +++++ bids-validator/src/types/file.ts | 21 ------------------- bids-validator/src/types/filetree.ts | 18 +++++++++++++++- bids-validator/src/types/issues.ts | 2 +- bids-validator/src/validators/bids.ts | 3 +-- bids-validator/src/validators/isBidsy.ts | 2 +- 15 files changed, 58 insertions(+), 40 deletions(-) delete mode 100644 bids-validator/src/types/file.ts diff --git a/bids-validator/src/files/browser.test.ts b/bids-validator/src/files/browser.test.ts index 3874da9a2..14246de1e 100644 --- a/bids-validator/src/files/browser.test.ts +++ b/bids-validator/src/files/browser.test.ts @@ -29,7 +29,11 @@ Deno.test('Browser implementation of FileTree', async (t) => { ] const tree = await fileListToTree(files) const expectedTree = new FileTree('', '/', undefined) - expectedTree.files = files.map((f) => new BIDSFileBrowser(f, ignore)) + expectedTree.files = files.map((f) => { + const file = new BIDSFileBrowser(f, ignore) + file.parent = expectedTree + return file + }) assertEquals(tree, expectedTree) }) await t.step('converts a simple FileList with several levels', async () => { @@ -62,9 +66,14 @@ Deno.test('Browser implementation of FileTree', async (t) => { const anatTree = new FileTree('sub-01/anat', 'anat', sub01Tree) expectedTree.files = files .slice(0, 3) - .map((f) => new BIDSFileBrowser(f, ignore)) + .map((f) => { + const file = new BIDSFileBrowser(f, ignore) + file.parent = expectedTree + return file + }) expectedTree.directories.push(sub01Tree) anatTree.files = [new BIDSFileBrowser(files[3], ignore)] + anatTree.files[0].parent = anatTree sub01Tree.directories.push(anatTree) assertEquals(tree, expectedTree) }) @@ -82,7 +91,11 @@ Deno.test('Spread copies of BIDSFileBrowser contain name and path properties', a ] const tree = await fileListToTree(files) const expectedTree = new FileTree('', '/', undefined) - expectedTree.files = files.map((f) => new BIDSFileBrowser(f, ignore)) + expectedTree.files = files.map((f) => { + const file = new BIDSFileBrowser(f, ignore) + file.parent = expectedTree + return file + }) assertEquals(tree, expectedTree) const spreadFile = { ...expectedTree.files[0], evidence: 'test evidence' } assertObjectMatch(spreadFile, { diff --git a/bids-validator/src/files/browser.ts b/bids-validator/src/files/browser.ts index 296c402e4..83a2860e7 100644 --- a/bids-validator/src/files/browser.ts +++ b/bids-validator/src/files/browser.ts @@ -1,5 +1,4 @@ -import { BIDSFile } from '../types/file.ts' -import { FileTree } from '../types/filetree.ts' +import { BIDSFile, FileTree } from '../types/filetree.ts' import { FileIgnoreRules } from './ignore.ts' import { parse, posix, SEPARATOR_PATTERN } from '../deps/path.ts' @@ -11,6 +10,7 @@ export class BIDSFileBrowser implements BIDSFile { #file: File name: string path: string + parent?: FileTree constructor(file: File, ignore: FileIgnoreRules) { this.#file = file @@ -53,6 +53,7 @@ export function fileListToTree(files: File[]): Promise { const fPath = parse(file.path) if (fPath.dir === '/') { // Top level file + file.parent = tree tree.files.push(file) } else { const levels = fPath.dir.split(SEPARATOR_PATTERN).slice(1) @@ -76,6 +77,7 @@ export function fileListToTree(files: File[]): Promise { } } // At the terminal leaf, add files + file.parent = currentLevelTree currentLevelTree.files.push(file) } } diff --git a/bids-validator/src/files/deno.ts b/bids-validator/src/files/deno.ts index 637c21776..749957ec7 100644 --- a/bids-validator/src/files/deno.ts +++ b/bids-validator/src/files/deno.ts @@ -2,8 +2,7 @@ * Deno specific implementation for reading files */ import { basename, join, posix } from '../deps/path.ts' -import { BIDSFile } from '../types/file.ts' -import { FileTree } from '../types/filetree.ts' +import { BIDSFile, FileTree } from '../types/filetree.ts' import { requestReadPermission } from '../setup/requestPermissions.ts' import { FileIgnoreRules, readBidsIgnore } from './ignore.ts' @@ -24,6 +23,7 @@ export class BIDSFileDeno implements BIDSFile { #ignore: FileIgnoreRules name: string path: string + parent?: FileTree #fileInfo?: Deno.FileInfo #datasetAbsPath: string @@ -125,6 +125,7 @@ export async function _readFileTree( posix.join(relativePath, dirEntry.name), ignore, ) + file.parent = tree // For .bidsignore, read in immediately and add the rules if (dirEntry.name === '.bidsignore') { ignore.add(await readBidsIgnore(file)) diff --git a/bids-validator/src/files/ignore.ts b/bids-validator/src/files/ignore.ts index 98e27d5ef..c444014ad 100644 --- a/bids-validator/src/files/ignore.ts +++ b/bids-validator/src/files/ignore.ts @@ -1,4 +1,4 @@ -import { BIDSFile } from '../types/file.ts' +import { BIDSFile } from '../types/filetree.ts' import { default as ignore } from 'npm:ignore@5.2.4' import type { Ignore } from 'npm:ignore@5.2.4' diff --git a/bids-validator/src/files/nifti.ts b/bids-validator/src/files/nifti.ts index 658ced48c..ba45ea8b2 100644 --- a/bids-validator/src/files/nifti.ts +++ b/bids-validator/src/files/nifti.ts @@ -1,5 +1,5 @@ import 'https://raw.githubusercontent.com/rii-mango/NIFTI-Reader-JS/v0.6.4/release/current/nifti-reader-min.js' -import { BIDSFile } from '../types/file.ts' +import { BIDSFile } from '../types/filetree.ts' import { logger } from '../utils/logger.ts' export async function loadHeader(file: BIDSFile) { diff --git a/bids-validator/src/issues/datasetIssues.test.ts b/bids-validator/src/issues/datasetIssues.test.ts index f8ef3627e..08c976690 100644 --- a/bids-validator/src/issues/datasetIssues.test.ts +++ b/bids-validator/src/issues/datasetIssues.test.ts @@ -1,5 +1,5 @@ import { assertEquals, assertObjectMatch } from '../deps/asserts.ts' -import { BIDSFile } from '../types/file.ts' +import { BIDSFile } from '../types/filetree.ts' import { IssueFile } from '../types/issues.ts' import { DatasetIssues } from './datasetIssues.ts' diff --git a/bids-validator/src/schema/associations.ts b/bids-validator/src/schema/associations.ts index 6cb383843..0b97c6320 100644 --- a/bids-validator/src/schema/associations.ts +++ b/bids-validator/src/schema/associations.ts @@ -1,6 +1,5 @@ import { ContextAssociations, ContextAssociationsEvents } from '../types/context.ts' -import { BIDSFile } from '../types/file.ts' -import { FileTree } from '../types/filetree.ts' +import { BIDSFile, FileTree } from '../types/filetree.ts' import { BIDSContext } from './context.ts' import { readEntities } from './entities.ts' import { parseTSV } from '../files/tsv.ts' diff --git a/bids-validator/src/schema/context.ts b/bids-validator/src/schema/context.ts index 1328c0479..7182477ba 100644 --- a/bids-validator/src/schema/context.ts +++ b/bids-validator/src/schema/context.ts @@ -6,8 +6,7 @@ import { ContextNiftiHeader, ContextSubject, } from '../types/context.ts' -import { BIDSFile } from '../types/file.ts' -import { FileTree } from '../types/filetree.ts' +import { BIDSFile, FileTree } from '../types/filetree.ts' import { ColumnsMap } from '../types/columns.ts' import { BIDSEntities, readEntities } from './entities.ts' import { DatasetIssues } from '../issues/datasetIssues.ts' diff --git a/bids-validator/src/schema/fixtures.test.ts b/bids-validator/src/schema/fixtures.test.ts index fea0f6aec..889adc54e 100644 --- a/bids-validator/src/schema/fixtures.test.ts +++ b/bids-validator/src/schema/fixtures.test.ts @@ -40,6 +40,7 @@ export const dataFile = { ignored: false, stream: new ReadableStream(), readBytes: nullReadBytes, + parent: anatFileTree, } anatFileTree.files = [ @@ -52,6 +53,7 @@ anatFileTree.files = [ ignored: false, stream: new ReadableStream(), readBytes: nullReadBytes, + parent: anatFileTree, }, ] @@ -67,6 +69,7 @@ subjectFileTree.files = [ ignored: false, stream: new ReadableStream(), readBytes: nullReadBytes, + parent: subjectFileTree, }, ] subjectFileTree.directories = [sessionFileTree] @@ -80,6 +83,7 @@ stimuliFileTree.files = [...Array(10).keys()].map((i) => ( ignored: false, stream: new ReadableStream(), readBytes: nullReadBytes, + parent: stimuliFileTree, } )) @@ -92,6 +96,7 @@ rootFileTree.files = [ ignored: false, stream: new ReadableStream(), readBytes: nullReadBytes, + parent: rootFileTree, }, ] rootFileTree.directories = [stimuliFileTree, subjectFileTree] diff --git a/bids-validator/src/tests/simple-dataset.ts b/bids-validator/src/tests/simple-dataset.ts index 29faba1b9..07947c287 100644 --- a/bids-validator/src/tests/simple-dataset.ts +++ b/bids-validator/src/tests/simple-dataset.ts @@ -16,6 +16,7 @@ anatFileTree.files = [ ignored: false, stream: new ReadableStream(), readBytes: nullReadBytes, + parent: anatFileTree, }, ] subjectFileTree.files = [] @@ -29,6 +30,7 @@ rootFileTree.files = [ ignored: false, stream: new ReadableStream(), readBytes: nullReadBytes, + parent: rootFileTree, }, { text, @@ -38,6 +40,7 @@ rootFileTree.files = [ ignored: false, stream: new ReadableStream(), readBytes: nullReadBytes, + parent: rootFileTree, }, { text, @@ -47,6 +50,7 @@ rootFileTree.files = [ ignored: false, stream: new ReadableStream(), readBytes: nullReadBytes, + parent: rootFileTree, }, { text, @@ -56,6 +60,7 @@ rootFileTree.files = [ ignored: false, stream: new ReadableStream(), readBytes: nullReadBytes, + parent: rootFileTree, }, ] rootFileTree.directories = [subjectFileTree] diff --git a/bids-validator/src/types/file.ts b/bids-validator/src/types/file.ts deleted file mode 100644 index fc9eb62a6..000000000 --- a/bids-validator/src/types/file.ts +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Abstract validation File for all environments (Deno, Browser, Python) - */ - -// Avoid overloading the default File type -export interface BIDSFile { - // Filename - name: string - // Dataset relative path for the file - path: string - // File size in bytes - size: number - // BIDS ignore status of the file - ignored: boolean - // ReadableStream to file raw contents - stream: ReadableStream - // Resolve stream to decoded utf-8 text - text: () => Promise - // Read a range of bytes - readBytes: (size: number, offset?: number) => Promise -} diff --git a/bids-validator/src/types/filetree.ts b/bids-validator/src/types/filetree.ts index c204551af..3fbc7f1d4 100644 --- a/bids-validator/src/types/filetree.ts +++ b/bids-validator/src/types/filetree.ts @@ -1,7 +1,23 @@ /** * Abstract FileTree for all environments (Deno, Browser, Python) */ -import { BIDSFile } from '../types/file.ts' +export interface BIDSFile { + // Filename + name: string + // Dataset relative path for the file + path: string + // File size in bytes + size: number + // BIDS ignore status of the file + ignored: boolean + // ReadableStream to file raw contents + stream: ReadableStream + // Resolve stream to decoded utf-8 text + text: () => Promise + // Read a range of bytes + readBytes: (size: number, offset?: number) => Promise + parent?: FileTree +} export class FileTree { // Relative path to this FileTree location diff --git a/bids-validator/src/types/issues.ts b/bids-validator/src/types/issues.ts index 1568a36d3..fd132ce8c 100644 --- a/bids-validator/src/types/issues.ts +++ b/bids-validator/src/types/issues.ts @@ -1,4 +1,4 @@ -import { BIDSFile } from './file.ts' +import { BIDSFile } from './filetree.ts' export type Severity = 'warning' | 'error' | 'ignore' diff --git a/bids-validator/src/validators/bids.ts b/bids-validator/src/validators/bids.ts index ca448c99b..b431c3ffd 100644 --- a/bids-validator/src/validators/bids.ts +++ b/bids-validator/src/validators/bids.ts @@ -1,5 +1,5 @@ import { CheckFunction } from '../types/check.ts' -import { FileTree } from '../types/filetree.ts' +import { BIDSFile, FileTree } from '../types/filetree.ts' import { IssueFile } from '../types/issues.ts' import { GenericSchema } from '../types/schema.ts' import { ValidationResult } from '../types/validation-result.ts' @@ -13,7 +13,6 @@ import { filenameValidate } from './filenameValidate.ts' import { DatasetIssues } from '../issues/datasetIssues.ts' import { emptyFile } from './internal/emptyFile.ts' import { BIDSContext, BIDSContextDataset } from '../schema/context.ts' -import { BIDSFile } from '../types/file.ts' import { parseOptions } from '../setup/options.ts' /** diff --git a/bids-validator/src/validators/isBidsy.ts b/bids-validator/src/validators/isBidsy.ts index 3a874b7cc..bc55b70e4 100644 --- a/bids-validator/src/validators/isBidsy.ts +++ b/bids-validator/src/validators/isBidsy.ts @@ -5,7 +5,7 @@ // @ts-nocheck import { BIDSContext } from '../schema/context.ts' import { CheckFunction } from '../../types/check.ts' -import { BIDSFile } from '../types/file.ts' +import { BIDSFile } from '../types/filetree.ts' import { Schema } from '../types/schema.ts' export const isBidsyFilename: CheckFunction = (schema, context) => { From 0c3c40ae6e5d111bec6526002c058e56b4bae339 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 30 Jul 2024 21:16:09 -0400 Subject: [PATCH 3/7] feat(inheritance): Centralize inheritance principle implementation --- bids-validator/src/files/inheritance.ts | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 bids-validator/src/files/inheritance.ts diff --git a/bids-validator/src/files/inheritance.ts b/bids-validator/src/files/inheritance.ts new file mode 100644 index 000000000..0564bbbd6 --- /dev/null +++ b/bids-validator/src/files/inheritance.ts @@ -0,0 +1,47 @@ +import { BIDSFile, FileTree } from '../types/filetree.ts' +import { Context } from '../types/context.ts' +import { readEntities } from '../schema/entities.ts' + +export function* walkBack( + source: BIDSFile, + inherit: boolean = true, + targetExtensions: string[] = ['.json'], + targetSuffix?: string, +): Generator { + const sourceParts = readEntities(source.name) + + targetSuffix = targetSuffix || sourceParts.suffix + + let fileTree = source.parent + while (fileTree) { + const candidates = fileTree.files.filter((file) => { + const { suffix, extension, entities } = readEntities(file.name) + return ( + targetExtensions.includes(extension) && + suffix === targetSuffix && + Object.keys(entities).every((entity) => entities[entity] === sourceParts.entities[entity]) + ) + }) + if (candidates.length > 1) { + const exactMatch = candidates.find((file) => { + const { suffix, extension, entities } = readEntities(file.name) + return Object.keys(sourceParts.entities).every((entity) => + entities[entity] === sourceParts.entities[entity] + ) + }) + if (exactMatch) { + yield exactMatch + } else { + console.warn(` +Multiple candidates detected for '${source.path}' + +${candidates.map((file) => `* ${file.path}`).join('\n')} +`) + } + } else if (candidates.length === 1) { + yield candidates[0] + } + if (!inherit) break + fileTree = fileTree.parent + } +} From 8f682d031f292cb1a69294d6b4155a1e75af110e Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 30 Jul 2024 21:17:56 -0400 Subject: [PATCH 4/7] rf(inheritance): Use common walkBack for building sidecars and associations --- bids-validator/src/schema/associations.ts | 55 ++++------------------ bids-validator/src/schema/context.test.ts | 2 +- bids-validator/src/schema/context.ts | 56 +++++------------------ 3 files changed, 22 insertions(+), 91 deletions(-) diff --git a/bids-validator/src/schema/associations.ts b/bids-validator/src/schema/associations.ts index 0b97c6320..57cdfaa93 100644 --- a/bids-validator/src/schema/associations.ts +++ b/bids-validator/src/schema/associations.ts @@ -4,6 +4,7 @@ import { BIDSContext } from './context.ts' import { readEntities } from './entities.ts' import { parseTSV } from '../files/tsv.ts' import { parseBvalBvec } from '../files/dwi.ts' +import { walkBack } from '../files/inheritance.ts' // type AssociationsLookup = Record { const associations: ContextAssociations = {} - for (const key in associationLookup as typeof associationLookup) { - const { suffix, extensions, inherit } = - associationLookup[key as keyof typeof associationLookup] - const paths = getPaths(fileTree, source, suffix, extensions) - if (paths.length === 0) { - continue - } - if (paths.length > 1) { - // error? - } - // @ts-expect-error - associations[key] = await associationLookup[key].load(paths[0]) - } - return Promise.resolve(associations) -} - -function getPaths( - fileTree: FileTree, - source: BIDSContext, - targetSuffix: string, - targetExtensions: string[], -) { - const validAssociations = fileTree.files.filter((file) => { - const { suffix, extension, entities } = readEntities(file.name) - return ( - targetExtensions.includes(extension) && - suffix === targetSuffix && - Object.keys(entities).every((entity) => { - return ( - entity in source.entities && - entities[entity] === source.entities[entity] - ) - }) - ) - }) - const nextDir = fileTree.directories.find((directory) => { - return source.file.path.startsWith(directory.path) - }) + for (const [key, value] of Object.entries(associationLookup)) { + const { suffix, extensions, inherit, load } = value + const path = walkBack(source, inherit, extensions, suffix).next().value - if (nextDir) { - validAssociations.push( - ...getPaths(nextDir, source, targetSuffix, targetExtensions), - ) + if (path) { + // @ts-expect-error + associations[key] = await load(path) + } } - return validAssociations + return Promise.resolve(associations) } diff --git a/bids-validator/src/schema/context.test.ts b/bids-validator/src/schema/context.test.ts index 958d6f9ca..95dc4ecab 100644 --- a/bids-validator/src/schema/context.test.ts +++ b/bids-validator/src/schema/context.test.ts @@ -5,7 +5,7 @@ import { dataFile, rootFileTree } from './fixtures.test.ts' Deno.test('test context LoadSidecar', async (t) => { const context = new BIDSContext(rootFileTree, dataFile, new DatasetIssues()) - await context.loadSidecar(rootFileTree) + await context.loadSidecar() await t.step('sidecar overwrites correct fields', () => { const { rootOverwrite, subOverwrite } = context.sidecar assert(rootOverwrite === 'anat') diff --git a/bids-validator/src/schema/context.ts b/bids-validator/src/schema/context.ts index 7182477ba..ef5b02a61 100644 --- a/bids-validator/src/schema/context.ts +++ b/bids-validator/src/schema/context.ts @@ -10,6 +10,7 @@ import { BIDSFile, FileTree } from '../types/filetree.ts' import { ColumnsMap } from '../types/columns.ts' import { BIDSEntities, readEntities } from './entities.ts' import { DatasetIssues } from '../issues/datasetIssues.ts' +import { walkBack } from '../files/inheritance.ts' import { parseTSV } from '../files/tsv.ts' import { loadHeader } from '../files/nifti.ts' import { buildAssociations } from './associations.ts' @@ -159,54 +160,19 @@ export class BIDSContext implements Context { } /** - * Crawls fileTree from root to current context file, loading any valid - * json sidecars found. + * Walks the fileTree backwards from the current file to the root, + * loading any valid json sidecars found. + * Earlier (deeper) sidecars take precedence over later ones. */ - async loadSidecar(fileTree?: FileTree) { - if (!fileTree) { - fileTree = this.fileTree - } - const validSidecars = fileTree.files.filter((file) => { - const { suffix, extension, entities } = readEntities(file.name) - return ( - extension === '.json' && - suffix === this.suffix && - Object.keys(entities).every((entity) => { - return ( - entity in this.entities && - entities[entity] === this.entities[entity] - ) - }) - ) - }) - - if (validSidecars.length > 1) { - const exactMatch = validSidecars.find( - (sidecar) => sidecar.path == this.file.path.replace(this.extension, '.json'), - ) - if (exactMatch) { - validSidecars.splice(1) - validSidecars[0] = exactMatch - } else { - logger.warning( - `Multiple sidecar files detected for '${this.file.path}'`, - ) - } - } - - if (validSidecars.length === 1) { - const json = await validSidecars[0] + async loadSidecar() { + const sidecars = walkBack(this.file) + for (const file of sidecars) { + const json = await file .text() .then((text) => JSON.parse(text)) .catch((error) => {}) - this.sidecar = { ...this.sidecar, ...json } - Object.keys(json).map((x) => this.sidecarKeyOrigin[x] = validSidecars[0].path) - } - const nextDir = fileTree.directories.find((directory) => { - return this.file.path.startsWith(`${directory.path}/`) - }) - if (nextDir) { - await this.loadSidecar(nextDir) + this.sidecar = { ...json, ...this.sidecar } + Object.keys(json).map((x) => this.sidecarKeyOrigin[x] ??= file.path) } } @@ -239,7 +205,7 @@ export class BIDSContext implements Context { } async loadAssociations(): Promise { - this.associations = await buildAssociations(this.fileTree, this) + this.associations = await buildAssociations(this.file) return } From a7519cf059ab7824dec0938cd6aa744dc3ad3c5a Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 31 Jul 2024 02:22:19 -0400 Subject: [PATCH 5/7] feat(deps): Reexport isCompressed/readHeader from nifti-reader-js --- bids-validator/src/deps/nifti.ts | 1 + 1 file changed, 1 insertion(+) create mode 100644 bids-validator/src/deps/nifti.ts diff --git a/bids-validator/src/deps/nifti.ts b/bids-validator/src/deps/nifti.ts new file mode 100644 index 000000000..85e6aff54 --- /dev/null +++ b/bids-validator/src/deps/nifti.ts @@ -0,0 +1 @@ +export { isCompressed, readHeader } from 'https://esm.sh/nifti-reader-js@0.6.8' From 45ff9dd9392d2290ab34fcf834dc6e31f9dff73e Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 31 Jul 2024 02:23:05 -0400 Subject: [PATCH 6/7] fix(nifti): Decompress nii.gz headers ourselves --- bids-validator/src/files/nifti.ts | 37 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/bids-validator/src/files/nifti.ts b/bids-validator/src/files/nifti.ts index 658ced48c..2c449b024 100644 --- a/bids-validator/src/files/nifti.ts +++ b/bids-validator/src/files/nifti.ts @@ -1,19 +1,46 @@ -import 'https://raw.githubusercontent.com/rii-mango/NIFTI-Reader-JS/v0.6.4/release/current/nifti-reader-min.js' +import { isCompressed, readHeader } from '../deps/nifti.ts' import { BIDSFile } from '../types/file.ts' import { logger } from '../utils/logger.ts' +import { ContextNiftiHeader } from '../types/context.ts' -export async function loadHeader(file: BIDSFile) { +async function extract(buffer: Uint8Array, nbytes: number): Promise { + // The fflate decompression that is used in nifti-reader does not like + // truncated data, so pretend that we have a stream and stop reading + // when we have enough bytes. + const result = new Uint8Array(nbytes) + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(buffer) + }, + }) + const reader = stream.pipeThrough(new DecompressionStream('gzip')).getReader() + let offset = 0 + while (offset < nbytes) { + const { value, done } = await reader.read() + if (done) { + break + } + result.set(value.subarray(0, Math.min(value.length, nbytes - offset)), offset) + offset += value.length + } + await reader.cancel() + return result +} + +export async function loadHeader(file: BIDSFile): Promise { try { const buf = await file.readBytes(1024) - // @ts-expect-error NIFTI-Reader-JS required mangling globals - const header = globalThis.nifti.readHeader(buf.buffer) + const data = isCompressed(buf.buffer) ? await extract(buf, 540) : buf + const header = readHeader(data.buffer) // normalize between nifti-reader and spec schema // https://github.com/bids-standard/bids-specification/blob/master/src/schema/meta/context.yaml#L200 if (header) { + // @ts-expect-error header.pixdim = header.pixDims + // @ts-expect-error header.dim = header.dims } - return header + return header as unknown as ContextNiftiHeader } catch (err) { logger.warning(`NIfTI file could not be opened or read ${file.path}`) logger.debug(err) From 9d77a9340a1d37c0b8eb250394640242f44c6f0f Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Wed, 31 Jul 2024 09:15:25 -0500 Subject: [PATCH 7/7] add basic test for loading nifti header --- bids-validator/src/files/nifti.test.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 bids-validator/src/files/nifti.test.ts diff --git a/bids-validator/src/files/nifti.test.ts b/bids-validator/src/files/nifti.test.ts new file mode 100644 index 000000000..cf8434c31 --- /dev/null +++ b/bids-validator/src/files/nifti.test.ts @@ -0,0 +1,26 @@ +import { assert } from '../deps/asserts.ts' +import { FileIgnoreRules } from './ignore.ts' +import { BIDSFileDeno } from './deno.ts' + +import { loadHeader } from './nifti.ts' + +Deno.test('Test loading nifti header', async (t) => { + const ignore = new FileIgnoreRules([]) + await t.step('Load header from compressed file', async () => { + const path = "sub-01/func/sub-01_task-rhymejudgment_bold.nii.gz" + const root = "./tests/data/valid_headers" + const file = new BIDSFileDeno(root, path, ignore) + const header = await loadHeader(file) + assert(header !== undefined) + assert(header['pixdim'].length === 8) + }) + await t.step('Fail on non-nifti file', async () => { + const path = "sub-01/func/sub-01_task-rhymejudgment_events.tsv" + const root = "./tests/data/valid_headers" + const file = new BIDSFileDeno(root, path, ignore) + const header = await loadHeader(file) + assert(header !== undefined) + assert(header === null) + }) + +})