Merge branch 'master' into rf/json-validator

bids-validator/package.json

@@ -45,7 +45,7 @@
     "date-fns": "^3.6.0",
     "events": "^3.3.0",
     "exifreader": "^4.23.3",
-    "hed-validator": "^3.15.1",
+    "hed-validator": "^3.15.2",
     "ignore": "^5.3.1",
     "is-utf8": "^0.2.1",
     "jest": "^29.7.0",

bids-validator/src/deps/nifti.ts

@@ -0,0 +1 @@
+export { isCompressed, readHeader } from 'https://esm.sh/[email protected]'

bids-validator/src/files/browser.test.ts

@@ -29,7 +29,11 @@ Deno.test('Browser implementation of FileTree', async (t) => {
     ]
     const tree = await fileListToTree(files)
     const expectedTree = new FileTree('', '/', undefined)
-    expectedTree.files = files.map((f) => new BIDSFileBrowser(f, ignore))
+    expectedTree.files = files.map((f) => {
+      const file = new BIDSFileBrowser(f, ignore)
+      file.parent = expectedTree
+      return file
+    })
     assertEquals(tree, expectedTree)
   })
   await t.step('converts a simple FileList with several levels', async () => {
@@ -62,9 +66,14 @@ Deno.test('Browser implementation of FileTree', async (t) => {
     const anatTree = new FileTree('sub-01/anat', 'anat', sub01Tree)
     expectedTree.files = files
       .slice(0, 3)
-      .map((f) => new BIDSFileBrowser(f, ignore))
+      .map((f) => {
+        const file = new BIDSFileBrowser(f, ignore)
+        file.parent = expectedTree
+        return file
+      })
     expectedTree.directories.push(sub01Tree)
     anatTree.files = [new BIDSFileBrowser(files[3], ignore)]
+    anatTree.files[0].parent = anatTree
     sub01Tree.directories.push(anatTree)
     assertEquals(tree, expectedTree)
   })
@@ -82,7 +91,11 @@ Deno.test('Spread copies of BIDSFileBrowser contain name and path properties', a
   ]
   const tree = await fileListToTree(files)
   const expectedTree = new FileTree('', '/', undefined)
-  expectedTree.files = files.map((f) => new BIDSFileBrowser(f, ignore))
+  expectedTree.files = files.map((f) => {
+    const file = new BIDSFileBrowser(f, ignore)
+    file.parent = expectedTree
+    return file
+  })
   assertEquals(tree, expectedTree)
   const spreadFile = { ...expectedTree.files[0], evidence: 'test evidence' }
   assertObjectMatch(spreadFile, {

bids-validator/src/files/browser.ts

@@ -1,5 +1,4 @@
-import { BIDSFile } from '../types/file.ts'
-import { FileTree } from '../types/filetree.ts'
+import { BIDSFile, FileTree } from '../types/filetree.ts'
 import { FileIgnoreRules } from './ignore.ts'
 import { parse, posix, SEPARATOR_PATTERN } from '../deps/path.ts'
 
@@ -11,6 +10,7 @@ export class BIDSFileBrowser implements BIDSFile {
   #file: File
   name: string
   path: string
+  parent?: FileTree
 
   constructor(file: File, ignore: FileIgnoreRules) {
     this.#file = file
@@ -53,6 +53,7 @@ export function fileListToTree(files: File[]): Promise<FileTree> {
     const fPath = parse(file.path)
     if (fPath.dir === '/') {
       // Top level file
+      file.parent = tree
       tree.files.push(file)
     } else {
       const levels = fPath.dir.split(SEPARATOR_PATTERN).slice(1)
@@ -76,6 +77,7 @@ export function fileListToTree(files: File[]): Promise<FileTree> {
         }
       }
       // At the terminal leaf, add files
+      file.parent = currentLevelTree
       currentLevelTree.files.push(file)
     }
   }

bids-validator/src/files/deno.ts

@@ -2,8 +2,7 @@
  * Deno specific implementation for reading files
  */
 import { basename, join, posix } from '../deps/path.ts'
-import { BIDSFile } from '../types/file.ts'
-import { FileTree } from '../types/filetree.ts'
+import { BIDSFile, FileTree } from '../types/filetree.ts'
 import { requestReadPermission } from '../setup/requestPermissions.ts'
 import { FileIgnoreRules, readBidsIgnore } from './ignore.ts'
 
@@ -24,6 +23,7 @@ export class BIDSFileDeno implements BIDSFile {
   #ignore: FileIgnoreRules
   name: string
   path: string
+  parent?: FileTree
   #fileInfo?: Deno.FileInfo
   #datasetAbsPath: string
 
@@ -125,6 +125,7 @@ export async function _readFileTree(
         posix.join(relativePath, dirEntry.name),
         ignore,
       )
+      file.parent = tree
       // For .bidsignore, read in immediately and add the rules
       if (dirEntry.name === '.bidsignore') {
         ignore.add(await readBidsIgnore(file))

bids-validator/src/files/ignore.ts

@@ -1,4 +1,4 @@
-import { BIDSFile } from '../types/file.ts'
+import { BIDSFile } from '../types/filetree.ts'
 import { default as ignore } from 'npm:[email protected]'
 import type { Ignore } from 'npm:[email protected]'
 

bids-validator/src/files/inheritance.ts

@@ -0,0 +1,47 @@
+import { BIDSFile, FileTree } from '../types/filetree.ts'
+import { Context } from '../types/context.ts'
+import { readEntities } from '../schema/entities.ts'
+
+export function* walkBack(
+  source: BIDSFile,
+  inherit: boolean = true,
+  targetExtensions: string[] = ['.json'],
+  targetSuffix?: string,
+): Generator<BIDSFile> {
+  const sourceParts = readEntities(source.name)
+
+  targetSuffix = targetSuffix || sourceParts.suffix
+
+  let fileTree = source.parent
+  while (fileTree) {
+    const candidates = fileTree.files.filter((file) => {
+      const { suffix, extension, entities } = readEntities(file.name)
+      return (
+        targetExtensions.includes(extension) &&
+        suffix === targetSuffix &&
+        Object.keys(entities).every((entity) => entities[entity] === sourceParts.entities[entity])
+      )
+    })
+    if (candidates.length > 1) {
+      const exactMatch = candidates.find((file) => {
+        const { suffix, extension, entities } = readEntities(file.name)
+        return Object.keys(sourceParts.entities).every((entity) =>
+          entities[entity] === sourceParts.entities[entity]
+        )
+      })
+      if (exactMatch) {
+        yield exactMatch
+      } else {
+        console.warn(`
+Multiple candidates detected for '${source.path}'
+
+${candidates.map((file) => `* ${file.path}`).join('\n')}
+`)
+      }
+    } else if (candidates.length === 1) {
+      yield candidates[0]
+    }
+    if (!inherit) break
+    fileTree = fileTree.parent
+  }
+}

bids-validator/src/files/nifti.test.ts

@@ -0,0 +1,26 @@
+import { assert } from '../deps/asserts.ts'
+import { FileIgnoreRules } from './ignore.ts'
+import { BIDSFileDeno } from './deno.ts'
+
+import { loadHeader } from './nifti.ts'
+
+Deno.test('Test loading nifti header', async (t) => {
+  const ignore = new FileIgnoreRules([])
+  await t.step('Load header from compressed file', async () => {
+    const path = "sub-01/func/sub-01_task-rhymejudgment_bold.nii.gz"
+    const root = "./tests/data/valid_headers"
+    const file = new BIDSFileDeno(root, path, ignore)
+    const header = await loadHeader(file)
+    assert(header !== undefined)
+    assert(header['pixdim'].length === 8)
+  })
+  await t.step('Fail on non-nifti file', async () => {
+    const path = "sub-01/func/sub-01_task-rhymejudgment_events.tsv"
+    const root = "./tests/data/valid_headers"
+    const file = new BIDSFileDeno(root, path, ignore)
+    const header = await loadHeader(file)
+    assert(header !== undefined)
+    assert(header === null)
+  })
+
+})

bids-validator/src/files/nifti.ts

@@ -1,19 +1,46 @@
-import 'https://raw.githubusercontent.com/rii-mango/NIFTI-Reader-JS/v0.6.4/release/current/nifti-reader-min.js'
-import { BIDSFile } from '../types/file.ts'
+import { isCompressed, readHeader } from '../deps/nifti.ts'
+import { BIDSFile } from '../types/filetree.ts'
 import { logger } from '../utils/logger.ts'
+import { ContextNiftiHeader } from '../types/context.ts'
 
-export async function loadHeader(file: BIDSFile) {
+async function extract(buffer: Uint8Array, nbytes: number): Promise<Uint8Array> {
+  // The fflate decompression that is used in nifti-reader does not like
+  // truncated data, so pretend that we have a stream and stop reading
+  // when we have enough bytes.
+  const result = new Uint8Array(nbytes)
+  const stream = new ReadableStream({
+    start(controller) {
+      controller.enqueue(buffer)
+    },
+  })
+  const reader = stream.pipeThrough(new DecompressionStream('gzip')).getReader()
+  let offset = 0
+  while (offset < nbytes) {
+    const { value, done } = await reader.read()
+    if (done) {
+      break
+    }
+    result.set(value.subarray(0, Math.min(value.length, nbytes - offset)), offset)
+    offset += value.length
+  }
+  await reader.cancel()
+  return result
+}
+
+export async function loadHeader(file: BIDSFile): Promise<ContextNiftiHeader | undefined> {
   try {
     const buf = await file.readBytes(1024)
-    // @ts-expect-error NIFTI-Reader-JS required mangling globals
-    const header = globalThis.nifti.readHeader(buf.buffer)
+    const data = isCompressed(buf.buffer) ? await extract(buf, 540) : buf
+    const header = readHeader(data.buffer)
     // normalize between nifti-reader and spec schema
     // https://github.com/bids-standard/bids-specification/blob/master/src/schema/meta/context.yaml#L200
     if (header) {
+      // @ts-expect-error
       header.pixdim = header.pixDims
+      // @ts-expect-error
       header.dim = header.dims
     }
-    return header
+    return header as unknown as ContextNiftiHeader
   } catch (err) {
     logger.warning(`NIfTI file could not be opened or read ${file.path}`)
     logger.debug(err)

bids-validator/src/issues/datasetIssues.test.ts

@@ -1,5 +1,5 @@
 import { assertEquals, assertObjectMatch } from '../deps/asserts.ts'
-import { BIDSFile } from '../types/file.ts'
+import { BIDSFile } from '../types/filetree.ts'
 import { IssueFile } from '../types/issues.ts'
 import { DatasetIssues } from './datasetIssues.ts'
 

bids-validator/src/schema/associations.ts

@@ -1,10 +1,10 @@
 import { ContextAssociations, ContextAssociationsEvents } from '../types/context.ts'
-import { BIDSFile } from '../types/file.ts'
-import { FileTree } from '../types/filetree.ts'
+import { BIDSFile, FileTree } from '../types/filetree.ts'
 import { BIDSContext } from './context.ts'
 import { readEntities } from './entities.ts'
 import { parseTSV } from '../files/tsv.ts'
 import { parseBvalBvec } from '../files/dwi.ts'
+import { walkBack } from '../files/inheritance.ts'
 
 // type AssociationsLookup = Record<keyof ContextAssociations, { extensions: string[], inherit: boolean, load: ... }
 
@@ -127,54 +127,18 @@ const associationLookup = {
 }
 
 export async function buildAssociations(
-  fileTree: FileTree,
-  source: BIDSContext,
+  source: BIDSFile,
 ): Promise<ContextAssociations> {
   const associations: ContextAssociations = {}
-  for (const key in associationLookup as typeof associationLookup) {
-    const { suffix, extensions, inherit } =
-      associationLookup[key as keyof typeof associationLookup]
-    const paths = getPaths(fileTree, source, suffix, extensions)
-    if (paths.length === 0) {
-      continue
-    }
-    if (paths.length > 1) {
-      // error?
-    }
-    // @ts-expect-error
-    associations[key] = await associationLookup[key].load(paths[0])
-  }
-  return Promise.resolve(associations)
-}
-
-function getPaths(
-  fileTree: FileTree,
-  source: BIDSContext,
-  targetSuffix: string,
-  targetExtensions: string[],
-) {
-  const validAssociations = fileTree.files.filter((file) => {
-    const { suffix, extension, entities } = readEntities(file.name)
-    return (
-      targetExtensions.includes(extension) &&
-      suffix === targetSuffix &&
-      Object.keys(entities).every((entity) => {
-        return (
-          entity in source.entities &&
-          entities[entity] === source.entities[entity]
-        )
-      })
-    )
-  })
 
-  const nextDir = fileTree.directories.find((directory) => {
-    return source.file.path.startsWith(directory.path)
-  })
+  for (const [key, value] of Object.entries(associationLookup)) {
+    const { suffix, extensions, inherit, load } = value
+    const path = walkBack(source, inherit, extensions, suffix).next().value
 
-  if (nextDir) {
-    validAssociations.push(
-      ...getPaths(nextDir, source, targetSuffix, targetExtensions),
-    )
+    if (path) {
+      // @ts-expect-error
+      associations[key] = await load(path)
+    }
   }
-  return validAssociations
+  return Promise.resolve(associations)
 }

bids-validator/src/schema/context.test.ts

@@ -5,7 +5,7 @@ import { dataFile, rootFileTree } from './fixtures.test.ts'
 
 Deno.test('test context LoadSidecar', async (t) => {
   const context = new BIDSContext(rootFileTree, dataFile, new DatasetIssues())
-  await context.loadSidecar(rootFileTree)
+  await context.loadSidecar()
   await t.step('sidecar overwrites correct fields', () => {
     const { rootOverwrite, subOverwrite } = context.sidecar
     assert(rootOverwrite === 'anat')