Merge branch 'master' into JA/patchBrukerIMT

.github/workflows/Semgrep.yml

@@ -0,0 +1,66 @@
+# Based on:
+# https://semgrep.dev/docs/semgrep-ci/sample-ci-configs#github-actions
+# https://0xdbe.github.io/GitHub-HowToEnableCodeScanningWithSemgrep/
+# https://medium.com/@mostafa.elnakeb/supercharging-your-code-quality-with-semgrep-sast-in-github-actions-c8f30eb26655
+# Name of this GitHub Actions workflow.
+name: Semgrep OSS scan
+
+on:
+  # Scan on-demand through GitHub Actions interface:
+  workflow_dispatch:
+    branches:
+      - master
+  # Schedule the CI job (this method uses cron syntax):
+  schedule:
+    - cron: '0 0 * * 1' # Run at start of week 
+
+jobs:
+  semgrep:
+    # User definable name of this GitHub Actions job.
+    name: semgrep-oss/scan
+    # If you are self-hosting, change the following `runs-on` value: 
+    runs-on: ubuntu-latest
+
+    steps:
+        # Checkout the repository.
+      - name: Clone source code
+        uses: actions/checkout@v4
+
+        # Checkout custom rules
+      - name: Checkout custom rules
+        uses: actions/checkout@v4
+        with:
+          repository: JuliaComputing/semgrep-rules-julia
+          ref: main
+          path: ./JuliaRules
+
+        # Prepare Python
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+        # Install Semgrep
+      - name: Install Semgrep
+        run: python3 -m pip install semgrep
+
+        # Run Semgrep
+      - name: Scan with Semgrep
+        run: | 
+            semgrep scan \
+                --config ./JuliaRules/rules \
+                --metrics=off \
+                --sarif --output report.sarif \
+                --oss-only \
+                --exclude=JuliaRules
+    
+      - name: Save Semgrep report
+        uses: actions/upload-artifact@v4
+        with:
+          name: report.sarif
+          path: report.sarif
+
+      - name: Upload Semgrep report
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: report.sarif
+          category: semgrep

.github/workflows/breakage.yml

@@ -1,15 +1,15 @@
 name: Breakage 
 # Based on: https://github.com/JuliaSmoothOptimizers/LinearOperators.jl/blob/main/.github/workflows/Breakage.yml
 on:
-    pull_request:
-        branches:
-          - master
-    push:
-        branches:
-          - master
-        tags: '*'
+  pull_request:
+    branches:
+      - master
+    types: [opened, reopened, labeled, synchronize]
+
+
 jobs:
   break:
+    if: ${{  contains(github.event.pull_request.labels.*.name, 'breakage')  }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false

.github/workflows/ci.yml

@@ -43,7 +43,7 @@ jobs:
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v4
+      - uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:

.gitignore

@@ -25,5 +25,7 @@ test/data/*
 # committed for packages, but should be committed for applications that require a static
 # environment.
 Manifest.toml
+# also ignore julia-version specific Manifests (>1.11)
+Manifest-v*.toml
 
 .vscode

Project.toml

@@ -1,7 +1,7 @@
 name = "MPIFiles"
 uuid = "371237a9-e6c1-5201-9adb-3d8cfa78fa9f"
 authors = ["Tobias Knopp <[email protected]>"]
-version = "0.16.0"
+version = "0.16.1"
 
 [deps]
 AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9"
@@ -35,25 +35,41 @@ UnitfulAngles = "6fb2a4bd-7999-5318-a3b2-8ad61056cd98"
 UnitfulParsableString = "06c00241-927a-4d5b-bb5e-6b5a2ada3567"
 
 [compat]
-Aqua = "0.6"
+Aqua = "0.8"
 AxisArrays = "0.3, 0.4"
 CodecZlib = "0.7"
+Dates = "1"
 DelimitedFiles = "1"
 DocStringExtensions = "0.8, 0.9"
 FFTW = "1.3"
 FileIO = "1.0"
 Graphics = "0.4, 1.0"
-HDF5 = "0.14, 0.15, 0.16"
+HDF5 = "0.14, 0.15, 0.16, 0.17"
+HTTP = "1"
 ImageAxes = "0.6"
 ImageMetadata = "0.9"
 Inflate = "0.1.2"
-Interpolations = "0.12, 0.13, 0.14"
-LinearOperatorCollection = "1.1"
-StableRNGs = "1.0.0"
+InteractiveUtils = "1"
+Interpolations = "0.12, 0.13, 0.14, 0.15"
+LazyArtifacts = "1"
+LinearAlgebra = "1"
+LinearOperatorCollection = "1, 2"
+Mmap = "1"
+Pkg = "1"
+REPL = "1"
+Random = "1"
+Scratch = "1"
+SHA = "0.7"
+StableRNGs = "1"
+Statistics = "1"
+Test = "1"
+TOML = "1"
+Tar = "1"
+UUIDs = "1"
 Unitful = "1.13, 1.14, 1.15, 1.16, 1.17"
-UnitfulAngles = "0.6.1"
+UnitfulAngles = "0.6.1, 0.7"
 UnitfulParsableString = "0.1.6"
-julia = "1.5"
+julia = "1.9"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"

src/Brukerfile.jl

@@ -186,7 +186,7 @@ studyNameOld(b::BrukerFile) = string(latin1toutf8(b["VisuSubjectId"])*latin1tout
                                   b["VisuStudyNumber"])
 studyNumber(b::BrukerFile) = parse(Int64,b["VisuStudyNumber"])
 function studyUuid(b::BrukerFile)
-  rng = MersenneTwister(hash(b["VisuStudyUid"])) # use VisuStudyUid as seed to generate uuid4
+  rng = StableRNG(hash(b["VisuStudyUid"])) # use VisuStudyUid as seed to generate uuid4
   return uuid4(rng)
 end
 studyDescription(b::BrukerFile) = "n.a."
@@ -207,7 +207,7 @@ function experimentNumber(b::BrukerFile)
 end
 
 function experimentUuid(b::BrukerFile)
-  rng = MersenneTwister(hash(b["VisuUid"])) # use VisuUid as seed to generate uuid4
+  rng = StableRNG(hash(b["VisuUid"])) # use VisuUid as seed to generate uuid4
   return uuid4(rng)
 end
 experimentDescription(b::BrukerFile) = latin1toutf8(b["ACQ_scan_name"])

src/Derived.jl

@@ -4,19 +4,12 @@ export acqNumFGFrames, acqNumBGFrames, acqOffsetFieldShift, acqFramePeriod,
        rxNumFrequencies, rxFrequencies, rxTimePoints,
        measFGFrameIdx, measBGFrameIdx, measBGFrameBlockLengths
 
-rxNumFrequencies(f::MPIFile, numPeriodGrouping=1) = floor(Int,rxNumSamplingPoints(f)*numPeriodGrouping ./ 2 .+ 1)
+rxNumFrequencies(f::MPIFile, numPeriodGrouping=1) = length(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping))
 
-function rxFrequencies(f::MPIFile)
-  numFreq = rxNumFrequencies(f)
-  a = collect(0:(numFreq-1))./(numFreq-1).*rxBandwidth(f)
-  return a
-end
+rxFrequencies(f::MPIFile, numPeriodGrouping=1) = rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)) |> collect
+
+rxTimePoints(f::MPIFile, numPeriodGrouping=1) = range(0, step=1/2rxBandwidth(f), length=rxNumSamplingPoints(f)*numPeriodGrouping) |> collect
 
-function rxTimePoints(f::MPIFile)
-  numTP = rxNumSamplingPoints(f)
-  a = collect(0:(numTP-1))./(numTP).*dfCycle(f)
-  return a
-end
 
 function acqGradientDiag(f::MPIFile)
   g = acqGradient(f)
@@ -44,36 +37,8 @@ end
 acqNumFGFrames(f::MPIFile) = acqNumFrames(f) - acqNumBGFrames(f)
 acqNumBGFrames(f::MPIFile) = sum(measIsBGFrame(f))
 
-function measBGFrameIdx(f::MPIFile)
-  idx = zeros(Int64, acqNumBGFrames(f))
-  j = 1
-  mask = measIsBGFrame(f)
-  for i=1:acqNumFrames(f)
-    if mask[i]
-      idx[j] = i
-      j += 1
-    end
-  end
-  return idx
-end
-
-function measFGFrameIdx(f::MPIFile)
-  mask = measIsBGFrame(f)
-  if !any(mask)
-    #shortcut
-    return 1:acqNumFrames(f)
-  end
-  idx = zeros(Int64, acqNumFGFrames(f))
-  j = 1
-  for i=1:acqNumFrames(f)
-    if !mask[i]
-      idx[j] = i
-      j += 1
-    end
-  end
-  return idx
-end
-
+measBGFrameIdx(f::MPIFile) = findall(measIsBGFrame(f))
+measFGFrameIdx(f::MPIFile) = findall(.!measIsBGFrame(f))
 
 function measBGFrameBlockLengths(mask)
   len = Vector{Int}(undef,0)

src/FrequencyFilter.jl

@@ -28,18 +28,21 @@ Supported keyword arguments:
 * stepsize
 * maxMixingOrder
 * sortByMixFactors
+* numPeriodGrouping
+* numSideBandFreqs
+* stopBands
 """
 function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0,
                maxFreq=rxBandwidth(f), recChannels=1:rxNumChannels(f),numUsedFreqs=-1, stepsize=1,
-                maxMixingOrder=-1, numPeriodAverages=1, numPeriodGrouping=1, numSidebandFreqs = -1)
+                maxMixingOrder=-1, numPeriodAverages=1, numPeriodGrouping=1, numSidebandFreqs = -1, stopBands = nothing)
 
   nFreq = rxNumFrequencies(f, numPeriodGrouping)
   nReceivers = rxNumChannels(f)
   nPeriods = 1 #acqNumPeriodsPerFrame(f)
   freqs = measIsFrequencySelection(f) ? measFrequencySelection(f) : 1:nFreq
 
   if numPeriodGrouping == 1
-    freqIndices = vec([CartesianIndex{2}(i, j) for i in freqs, j in recChannels])
+    freqIndices = vec([CartesianIndex{2}(i, j) for i in freqs, j in intersect(1:nReceivers, recChannels)])
   else
     freqIndices = collect(vec(CartesianIndices((nFreq, nReceivers))))
     filterFrequenciesBySelection!(freqIndices, f) 
@@ -88,6 +91,10 @@ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0,
     filterFrequenciesByStepsize!(freqIndices, stepsize)
   end
 
+  if !isnothing(stopBands)
+    filterFrequenciesByStopBands!(freqIndices, f, stopBands; numPeriodGrouping = numPeriodGrouping)
+  end
+
   return collect(vec(freqIndices))
 end
 
@@ -104,19 +111,17 @@ filterFrequenciesByChannel!(indices, channels, sorted = issorted(channels)) = fi
 
 export filterFrequenciesByMinFreq!
 function filterFrequenciesByMinFreq!(indices, f::MPIFile, minFreq; numPeriodGrouping = 1)
-  nFreq = rxNumFrequencies(f, numPeriodGrouping)
-  minIdx = floor(Int, minFreq / rxBandwidth(f) * (nFreq-1) ) + 1
+  minIdx = searchsortedfirst(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), minFreq)
   return filterFrequenciesByMinIdx!(indices, minIdx)
 end
-filterFrequenciesByMinIdx!(indices, minIdx) = minIdx > 0 ?  filter!(x -> x[1] > minIdx, indices) : indices 
+filterFrequenciesByMinIdx!(indices, minIdx) = minIdx > 0 ?  filter!(x -> x[1] >= minIdx, indices) : indices 
 
 export filterFrequenciesByMaxFreq!
 function filterFrequenciesByMaxFreq!(indices, f::MPIFile, maxFreq; numPeriodGrouping = 1)
-  nFreq = rxNumFrequencies(f, numPeriodGrouping)
-  maxIdx = ceil(Int, maxFreq / rxBandwidth(f) * (nFreq-1) ) + 1
+  maxIdx = searchsortedlast(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), maxFreq)
   return filterFrequenciesByMaxIdx!(indices, maxIdx)
 end
-filterFrequenciesByMaxIdx!(indices, maxIdx) = filter!(x-> x[1] < maxIdx, indices)
+filterFrequenciesByMaxIdx!(indices, maxIdx) = filter!(x-> x[1] <= maxIdx, indices)
 
 export filterFrequenciesByMaxMixingOrder!
 filterFrequenciesByMaxMixingOrder!(indices, maxMixingOrder, f::MPIFile) = filterFrequenciesByMaxMixingOrder!(indices, maxMixingOrder, mixingFactors(f))
@@ -177,6 +182,29 @@ function filterFrequenciesByStepsize!(indices, stepsize)
   filter!(x -> insorted(x[1], stepIndices), indices)
 end
 
+export filterFrequenciesByStopBands!, filterFrequenciesByStopBand!
+function filterFrequenciesByStopBands!(indices, f::MPIFile, stopBands::Vector; kwargs...)
+  for stopBand in stopBands
+    filterFrequenciesByStopBand!(indices, f, stopBand; kwargs...)
+  end
+end
+filterFrequenciesByStopBands!(indices, f::MPIFile, stopBand::Union{Vector{Int64}, UnitRange, NTuple}; kwargs...) = filterFrequenciesByStopBands!(indices, f, [stopBand]; kwargs...)
+function filterFrequenciesByStopBand!(indices, f::MPIFile, stopBand::Vector{Int64}; numPeriodGrouping = 1)
+  if length(stopBand) != 2
+    error("Stop band are only defined for a start and stop value. Found $(length(stopBand)) values")
+  end
+  nFreq = rxNumFrequencies(f, numPeriodGrouping)
+  minIdx = searchsortedlast(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), first(stopBand))
+  maxIdx = searchsortedlast(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), last(stopBand))
+  return filterFrequenciesByStopBand!(indices, minIdx, maxIdx)
+end
+function filterFrequenciesByStopBand!(indices, f::MPIFile, stopBand::Union{UnitRange, NTuple{2, Int64}}; numPeriodGrouping = 1)
+  minIdx = searchsortedlast(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), first(stopBand))
+  maxIdx = searchsortedlast(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), last(stopBand))
+  return filterFrequenciesByStopBand!(indices, minIdx, maxIdx)
+end
+filterFrequenciesByStopBand!(indices, minIdx, maxIdx) = filter!(x-> x[1] < minIdx || x[1] > maxIdx, indices)
+
 function sortFrequencies(indices, f::MPIFile; numPeriodGrouping = 1, stepsize = 1, sortBySNR = false, sortByMixFactors = false)
   if sortBySNR && !sortByMixFactors
     indices = sortFrequenciesBySNR(indices, f, numPeriodGrouping = numPeriodGrouping, stepsize = stepsize)

src/MDF.jl

@@ -226,7 +226,7 @@ dfBaseFrequency(f::MDFFile)::Union{Float64, Missing} = @keyrequired f["/acquisit
 dfCustomWaveform(f::MDFFileV2)::Union{String, Nothing} = @keyoptional f["/acquisition/drivefield/customWaveform"]
 dfDivider(f::MDFFileV1) = @keyrequired addTrailingSingleton(
                 f["/acquisition/drivefield/divider"],2)
-dfDivider(f::MDFFileV2) = f["/acquisition/drivefield/divider"]
+dfDivider(f::MDFFileV2) = @keyrequired f["/acquisition/drivefield/divider"]
 dfWaveform(f::MDFFileV1) = "sine"
 function dfWaveform(f::MDFFileV2)::Union{Array{String, 2}, Missing}
   value = @keyrequired f["/acquisition/drivefield/waveform"]
@@ -401,7 +401,7 @@ measFramePermutation(f::MDFFileV1)::Union{Vector{Int64}, Nothing} = nothing
 measFramePermutation(f::MDFFileV2)::Union{Vector{Int64}, Nothing} = @keyoptional f["/measurement/framePermutation"]
 measSparsityTransformation(f::MDFFileV1)::Union{String, Nothing} = nothing
 measSparsityTransformation(f::MDFFileV2)::Union{String, Nothing} = @keyoptional f["/measurement/sparsityTransformation"]
-measSubsamplingIndices(f::MDFFileV2)::Union{Array{Integer, 4}, Nothing} = @keyoptional f["/measurement/subsamplingIndices"]
+measSubsamplingIndices(f::MDFFileV2)::Union{Array{Int64, 4}, Nothing} = @keyoptional f["/measurement/subsamplingIndices"]
 
 fullFramePermutation(f::MDFFile) = fullFramePermutation(f, calibIsMeanderingGrid(f))