From 28eba691126704bbd0c5f660f2fb68d3f7e21bdd Mon Sep 17 00:00:00 2001 From: jonschumacher Date: Mon, 17 Jun 2024 13:21:56 +0200 Subject: [PATCH 01/31] Ensure concrete type to prevent HDF5 errors --- src/RecoData.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/RecoData.jl b/src/RecoData.jl index d568df86..db7a52e9 100644 --- a/src/RecoData.jl +++ b/src/RecoData.jl @@ -21,8 +21,8 @@ function saveRecoData(filename, image::ImageMeta) params = properties(image) params[:recoData] = c - params[:recoFov] = collect(grid) .* collect(converttometer(pixelspacing(image))) - params[:recoFovCenter] = collect(converttometer(imcenter(image)))[1:3] + params[:recoFov] = Float64.(collect(grid) .* collect(converttometer(pixelspacing(image)))) + params[:recoFovCenter] = Float64.(collect(converttometer(imcenter(image)))[1:3]) params[:recoSize] = collect(grid) params[:recoOrder] = "xyz" if haskey(params,:recoParams) From e22de2f8ae25369183425c5b3e64df81c665b53e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:37:40 +0000 Subject: [PATCH 02/31] CompatHelper: add new compat entry for TOML at version 1, (keep existing compat) (#94) Co-authored-by: CompatHelper Julia Co-authored-by: Jonas Schumacher --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index 3c8cd589..b2c64480 100644 --- a/Project.toml +++ b/Project.toml @@ -50,6 +50,7 @@ Inflate = "0.1.2" Interpolations = "0.12, 0.13, 0.14" LinearOperatorCollection = "1.1" StableRNGs = "1.0.0" +TOML = "1" Unitful = "1.13, 1.14, 1.15, 1.16, 1.17" UnitfulAngles = "0.6.1" UnitfulParsableString = "0.1.6" From 9bc8af782cb562325fa31ee4f0643343de4026eb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:37:57 +0000 Subject: [PATCH 03/31] CompatHelper: bump compat for Interpolations to 0.15, (keep existing compat) (#88) Co-authored-by: CompatHelper Julia Co-authored-by: Jonas Schumacher --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index b2c64480..18b03e21 100644 --- a/Project.toml +++ b/Project.toml @@ -47,7 +47,7 @@ HDF5 = "0.14, 0.15, 0.16" ImageAxes = "0.6" ImageMetadata = "0.9" Inflate = "0.1.2" -Interpolations = "0.12, 0.13, 0.14" +Interpolations = "0.12, 0.13, 0.14, 0.15" LinearOperatorCollection = "1.1" StableRNGs = "1.0.0" TOML = "1" From 972b72758b6f10214c5b22ab63f4105f0ce39328 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:38:42 +0000 Subject: [PATCH 04/31] CompatHelper: bump compat for UnitfulAngles to 0.7, (keep existing compat) (#93) Co-authored-by: CompatHelper Julia Co-authored-by: Jonas Schumacher --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 18b03e21..02a38853 100644 --- a/Project.toml +++ b/Project.toml @@ -52,7 +52,7 @@ LinearOperatorCollection = "1.1" StableRNGs = "1.0.0" TOML = "1" Unitful = "1.13, 1.14, 1.15, 1.16, 1.17" -UnitfulAngles = "0.6.1" +UnitfulAngles = "0.6.1, 0.7" UnitfulParsableString = "0.1.6" julia = "1.5" From 5ff1411e6e85322553f4a7e6174e31ae191bc55b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:09:50 +0000 Subject: [PATCH 05/31] CompatHelper: bump compat for HDF5 to 0.17, (keep existing compat) (#92) Co-authored-by: CompatHelper Julia Co-authored-by: Jonas Schumacher --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 02a38853..bc7bfb03 100644 --- a/Project.toml +++ b/Project.toml @@ -43,7 +43,7 @@ DocStringExtensions = "0.8, 0.9" FFTW = "1.3" FileIO = "1.0" Graphics = "0.4, 1.0" -HDF5 = "0.14, 0.15, 0.16" +HDF5 = "0.14, 0.15, 0.16, 0.17" ImageAxes = "0.6" ImageMetadata = "0.9" Inflate = "0.1.2" From ae701de006aeaac0646b7eee8067d32af2c7ceb8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:12:15 +0000 Subject: [PATCH 06/31] CompatHelper: add new compat entry for SHA at version 0.7, (keep existing compat) (#91) Co-authored-by: CompatHelper Julia Co-authored-by: Jonas Schumacher --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index bc7bfb03..c489143c 100644 --- a/Project.toml +++ b/Project.toml @@ -49,6 +49,7 @@ ImageMetadata = "0.9" Inflate = "0.1.2" Interpolations = "0.12, 0.13, 0.14, 0.15" LinearOperatorCollection = "1.1" +SHA = "0.7" StableRNGs = "1.0.0" TOML = "1" Unitful = "1.13, 1.14, 1.15, 1.16, 1.17" From 49826a2195fd6134d2f47f4fa5a67256affe2217 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:23:06 +0000 Subject: [PATCH 07/31] CompatHelper: add new compat entry for Statistics at version 1, (keep existing compat) (#90) Co-authored-by: CompatHelper Julia Co-authored-by: Jonas Schumacher --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index c489143c..9b06ac88 100644 --- a/Project.toml +++ b/Project.toml @@ -51,6 +51,7 @@ Interpolations = "0.12, 0.13, 0.14, 0.15" LinearOperatorCollection = "1.1" SHA = "0.7" StableRNGs = "1.0.0" +Statistics = "1" TOML = "1" Unitful = "1.13, 1.14, 1.15, 1.16, 1.17" UnitfulAngles = "0.6.1, 0.7" From abe8c9b68c5ec9a0e409ddc65fb56f99a0603c8c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:27:29 +0000 Subject: [PATCH 08/31] CompatHelper: add new compat entry for Tar at version 1, (keep existing compat) (#89) Co-authored-by: CompatHelper Julia Co-authored-by: Jonas Schumacher --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index 9b06ac88..c33d023f 100644 --- a/Project.toml +++ b/Project.toml @@ -53,6 +53,7 @@ SHA = "0.7" StableRNGs = "1.0.0" Statistics = "1" TOML = "1" +Tar = "1" Unitful = "1.13, 1.14, 1.15, 1.16, 1.17" UnitfulAngles = "0.6.1, 0.7" UnitfulParsableString = "0.1.6" From d230d0e67dc309a272dcea03ec7d2ffbf2137e5c Mon Sep 17 00:00:00 2001 From: nHackel Date: Thu, 11 Jul 2024 16:36:53 +0200 Subject: [PATCH 09/31] Increase LinearOperatorCollection dep --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index c33d023f..701920bd 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MPIFiles" uuid = "371237a9-e6c1-5201-9adb-3d8cfa78fa9f" authors = ["Tobias Knopp "] -version = "0.16.0" +version = "0.16.1" [deps] AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" @@ -48,7 +48,7 @@ ImageAxes = "0.6" ImageMetadata = "0.9" Inflate = "0.1.2" Interpolations = "0.12, 0.13, 0.14, 0.15" -LinearOperatorCollection = "1.1" +LinearOperatorCollection = "1, 2" SHA = "0.7" StableRNGs = "1.0.0" Statistics = "1" From 60831219dc4a22b1d89e99573b1983468ea9431d Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 26 Jul 2024 16:54:13 +0200 Subject: [PATCH 10/31] Add stopbands keyword to filterFrequencies --- src/FrequencyFilter.jl | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index f5f60a1b..646e5f0d 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -28,10 +28,13 @@ Supported keyword arguments: * stepsize * maxMixingOrder * sortByMixFactors +* numPeriodGrouping +* numSideBandFreqs +* stopBands """ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0, maxFreq=rxBandwidth(f), recChannels=1:rxNumChannels(f),numUsedFreqs=-1, stepsize=1, - maxMixingOrder=-1, numPeriodAverages=1, numPeriodGrouping=1, numSidebandFreqs = -1) + maxMixingOrder=-1, numPeriodAverages=1, numPeriodGrouping=1, numSidebandFreqs = -1, stopBands = nothing) nFreq = rxNumFrequencies(f, numPeriodGrouping) nReceivers = rxNumChannels(f) @@ -88,6 +91,10 @@ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0, filterFrequenciesByStepsize!(freqIndices, stepsize) end + if !isnothing(stopBands) + filterFrequenciesByStopBands!(freqIndices, stopBands) + end + return collect(vec(freqIndices)) end @@ -177,6 +184,13 @@ function filterFrequenciesByStepsize!(indices, stepsize) filter!(x -> insorted(x[1], stepIndices), indices) end +export filterFrequenciesByStopBands!, filterFrequenciesByStopBand! +function filterFrequenciesByStopBands!(indices, stopBands::Vector) + filter!(x -> any(stopBand -> !in(x[1], stopBand), stopBands), indices) +end +filterFrequenciesByStopBands!(indices, stopBand) = filterFrequenciesByStopBands!(indices, [stopBand]) +filterFrequenciesByStopBand!(indices, stopBand) = filter!(x -> !in(x[1], stopBand), indices) + function sortFrequencies(indices, f::MPIFile; numPeriodGrouping = 1, stepsize = 1, sortBySNR = false, sortByMixFactors = false) if sortBySNR && !sortByMixFactors indices = sortFrequenciesBySNR(indices, f, numPeriodGrouping = numPeriodGrouping, stepsize = stepsize) From 883d4f04115fe5b0c55a4c38bf1b4c403f5eb071 Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 26 Jul 2024 18:12:26 +0200 Subject: [PATCH 11/31] Add frequency filter tests --- test/FrequencyFilter.jl | 73 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 test/FrequencyFilter.jl diff --git a/test/FrequencyFilter.jl b/test/FrequencyFilter.jl new file mode 100644 index 00000000..f41500e0 --- /dev/null +++ b/test/FrequencyFilter.jl @@ -0,0 +1,73 @@ +@testset "Frequency Filter" begin + mdf = MDFv2InMemory() + numChannels = 3 + bandwidth = 256 + numSamplingPoints = 2 * bandwidth + recv = MDFv2Receiver(;bandwidth = bandwidth, numSamplingPoints = numSamplingPoints, numChannels = numChannels) + mdf.acquisition = MDFv2Acquisition(;receiver = recv) + mdf.measurement = MDFv2Measurement(;isFrequencySelection = false) + + unfiltered = filterFrequencies(mdf) + @test length(unfiltered) == (bandwidth + 1) * numChannels + @test eltype(unfiltered) == CartesianIndex{2} + nFreq = bandwidth + 1 + + @testset "Filter Receive Channels" begin + @test length(filterFrequencies(mdf, recChannels = 1:numChannels)) == length(unfiltered) + @test length(filterFrequencies(mdf, recChannels = 1:numChannels + 1)) == length(unfiltered) + + freqs = filterFrequencies(mdf, recChannels = 1:numChannels - 1) + @test length(freqs) == length(unfiltered) - (nFreq) + @test all(i -> in(i[2], 1:(numChannels - 1)), freqs) + + + freqs = filterFrequencies(mdf, recChannels = 2:2) + @test length(freqs) == nFreq + @test all(i -> in(i[2], 2:2), freqs) + + chSelection = [1, 3] + freqs = filterFrequencies(mdf, recChannels = chSelection) + @test length(freqs) == length(unfiltered) - (nFreq) + @test all(i -> in(i[2], chSelection), freqs) + + @test length(filterFrequencies(mdf, recChannels = 0:0)) == 0 + @test length(filterFrequencies(mdf, recChannels = numChannels+1:numChannels+2)) == 0 + end + + @testset "Min and Max Frequencies" begin + # minFreq = 0 -> Offset Frequency with index 1 + @test length(filterFrequencies(mdf, minFreq = 0)) == length(unfiltered) + @test length(filterFrequencies(mdf, minFreq = 1)) == length(unfiltered) - numChannels skip = true + @test length(filterFrequencies(mdf, minFreq = bandwidth)) == numChannels skip = true + @test length(filterFrequencies(mdf, minFreq = nFreq)) == 0 + + @test length(filterFrequencies(mdf, maxFreq = bandwidth)) == length(unfiltered) + @test length(filterFrequencies(mdf, maxFreq = bandwidth - 1)) == length(unfiltered) - numChannels skip = true + @test length(filterFrequencies(mdf, maxFreq = 0)) == numChannels skip = true + end + + snr = fill(0.0, nFreq, numChannels, 1) + # Each freq has an SNR equal to its index + snr[:, :, 1] .= 1:nFreq + mdf.calibration = MDFv2Calibration(;snr = snr) + + @testset "SNR Threshold" begin + @test length(filterFrequencies(mdf, SNRThresh = 0)) == length(unfiltered) + @test length(filterFrequencies(mdf, SNRThresh = -1)) == length(unfiltered) + @test length(filterFrequencies(mdf, SNRThresh = nFreq)) == numChannels + @test length(filterFrequencies(mdf, SNRThresh = nFreq + 1)) == 0 + + freqs = filterFrequencies(mdf, SNRThresh = floor(nFreq / 2)) + @test all(i -> i[1] >= floor(nFreq / 2), freqs) + end + + @testset "Num Used Frequencies" begin + @test length(filterFrequencies(mdf, numUsedFreqs = 1)) == numChannels skip = true + @test length(filterFrequencies(mdf, numUsedFreqs = nFreq)) == length(unfiltered) skip = true + @test length(filterFrequencies(mdf, numUsedFreqs = nFreq + 1)) == length(unfiltered) skip = true + @test length(filterFrequencies(mdf, numUsedFreqs = nFreq - 1)) == length(unfiltered) - numChannels skip = true + + freqs = filterFrequencies(mdf, numUsedFreqs = 1) + @test all(i -> i[1] == nFreq, freqs) skip = true + end +end \ No newline at end of file From b684661cf78226bcac7c65cc794a3279a74669cb Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 26 Jul 2024 18:12:53 +0200 Subject: [PATCH 12/31] Fix bug frequency filtering when recChannels are "more" than possible channels --- src/FrequencyFilter.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index 646e5f0d..b2b11c24 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -42,7 +42,7 @@ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0, freqs = measIsFrequencySelection(f) ? measFrequencySelection(f) : 1:nFreq if numPeriodGrouping == 1 - freqIndices = vec([CartesianIndex{2}(i, j) for i in freqs, j in recChannels]) + freqIndices = vec([CartesianIndex{2}(i, j) for i in freqs, j in intersect(1:nReceivers, recChannels)]) else freqIndices = collect(vec(CartesianIndices((nFreq, nReceivers)))) filterFrequenciesBySelection!(freqIndices, f) From 462bf8040406f541cfe09569d76dbf627e4c4ebc Mon Sep 17 00:00:00 2001 From: Jonas Schumacher Date: Wed, 31 Jul 2024 16:49:56 +0200 Subject: [PATCH 13/31] Add functions for getting parts of a file as struct (#97) * Add functions for getting parts of a file as struct * Add tests and further functions * Fix tests --- src/MDF.jl | 2 +- src/MDFInMemory.jl | 53 +++++++++++++++- test/MDFInMemory.jl | 148 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 200 insertions(+), 3 deletions(-) diff --git a/src/MDF.jl b/src/MDF.jl index 20051f38..6ed92345 100644 --- a/src/MDF.jl +++ b/src/MDF.jl @@ -401,7 +401,7 @@ measFramePermutation(f::MDFFileV1)::Union{Vector{Int64}, Nothing} = nothing measFramePermutation(f::MDFFileV2)::Union{Vector{Int64}, Nothing} = @keyoptional f["/measurement/framePermutation"] measSparsityTransformation(f::MDFFileV1)::Union{String, Nothing} = nothing measSparsityTransformation(f::MDFFileV2)::Union{String, Nothing} = @keyoptional f["/measurement/sparsityTransformation"] -measSubsamplingIndices(f::MDFFileV2)::Union{Array{Integer, 4}, Nothing} = @keyoptional f["/measurement/subsamplingIndices"] +measSubsamplingIndices(f::MDFFileV2)::Union{Array{Int64, 4}, Nothing} = @keyoptional f["/measurement/subsamplingIndices"] fullFramePermutation(f::MDFFile) = fullFramePermutation(f, calibIsMeanderingGrid(f)) diff --git a/src/MDFInMemory.jl b/src/MDFInMemory.jl index bde3346e..401de0cf 100644 --- a/src/MDFInMemory.jl +++ b/src/MDFInMemory.jl @@ -418,7 +418,7 @@ mutable struct MDFv2Measurement <: MDFv2InMemoryPart "Name of the applied sparsity transformation; optional if !isSparsityTransformed" sparsityTransformation::Union{String, Nothing} "Subsampling indices \\beta{j,c,k,b}; optional if !isSparsityTransformed" - subsamplingIndices::Union{Array{Integer, 4}, Nothing} + subsamplingIndices::Union{Array{Int64, 4}, Nothing} function MDFv2Measurement(; data = missing, @@ -1303,6 +1303,57 @@ for (fieldname, fieldtype) in zip(fieldnames(MDFv2InMemory), fieldtypes(MDFv2InM end end +# Extract individual parts of an MDFFile as in-memory parts +for (partFieldnameStr, partPrefix) ∈ prefixes + partFieldnameSymbol = Symbol(partFieldnameStr) + partInMemorySymbol = Symbol(lowercase(partFieldnameStr[6:end])) + type_ = getfield(MPIFiles, partFieldnameSymbol) + fields = fieldnames(type_) + functionNames = [let fieldString=string(field_); partPrefix != "" ? Symbol(partPrefix*uppercase(fieldString[1])*fieldString[2:end]) : Symbol(fieldString) end for field_ ∈ fields if field_ ∉ [:drivefield, :receiver]] + + @eval begin + @doc $""" + $partFieldnameStr(file::MDFFile) + + Create a `$partFieldnameStr` from the respective section in the given `file`. + """ + function $partFieldnameSymbol(file::MDFFile)::$type_ + instance = $partFieldnameSymbol() + for (fieldname_, functionName_) in zip($fields, $functionNames) + setfield!(instance, fieldname_, eval(functionName_)(file)) + end + + return instance + end + end + + if !(partFieldnameStr == "MDFv2Drivefield" || partFieldnameStr == "MDFv2Receiver") + @eval begin + @doc $""" + $partFieldnameStr(file::MDFv2InMemory) + + Create a `$partFieldnameStr` from the respective section in the given `file`. + """ + function $partFieldnameSymbol(file::MDFv2InMemory)::$type_ + return file.$partInMemorySymbol + end + end + else + @eval begin + @doc $""" + $partFieldnameStr(file::MDFv2InMemory) + + Create a `$partFieldnameStr` from the respective section in the given `file`. + """ + function $partFieldnameSymbol(file::MDFv2InMemory)::$type_ + return file.acquisition.$partInMemorySymbol + end + end + end +end +MDFv2Drivefield(part::MDFv2Acquisition) = part.drivefield +MDFv2Receiver(part::MDFv2Acquisition) = part.receiver + # And some utility functions measIsCalibProcessed(mdf::MDFv2InMemory)::Union{Bool, Missing} = measIsFramePermutation(mdf) && measIsFourierTransformed(mdf) && diff --git a/test/MDFInMemory.jl b/test/MDFInMemory.jl index c2b4ec47..27769499 100644 --- a/test/MDFInMemory.jl +++ b/test/MDFInMemory.jl @@ -41,6 +41,7 @@ end fnMeasV2_converted = joinpath(tmpdir,"mdfim","measurement_V2_converted.mdf") fnSMV2_converted = joinpath(tmpdir,"mdfim","systemMatrix_V2_converted.mdf") + fnPart_converted = joinpath(tmpdir,"mdfim","imparts.mdf") @testset "Conversion" begin @testset "Fields" begin @@ -396,7 +397,8 @@ end order = "xyz", positions = fill(0.0, (3, O)), size = [1, 1, 1], - snr = fill(0.0, (K, C, J)) + snr = fill(0.0, (K, C, J)), + isMeanderingGrid = false # Not in specs for MDF v2! ) mdf.reconstruction = MDFv2Reconstruction( data = fill(0, (S, P, Q)), @@ -582,4 +584,148 @@ end test_mdf_replacement(mdf, :recoSize, [1, 1, 2], "The product of `size` with `[1, 1, 2]` must equal P.") end end + + @testset "Part retrieval" begin + A = 1 # tracer materials/injections for multi-color MPI + N = 2 # acquired frames (N = O + E), same as a spatial position for calibration + E = 1 # acquired background frames (E = N − O) + O = 1 # acquired foreground frames (O = N − E) + B = 1 # coefficients stored after sparsity transformation (B \\le O) + J = 1 # periods within one frame + Y = 1 # partitions of each patch position + C = 1 # receive channels + D = 1 # drive-field channels + F = 1 # frequencies describing the drive-field waveform + V = 800 # points sampled at receiver during one drive-field period + W = V # sampling points containing processed data (W = V if no frequency selection or bandwidth reduction has been applied) + K = Int(V/2 + 1)# frequencies describing the processed data (K = V/2 + 1 if no frequency selection or bandwidth reduction has been applied) + Q = 1 # frames in the reconstructed MPI data set + P = 1 # voxels in the reconstructed MPI data set + S = 1 # channels in the reconstructed MPI data set + + mdf = MDFv2InMemory() + mdf.root = MDFv2Root( + time=DateTime("2021-04-26T17:12:21.686"), + uuid=UUID("946a039e-48de-47ee-957e-a15af437e0be"), + version=VersionNumber("2.1.0") + ) + mdf.study = MDFv2Study( + description = "n.a.", + name = "n.a.", + number = 1, + time = DateTime("2021-04-26T17:12:21.686"), + uuid = UUID("f2f3ae66-2fc3-49f7-91f9-71c1c2dc15e7") + ) + mdf.experiment = MDFv2Experiment( + description = "n.a.", + isSimulation = true, + name = "n.a.", + number = 1, + subject = "n.a.", + uuid = UUID("3076d4bc-a2ef-46ef-8a99-dcd047379b8a") + ) + mdf.tracer = MDFv2Tracer( + batch = fill("n.a.", A), + concentration = fill(1.0, A), + injectionTime = fill(DateTime("2021-04-26T17:12:21.686"), A), + name = fill("MyAwesomeTracer", A), + solute = fill("n.a.", A), + vendor = fill("Me", A), + volume = fill(1.0, A), + ) + mdf.scanner = MDFv2Scanner( + boreSize = 0.3, + facility = "MyAwesomeInstitute", + manufacturer = "MeMyselfAndI", + name = "MyAwesomeScanner", + operator = "JustMe", + topology = "FFL" # Of course ;) + ) + drivefield = MDFv2Drivefield( + baseFrequency = 40e3, + cycle = 1600/40e3, + divider = fill(1600, (F, D)), + numChannels = D, + phase = fill(0.0, (F, D, J)), + strength = fill(1.0, (F, D, J)), + waveform = fill("sine", (F, D)), + ) + receiver = MDFv2Receiver( + bandwidth = 20e3, + dataConversionFactor = repeat([1/2^16 0]', outer=(1, C)), + inductionFactor = fill(1.0, C), + numChannels = C, + numSamplingPoints = V, + transferFunction = fill(1+0.5im, (K, C)), + unit = "V" + ) + mdf.acquisition = MDFv2Acquisition( + gradient = fill(0.0, (3, 3, Y, J)), + numAverages = 1, + numFrames = N, + numPeriodsPerFrame = J, + offsetField = fill(0.0, (3, Y, J)), + startTime = DateTime("2021-04-26T17:12:21.686"), + drivefield = drivefield, + receiver = receiver + ) + mdf.measurement = MDFv2Measurement(; + data = fill(0, (K, C, J, N)), + # framePermutation = fill(0, N), + # frequencySelection = collect(1:K), + isBackgroundCorrected = false, + isBackgroundFrame = vcat(fill(true, E), fill(false, O)), + isFastFrameAxis = false, + isFourierTransformed = false, + isFramePermutation = false, + isFrequencySelection = false, + isSparsityTransformed = false, + isSpectralLeakageCorrected = false, + isTransferFunctionCorrected = false, + # sparsityTransformation = "DCT-I", + # subsamplingIndices = fill(0, (B, K, C, J)) + ) + mdf.calibration = MDFv2Calibration( + deltaSampleSize = fill(0.001, 3), + fieldOfView = fill(0.2, 3), + fieldOfViewCenter = fill(0.0, 3), + method = "robot", + offsetFields = fill(0.0, (3, O)), + order = "xyz", + positions = fill(0.0, (3, O)), + size = [1, 1, 1], + snr = fill(0.0, (K, C, J)), + isMeanderingGrid = false + ) + mdf.reconstruction = MDFv2Reconstruction( + data = fill(0, (S, P, Q)), + fieldOfView = fill(0.2, 3), + fieldOfViewCenter = fill(0.0, 3), + isOverscanRegion = fill(false, P), + order = "xyz", + positions = fill(0.0, (3, P)), + size = [1, 1, 1] + ) + + saveasMDF(fnPart_converted, mdf) + file = MPIFile(fnPart_converted) + + for partFieldnameStr in keys(MPIFiles.prefixes) + partInMemorySymbol = Symbol(lowercase(partFieldnameStr[6:end])) + + partim = eval(Symbol(partFieldnameStr))(mdf) + part = eval(Symbol(partFieldnameStr))(file) + + if !(partFieldnameStr == "MDFv2Drivefield" || partFieldnameStr == "MDFv2Receiver") + @test partim == getfield(mdf, partInMemorySymbol) + @test all([getfield(part, fieldname_) == getfield(getfield(mdf, partInMemorySymbol), fieldname_) for fieldname_ ∈ fieldnames(typeof(part)) if fieldname_ ∉ [:drivefield, :receiver]]) + else + @test partim == getfield(mdf.acquisition, partInMemorySymbol) + @test all([getfield(part, fieldname_) == getfield(getfield(mdf.acquisition, partInMemorySymbol), fieldname_) for fieldname_ ∈ fieldnames(typeof(part))]) + end + end + + @test MDFv2Drivefield(MDFv2Acquisition(mdf)) == drivefield + @test MDFv2Receiver(MDFv2Acquisition(mdf)) == receiver + end end \ No newline at end of file From 5d9f1e9fba16809667bd78b0f19d740df44c1e65 Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 2 Aug 2024 17:07:24 +0200 Subject: [PATCH 14/31] Implement stopBands and tests --- src/FrequencyFilter.jl | 27 ++++++++++++++++++++++----- test/FrequencyFilter.jl | 23 +++++++++++++++++++++++ test/runtests.jl | 1 + 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index b2b11c24..938dffda 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -92,7 +92,7 @@ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0, end if !isnothing(stopBands) - filterFrequenciesByStopBands!(freqIndices, stopBands) + filterFrequenciesByStopBands!(freqIndices, f, stopBands; numPeriodGrouping = numPeriodGrouping) end return collect(vec(freqIndices)) @@ -185,11 +185,28 @@ function filterFrequenciesByStepsize!(indices, stepsize) end export filterFrequenciesByStopBands!, filterFrequenciesByStopBand! -function filterFrequenciesByStopBands!(indices, stopBands::Vector) - filter!(x -> any(stopBand -> !in(x[1], stopBand), stopBands), indices) +function filterFrequenciesByStopBands!(indices, f::MPIFile, stopBands::Vector; kwargs...) + for stopBand in stopBands + filterFrequenciesByStopBand!(indices, f, stopBand; kwargs...) + end +end +filterFrequenciesByStopBands!(indices, f::MPIFile, stopBand::Union{Vector{Int64}, UnitRange, NTuple}; kwargs...) = filterFrequenciesByStopBands!(indices, f, [stopBand]; kwargs...) +function filterFrequenciesByStopBand!(indices, f::MPIFile, stopBand::Vector{Int64}; numPeriodGrouping = 1) + if length(stopBand) != 2 + error("Stop band are only defined for a start and stop value. Found $(length(stopBand)) values") + end + nFreq = rxNumFrequencies(f, numPeriodGrouping) + minIdx = floor(Int, first(stopBand) / rxBandwidth(f) * (nFreq-1) ) + 1 + maxIdx = ceil(Int, last(stopBand) / rxBandwidth(f) * (nFreq-1) ) + 1 + return filterFrequenciesByStopBand!(indices, minIdx, maxIdx) +end +function filterFrequenciesByStopBand!(indices, f::MPIFile, stopBand::Union{UnitRange, NTuple{2, Int64}}; numPeriodGrouping = 1) + nFreq = rxNumFrequencies(f, numPeriodGrouping) + minIdx = floor(Int, first(stopBand) / rxBandwidth(f) * (nFreq-1) ) + 1 + maxIdx = ceil(Int, last(stopBand) / rxBandwidth(f) * (nFreq-1) ) + 1 + return filterFrequenciesByStopBand!(indices, minIdx, maxIdx) end -filterFrequenciesByStopBands!(indices, stopBand) = filterFrequenciesByStopBands!(indices, [stopBand]) -filterFrequenciesByStopBand!(indices, stopBand) = filter!(x -> !in(x[1], stopBand), indices) +filterFrequenciesByStopBand!(indices, minIdx, maxIdx) = filter!(x-> x[1] < minIdx || x[1] > maxIdx, indices) function sortFrequencies(indices, f::MPIFile; numPeriodGrouping = 1, stepsize = 1, sortBySNR = false, sortByMixFactors = false) if sortBySNR && !sortByMixFactors diff --git a/test/FrequencyFilter.jl b/test/FrequencyFilter.jl index f41500e0..7366617b 100644 --- a/test/FrequencyFilter.jl +++ b/test/FrequencyFilter.jl @@ -46,6 +46,28 @@ @test length(filterFrequencies(mdf, maxFreq = 0)) == numChannels skip = true end + @testset "Stop Bands" begin + @test length(filterFrequencies(mdf, stopBands = [(0, 0)])) == length(unfiltered) - numChannels + @test length(filterFrequencies(mdf, stopBands = [[0, 0]])) == length(unfiltered) - numChannels + @test length(filterFrequencies(mdf, stopBands = [0:0])) == length(unfiltered) - numChannels + + @test length(filterFrequencies(mdf, stopBands = (0, 0))) == length(unfiltered) - numChannels + @test length(filterFrequencies(mdf, stopBands = [0, 0])) == length(unfiltered) - numChannels + @test length(filterFrequencies(mdf, stopBands = 0:0)) == length(unfiltered) - numChannels + + @test length(filterFrequencies(mdf, stopBands = 0:bandwidth)) == 0 + + freqs = filterFrequencies(mdf, stopBands = 5:10) + @test length(freqs) == length(unfiltered) - length(5:10) * numChannels + @test !any(i -> in(i[1], (5:10) .+ 1), freqs) + + stopBands = [(5, 10), [15, 20], 18:22] + freqs = filterFrequencies(mdf, stopBands = stopBands) + @test length(freqs) == length(unfiltered) - (length(5:10) + length(15:22)) * numChannels + @test !any(i -> in(i[1], (5:10) .+ 1) || in(i[1], (15:22) .+ 1), freqs) + end + + snr = fill(0.0, nFreq, numChannels, 1) # Each freq has an SNR equal to its index snr[:, :, 1] .= 1:nFreq @@ -70,4 +92,5 @@ freqs = filterFrequencies(mdf, numUsedFreqs = 1) @test all(i -> i[1] == nFreq, freqs) skip = true end + end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 6080e3d6..8263ae33 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -34,6 +34,7 @@ include("MultiMPIFile.jl") include("Reco.jl") include("IMT.jl") include("TransferFunction.jl") +include("FrequencyFilter.jl") include("CustomSFMeas.jl") include("MDFInMemory.jl") include("MagneticFieldMeasurement.jl") From 9d215153da6b1670da4a8fc9c0d638e4023d2ccc Mon Sep 17 00:00:00 2001 From: Justin <38977690+jusack@users.noreply.github.com> Date: Fri, 6 Sep 2024 15:21:50 +0200 Subject: [PATCH 15/31] Added keyrequired to dfDivider (#99) --- src/MDF.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MDF.jl b/src/MDF.jl index 6ed92345..9f965d8b 100644 --- a/src/MDF.jl +++ b/src/MDF.jl @@ -226,7 +226,7 @@ dfBaseFrequency(f::MDFFile)::Union{Float64, Missing} = @keyrequired f["/acquisit dfCustomWaveform(f::MDFFileV2)::Union{String, Nothing} = @keyoptional f["/acquisition/drivefield/customWaveform"] dfDivider(f::MDFFileV1) = @keyrequired addTrailingSingleton( f["/acquisition/drivefield/divider"],2) -dfDivider(f::MDFFileV2) = f["/acquisition/drivefield/divider"] +dfDivider(f::MDFFileV2) = @keyrequired f["/acquisition/drivefield/divider"] dfWaveform(f::MDFFileV1) = "sine" function dfWaveform(f::MDFFileV2)::Union{Array{String, 2}, Missing} value = @keyrequired f["/acquisition/drivefield/waveform"] From 0dd1acca91b02940acbef881d43111eb7e283c20 Mon Sep 17 00:00:00 2001 From: Justin Ackers Date: Wed, 23 Oct 2024 10:00:47 +0200 Subject: [PATCH 16/31] added tests for rxFrequencies --- test/General.jl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/General.jl b/test/General.jl index f6c72293..74e0a608 100644 --- a/test/General.jl +++ b/test/General.jl @@ -328,3 +328,17 @@ end # test new measurement parameters @test [any(measIsBGFrame(mdf))] == measIsBGFrame(mdfavg) end + +@testset "Testing rxFrequencies" begin + f = MDFv2InMemory(Dict{Symbol,Any}(:rxNumSamplingPoints=>625, :rxBandwidth=>125e6/8/2)) + @test rxFrequencies(f)[1] ≈ 0.0 + @test rxFrequencies(f)[2] ≈ 25e3 + @test rxNumFrequencies(f) == 313 + @test rxNumFrequencies(f, 10) == 3126 + + f = MDFv2InMemory(Dict{Symbol,Any}(:rxNumSamplingPoints=>96, :rxBandwidth=>125e6/50/2)) + @test rxFrequencies(f)[1] ≈ 0.0 + @test rxFrequencies(f)[2] ≈ 26041.666666666666 + @test rxNumFrequencies(f) == 49 + @test rxNumFrequencies(f, 10) == 481 +end \ No newline at end of file From 9ea8e4af3709340757f8d7c465fdd359ef7f8292 Mon Sep 17 00:00:00 2001 From: Justin Ackers Date: Wed, 23 Oct 2024 10:38:49 +0200 Subject: [PATCH 17/31] fixed rxFrequencies --- src/Derived.jl | 15 ++++----------- test/General.jl | 8 ++++++++ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Derived.jl b/src/Derived.jl index 051b1197..107692ae 100644 --- a/src/Derived.jl +++ b/src/Derived.jl @@ -4,19 +4,12 @@ export acqNumFGFrames, acqNumBGFrames, acqOffsetFieldShift, acqFramePeriod, rxNumFrequencies, rxFrequencies, rxTimePoints, measFGFrameIdx, measBGFrameIdx, measBGFrameBlockLengths -rxNumFrequencies(f::MPIFile, numPeriodGrouping=1) = floor(Int,rxNumSamplingPoints(f)*numPeriodGrouping ./ 2 .+ 1) +rxNumFrequencies(f::MPIFile, numPeriodGrouping=1) = length(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping)) -function rxFrequencies(f::MPIFile) - numFreq = rxNumFrequencies(f) - a = collect(0:(numFreq-1))./(numFreq-1).*rxBandwidth(f) - return a -end +rxFrequencies(f::MPIFile, numPeriodGrouping=1) = rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)) |> collect + +rxTimePoints(f::MPIFile, numPeriodGrouping=1) = range(0, step=1/2rxBandwidth(f), length=rxNumSamplingPoints(f)*numPeriodGrouping) |> collect -function rxTimePoints(f::MPIFile) - numTP = rxNumSamplingPoints(f) - a = collect(0:(numTP-1))./(numTP).*dfCycle(f) - return a -end function acqGradientDiag(f::MPIFile) g = acqGradient(f) diff --git a/test/General.jl b/test/General.jl index 74e0a608..c059177d 100644 --- a/test/General.jl +++ b/test/General.jl @@ -335,10 +335,18 @@ end @test rxFrequencies(f)[2] ≈ 25e3 @test rxNumFrequencies(f) == 313 @test rxNumFrequencies(f, 10) == 3126 + @test rxFrequencies(f,10)[1] ≈ 0 + @test rxFrequencies(f,10)[2] ≈ 2500 + @test rxFrequencies(f,10)[11] ≈ 25e3 + @test rxTimePoints(f,10)[1:625] ≈ rxTimePoints(f)[1:625] f = MDFv2InMemory(Dict{Symbol,Any}(:rxNumSamplingPoints=>96, :rxBandwidth=>125e6/50/2)) @test rxFrequencies(f)[1] ≈ 0.0 @test rxFrequencies(f)[2] ≈ 26041.666666666666 @test rxNumFrequencies(f) == 49 @test rxNumFrequencies(f, 10) == 481 + @test rxFrequencies(f,10)[1] ≈ 0 + @test rxFrequencies(f,10)[2] ≈ 26041.666666666666/10 + @test rxFrequencies(f,10)[11] ≈ 26041.666666666666 + @test rxTimePoints(f,10)[1:96] ≈ rxTimePoints(f)[1:96] end \ No newline at end of file From 29ce3c5106d2a4796e890fd6f8f1d4dbee4f23cd Mon Sep 17 00:00:00 2001 From: Justin Ackers Date: Wed, 23 Oct 2024 10:39:18 +0200 Subject: [PATCH 18/31] use built-in functions --- src/Derived.jl | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/src/Derived.jl b/src/Derived.jl index 107692ae..9bf8a950 100644 --- a/src/Derived.jl +++ b/src/Derived.jl @@ -37,36 +37,8 @@ end acqNumFGFrames(f::MPIFile) = acqNumFrames(f) - acqNumBGFrames(f) acqNumBGFrames(f::MPIFile) = sum(measIsBGFrame(f)) -function measBGFrameIdx(f::MPIFile) - idx = zeros(Int64, acqNumBGFrames(f)) - j = 1 - mask = measIsBGFrame(f) - for i=1:acqNumFrames(f) - if mask[i] - idx[j] = i - j += 1 - end - end - return idx -end - -function measFGFrameIdx(f::MPIFile) - mask = measIsBGFrame(f) - if !any(mask) - #shortcut - return 1:acqNumFrames(f) - end - idx = zeros(Int64, acqNumFGFrames(f)) - j = 1 - for i=1:acqNumFrames(f) - if !mask[i] - idx[j] = i - j += 1 - end - end - return idx -end - +measBGFrameIdx(f::MPIFile) = findall(measIsBGFrame(f)) +measFGFrameIdx(f::MPIFile) = findall(.!measIsBGFrame(f)) function measBGFrameBlockLengths(mask) len = Vector{Int}(undef,0) From c2c21a84d16553c83e19be47ad3d64217eed2b35 Mon Sep 17 00:00:00 2001 From: Justin Ackers Date: Wed, 23 Oct 2024 11:48:59 +0200 Subject: [PATCH 19/31] increase julia compat, because LinearOperatorCollection requires 1.9 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 701920bd..aa37f462 100644 --- a/Project.toml +++ b/Project.toml @@ -57,7 +57,7 @@ Tar = "1" Unitful = "1.13, 1.14, 1.15, 1.16, 1.17" UnitfulAngles = "0.6.1, 0.7" UnitfulParsableString = "0.1.6" -julia = "1.5" +julia = "1.9" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" From 981c8566b8a341636ad53099259762d2a14a3a6f Mon Sep 17 00:00:00 2001 From: Justin Ackers Date: Wed, 23 Oct 2024 11:49:32 +0200 Subject: [PATCH 20/31] ignore new manifest files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f698500e..8a6861d6 100644 --- a/.gitignore +++ b/.gitignore @@ -25,5 +25,7 @@ test/data/* # committed for packages, but should be committed for applications that require a static # environment. Manifest.toml +# also ignore julia-version specific Manifests (>1.11) +Manifest-v*.toml .vscode \ No newline at end of file From 52721d64144a42e97f1ab451699a7193ad902e5c Mon Sep 17 00:00:00 2001 From: Justin Ackers Date: Wed, 23 Oct 2024 11:57:22 +0200 Subject: [PATCH 21/31] bumped Aqua + added missing compats --- Project.toml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index aa37f462..38ba9250 100644 --- a/Project.toml +++ b/Project.toml @@ -35,25 +35,37 @@ UnitfulAngles = "6fb2a4bd-7999-5318-a3b2-8ad61056cd98" UnitfulParsableString = "06c00241-927a-4d5b-bb5e-6b5a2ada3567" [compat] -Aqua = "0.6" +Aqua = "0.8" AxisArrays = "0.3, 0.4" CodecZlib = "0.7" +Dates = "1" DelimitedFiles = "1" DocStringExtensions = "0.8, 0.9" FFTW = "1.3" FileIO = "1.0" Graphics = "0.4, 1.0" HDF5 = "0.14, 0.15, 0.16, 0.17" +HTTP = "1" ImageAxes = "0.6" ImageMetadata = "0.9" Inflate = "0.1.2" +InteractiveUtils = "1" Interpolations = "0.12, 0.13, 0.14, 0.15" +LazyArtifacts = "1" +LinearAlgebra = "1" LinearOperatorCollection = "1, 2" +Mmap = "1" +Pkg = "1" +REPL = "1" +Random = "1" +Scratch = "1" SHA = "0.7" -StableRNGs = "1.0.0" +StableRNGs = "1" Statistics = "1" +Test = "1" TOML = "1" Tar = "1" +UUIDs = "1" Unitful = "1.13, 1.14, 1.15, 1.16, 1.17" UnitfulAngles = "0.6.1, 0.7" UnitfulParsableString = "0.1.6" From 873602cdec8963cf6d2201259231e3c2ad6a8536 Mon Sep 17 00:00:00 2001 From: Justin Ackers Date: Thu, 24 Oct 2024 11:38:55 +0200 Subject: [PATCH 22/31] move to StableRNG because 1.11 changed rng behaviour --- src/Brukerfile.jl | 4 ++-- src/Positions/Positions.jl | 10 +++++----- test/Positions.jl | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Brukerfile.jl b/src/Brukerfile.jl index 9b11ff19..903daa7c 100644 --- a/src/Brukerfile.jl +++ b/src/Brukerfile.jl @@ -186,7 +186,7 @@ studyNameOld(b::BrukerFile) = string(latin1toutf8(b["VisuSubjectId"])*latin1tout b["VisuStudyNumber"]) studyNumber(b::BrukerFile) = parse(Int64,b["VisuStudyNumber"]) function studyUuid(b::BrukerFile) - rng = MersenneTwister(hash(b["VisuStudyUid"])) # use VisuStudyUid as seed to generate uuid4 + rng = StableRNG(hash(b["VisuStudyUid"])) # use VisuStudyUid as seed to generate uuid4 return uuid4(rng) end studyDescription(b::BrukerFile) = "n.a." @@ -207,7 +207,7 @@ function experimentNumber(b::BrukerFile) end function experimentUuid(b::BrukerFile) - rng = MersenneTwister(hash(b["VisuUid"])) # use VisuUid as seed to generate uuid4 + rng = StableRNG(hash(b["VisuUid"])) # use VisuUid as seed to generate uuid4 return uuid4(rng) end experimentDescription(b::BrukerFile) = latin1toutf8(b["ACQ_scan_name"]) diff --git a/src/Positions/Positions.jl b/src/Positions/Positions.jl index 36def6f6..9f95ee31 100644 --- a/src/Positions/Positions.jl +++ b/src/Positions/Positions.jl @@ -489,8 +489,8 @@ function getindex(rpos::UniformRandomPositions{AxisAlignedBox}, i::Integer) throw(BoundsError(rpos,i)) else # make sure Positions are randomly generated from given seed - mersenneTwister = MersenneTwister(seed(rpos)) - rP = rand(mersenneTwister, 3, i)[:,i] + rng = StableRNG(seed(rpos)) + rP = rand(rng, 3, i)[:,i] return (rP.-0.5).*fieldOfView(rpos)+fieldOfViewCenter(rpos) end end @@ -500,9 +500,9 @@ function getindex(rpos::UniformRandomPositions{Ball}, i::Integer) throw(BoundsError(rpos,i)) else # make sure Positions are randomly generated from given seed - mersenneTwister = MersenneTwister(seed(rpos)) - D = rand(mersenneTwister, i)[i] - P = randn(mersenneTwister, 3, i)[:,i] + rng = StableRNG(seed(rpos)) + D = rand(rng, i)[i] + P = randn(rng, 3, i)[:,i] return radius(rpos)*D^(1/3)*normalize(P)+fieldOfViewCenter(rpos) end end diff --git a/test/Positions.jl b/test/Positions.jl index 275a2bd4..d559df6b 100644 --- a/test/Positions.jl +++ b/test/Positions.jl @@ -228,9 +228,9 @@ pospath = joinpath(tmpdir,"positions","Positions.h5") @test domain.fov == fov @test domain.center == ctr rP1 = UniformRandomPositions(N,seed,domain) - @test rP1[1] == [0.09954904813158394,-0.13791259323857274,-1.446939519855107]Unitful.mm - @test rP1[2] == [-0.9812009131891462,1.3767776289892044,1.4206979394110573]Unitful.mm - @test rP1[3] == [-0.5883911667396526,-0.9692742011014337,1.3707474722677764]Unitful.mm + @test rP1[1] == [0.24154458805558643, -0.9262755616254505, 1.413400404619357]Unitful.mm + @test rP1[2] == [0.7303493695629726, -0.9870943521080697, 0.6143278667437428]Unitful.mm + @test rP1[3] == [-0.17686922698492702, 0.911915746834733, 0.817151846349423]Unitful.mm @test_throws BoundsError rP1[0] @test_throws BoundsError rP1[4] h5open(pospath, "w") do file @@ -249,7 +249,7 @@ pospath = joinpath(tmpdir,"positions","Positions.h5") @test domain.radius == radius @test domain.center == ctr rP3 = UniformRandomPositions(N,seed,domain) - @test rP3[1] == [-6.715713750009747,0.4103832286623821,-4.525933276650638]Unitful.mm + @test rP3[1] == [1.9129764588101597, 5.876973430472611, 5.6027641441895994]Unitful.mm @test_throws BoundsError rP3[0] @test_throws BoundsError rP3[4] h5open(pospath, "w") do file From ad2aa5abb4cbdaa1fe527cb9ca7283ddee2aa8df Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 1 Nov 2024 12:00:35 +0100 Subject: [PATCH 23/31] Init conditional breakage tests --- .github/workflows/breakage.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml index 495e05b4..0ecbd786 100644 --- a/.github/workflows/breakage.yml +++ b/.github/workflows/breakage.yml @@ -1,15 +1,13 @@ name: Breakage # Based on: https://github.com/JuliaSmoothOptimizers/LinearOperators.jl/blob/main/.github/workflows/Breakage.yml on: - pull_request: - branches: - - master - push: - branches: - - master - tags: '*' + issue_comment: + types: [created, edited] + jobs: break: + name: Breakage + if: ${{github.event.issue.pull_request && contains(github.event.comment.body, '[breakage]')}} runs-on: ubuntu-latest strategy: fail-fast: false From e93379483f2a66a255e6ec693d872d2c938ab406 Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 1 Nov 2024 12:22:17 +0100 Subject: [PATCH 24/31] Refactor conditional breakage workflow to exclude comments from bots --- .github/workflows/breakage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml index 0ecbd786..ac620a2e 100644 --- a/.github/workflows/breakage.yml +++ b/.github/workflows/breakage.yml @@ -7,7 +7,7 @@ on: jobs: break: name: Breakage - if: ${{github.event.issue.pull_request && contains(github.event.comment.body, '[breakage]')}} + if: ${{ github.event.issue.pull_request && github.event.issue.user.type != 'Bot' && contains(github.event.comment.body, '[breakage]') }} runs-on: ubuntu-latest strategy: fail-fast: false From f7c62307109412dedafa1b6b7352eecfc0548144 Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 1 Nov 2024 12:31:59 +0100 Subject: [PATCH 25/31] Test breakage --- .github/workflows/breakage.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml index ac620a2e..c5077c04 100644 --- a/.github/workflows/breakage.yml +++ b/.github/workflows/breakage.yml @@ -6,7 +6,6 @@ on: jobs: break: - name: Breakage if: ${{ github.event.issue.pull_request && github.event.issue.user.type != 'Bot' && contains(github.event.comment.body, '[breakage]') }} runs-on: ubuntu-latest strategy: From 436e04c44bcfd8ab3bd7a1c62c16198580c8b9b2 Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 1 Nov 2024 13:12:45 +0100 Subject: [PATCH 26/31] Trigger breakage if label exists --- .github/workflows/breakage.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml index c5077c04..a96586d7 100644 --- a/.github/workflows/breakage.yml +++ b/.github/workflows/breakage.yml @@ -1,12 +1,15 @@ name: Breakage # Based on: https://github.com/JuliaSmoothOptimizers/LinearOperators.jl/blob/main/.github/workflows/Breakage.yml on: - issue_comment: - types: [created, edited] + pull_request: + branches: + - master + types: [opened, reopened, labeled, synchronize] + jobs: break: - if: ${{ github.event.issue.pull_request && github.event.issue.user.type != 'Bot' && contains(github.event.comment.body, '[breakage]') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'breakage') }} runs-on: ubuntu-latest strategy: fail-fast: false From 91b970441e89a78398740e2ef082b588d027f3f4 Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 1 Nov 2024 14:09:39 +0100 Subject: [PATCH 27/31] Make min und maxFreq inclusive --- src/FrequencyFilter.jl | 4 ++-- test/FrequencyFilter.jl | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index 938dffda..c6c57d01 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -115,7 +115,7 @@ function filterFrequenciesByMinFreq!(indices, f::MPIFile, minFreq; numPeriodGrou minIdx = floor(Int, minFreq / rxBandwidth(f) * (nFreq-1) ) + 1 return filterFrequenciesByMinIdx!(indices, minIdx) end -filterFrequenciesByMinIdx!(indices, minIdx) = minIdx > 0 ? filter!(x -> x[1] > minIdx, indices) : indices +filterFrequenciesByMinIdx!(indices, minIdx) = minIdx > 0 ? filter!(x -> x[1] >= minIdx, indices) : indices export filterFrequenciesByMaxFreq! function filterFrequenciesByMaxFreq!(indices, f::MPIFile, maxFreq; numPeriodGrouping = 1) @@ -123,7 +123,7 @@ function filterFrequenciesByMaxFreq!(indices, f::MPIFile, maxFreq; numPeriodGrou maxIdx = ceil(Int, maxFreq / rxBandwidth(f) * (nFreq-1) ) + 1 return filterFrequenciesByMaxIdx!(indices, maxIdx) end -filterFrequenciesByMaxIdx!(indices, maxIdx) = filter!(x-> x[1] < maxIdx, indices) +filterFrequenciesByMaxIdx!(indices, maxIdx) = filter!(x-> x[1] <= maxIdx, indices) export filterFrequenciesByMaxMixingOrder! filterFrequenciesByMaxMixingOrder!(indices, maxMixingOrder, f::MPIFile) = filterFrequenciesByMaxMixingOrder!(indices, maxMixingOrder, mixingFactors(f)) diff --git a/test/FrequencyFilter.jl b/test/FrequencyFilter.jl index 7366617b..f814a568 100644 --- a/test/FrequencyFilter.jl +++ b/test/FrequencyFilter.jl @@ -11,6 +11,7 @@ @test length(unfiltered) == (bandwidth + 1) * numChannels @test eltype(unfiltered) == CartesianIndex{2} nFreq = bandwidth + 1 + frequencies = rxFrequencies(mdf) @testset "Filter Receive Channels" begin @test length(filterFrequencies(mdf, recChannels = 1:numChannels)) == length(unfiltered) @@ -36,14 +37,15 @@ @testset "Min and Max Frequencies" begin # minFreq = 0 -> Offset Frequency with index 1 - @test length(filterFrequencies(mdf, minFreq = 0)) == length(unfiltered) - @test length(filterFrequencies(mdf, minFreq = 1)) == length(unfiltered) - numChannels skip = true - @test length(filterFrequencies(mdf, minFreq = bandwidth)) == numChannels skip = true - @test length(filterFrequencies(mdf, minFreq = nFreq)) == 0 - - @test length(filterFrequencies(mdf, maxFreq = bandwidth)) == length(unfiltered) - @test length(filterFrequencies(mdf, maxFreq = bandwidth - 1)) == length(unfiltered) - numChannels skip = true - @test length(filterFrequencies(mdf, maxFreq = 0)) == numChannels skip = true + @test length(filterFrequencies(mdf, minFreq = frequencies[1])) == length(unfiltered) + @test length(filterFrequencies(mdf, minFreq = 1.0)) == length(unfiltered) - numChannels + @test length(filterFrequencies(mdf, minFreq = frequencies[end])) == numChannels + @test length(filterFrequencies(mdf, minFreq = frequencies[end] + 1.0)) == 0 + + @test length(filterFrequencies(mdf, maxFreq = frequencies[end])) == length(unfiltered) + @test length(filterFrequencies(mdf, maxFreq = frequencies[end - 1])) == length(unfiltered) - numChannels + @test length(filterFrequencies(mdf, maxFreq = 0.0)) == numChannels + @test isempty(filterFrequencies(mdf, maxFreq = -1.0)) end @testset "Stop Bands" begin @@ -89,8 +91,8 @@ @test length(filterFrequencies(mdf, numUsedFreqs = nFreq + 1)) == length(unfiltered) skip = true @test length(filterFrequencies(mdf, numUsedFreqs = nFreq - 1)) == length(unfiltered) - numChannels skip = true - freqs = filterFrequencies(mdf, numUsedFreqs = 1) - @test all(i -> i[1] == nFreq, freqs) skip = true + #freqs = filterFrequencies(mdf, numUsedFreqs = 1) + #@test all(i -> i[1] == nFreq, freqs) skip = true end end \ No newline at end of file From 58ced7db88254babf97fb84004720844300ba9c8 Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 8 Nov 2024 18:42:39 +0100 Subject: [PATCH 28/31] Compute frq indices during filtering with rfftfreq --- src/FrequencyFilter.jl | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index c6c57d01..df2f3d3d 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -111,16 +111,14 @@ filterFrequenciesByChannel!(indices, channels, sorted = issorted(channels)) = fi export filterFrequenciesByMinFreq! function filterFrequenciesByMinFreq!(indices, f::MPIFile, minFreq; numPeriodGrouping = 1) - nFreq = rxNumFrequencies(f, numPeriodGrouping) - minIdx = floor(Int, minFreq / rxBandwidth(f) * (nFreq-1) ) + 1 + minIdx = searchsortedfirst(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), minFreq) return filterFrequenciesByMinIdx!(indices, minIdx) end filterFrequenciesByMinIdx!(indices, minIdx) = minIdx > 0 ? filter!(x -> x[1] >= minIdx, indices) : indices export filterFrequenciesByMaxFreq! function filterFrequenciesByMaxFreq!(indices, f::MPIFile, maxFreq; numPeriodGrouping = 1) - nFreq = rxNumFrequencies(f, numPeriodGrouping) - maxIdx = ceil(Int, maxFreq / rxBandwidth(f) * (nFreq-1) ) + 1 + maxIdx = searchsortedlast(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), maxFreq) return filterFrequenciesByMaxIdx!(indices, maxIdx) end filterFrequenciesByMaxIdx!(indices, maxIdx) = filter!(x-> x[1] <= maxIdx, indices) @@ -196,14 +194,13 @@ function filterFrequenciesByStopBand!(indices, f::MPIFile, stopBand::Vector{Int6 error("Stop band are only defined for a start and stop value. Found $(length(stopBand)) values") end nFreq = rxNumFrequencies(f, numPeriodGrouping) - minIdx = floor(Int, first(stopBand) / rxBandwidth(f) * (nFreq-1) ) + 1 - maxIdx = ceil(Int, last(stopBand) / rxBandwidth(f) * (nFreq-1) ) + 1 + minIdx = searchsortedlast(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), first(stopBand)) + maxIdx = searchsortedlast(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), last(stopBand)) return filterFrequenciesByStopBand!(indices, minIdx, maxIdx) end function filterFrequenciesByStopBand!(indices, f::MPIFile, stopBand::Union{UnitRange, NTuple{2, Int64}}; numPeriodGrouping = 1) - nFreq = rxNumFrequencies(f, numPeriodGrouping) - minIdx = floor(Int, first(stopBand) / rxBandwidth(f) * (nFreq-1) ) + 1 - maxIdx = ceil(Int, last(stopBand) / rxBandwidth(f) * (nFreq-1) ) + 1 + minIdx = searchsortedlast(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), first(stopBand)) + maxIdx = searchsortedlast(rfftfreq(rxNumSamplingPoints(f)*numPeriodGrouping, 2rxBandwidth(f)), last(stopBand)) return filterFrequenciesByStopBand!(indices, minIdx, maxIdx) end filterFrequenciesByStopBand!(indices, minIdx, maxIdx) = filter!(x-> x[1] < minIdx || x[1] > maxIdx, indices) From ae935a7e20e92c3e6c37162256f73a4be5a1845e Mon Sep 17 00:00:00 2001 From: Niklas Hackelberg Date: Thu, 14 Nov 2024 14:51:46 +0100 Subject: [PATCH 29/31] Create Semgrep.yml --- .github/workflows/Semgrep.yml | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/Semgrep.yml diff --git a/.github/workflows/Semgrep.yml b/.github/workflows/Semgrep.yml new file mode 100644 index 00000000..2f727b90 --- /dev/null +++ b/.github/workflows/Semgrep.yml @@ -0,0 +1,66 @@ +# Based on: +# https://semgrep.dev/docs/semgrep-ci/sample-ci-configs#github-actions +# https://0xdbe.github.io/GitHub-HowToEnableCodeScanningWithSemgrep/ +# https://medium.com/@mostafa.elnakeb/supercharging-your-code-quality-with-semgrep-sast-in-github-actions-c8f30eb26655 +# Name of this GitHub Actions workflow. +name: Semgrep OSS scan + +on: + # Scan on-demand through GitHub Actions interface: + workflow_dispatch: + branches: + - main + # Schedule the CI job (this method uses cron syntax): + schedule: + - cron: '0 0 * * 1' # Run at start of week + +jobs: + semgrep: + # User definable name of this GitHub Actions job. + name: semgrep-oss/scan + # If you are self-hosting, change the following `runs-on` value: + runs-on: ubuntu-latest + + steps: + # Checkout the repository. + - name: Clone source code + uses: actions/checkout@v4 + + # Checkout custom rules + - name: Checkout custom rules + uses: actions/checkout@v4 + with: + repository: JuliaComputing/semgrep-rules-julia + ref: main + path: ./JuliaRules + + # Prepare Python + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + + # Install Semgrep + - name: Install Semgrep + run: python3 -m pip install semgrep + + # Run Semgrep + - name: Scan with Semgrep + run: | + semgrep scan \ + --config ./JuliaRules/rules \ + --metrics=off \ + --sarif --output report.sarif \ + --oss-only \ + --exclude=JuliaRules + + - name: Save Semgrep report + uses: actions/upload-artifact@v4 + with: + name: report.sarif + path: report.sarif + + - name: Upload Semgrep report + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: report.sarif + category: semgrep From a6601c352b8e15fbfef999e7e2ae27e0183aa272 Mon Sep 17 00:00:00 2001 From: Niklas Hackelberg Date: Thu, 14 Nov 2024 14:52:58 +0100 Subject: [PATCH 30/31] Update Semgrep.yml for master branch --- .github/workflows/Semgrep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Semgrep.yml b/.github/workflows/Semgrep.yml index 2f727b90..62f63cfe 100644 --- a/.github/workflows/Semgrep.yml +++ b/.github/workflows/Semgrep.yml @@ -9,7 +9,7 @@ on: # Scan on-demand through GitHub Actions interface: workflow_dispatch: branches: - - main + - master # Schedule the CI job (this method uses cron syntax): schedule: - cron: '0 0 * * 1' # Run at start of week From 83d43592a3f2da33af41488152142fc085386644 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 21:46:56 +0100 Subject: [PATCH 31/31] Bump codecov/codecov-action from 4 to 5 (#114) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4 to 5. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v4...v5) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfff1bf7..78d8128b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v4 + - uses: codecov/codecov-action@v5 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: