From 898d289a522eed2f4c2ac26ed0e0cb4015d9a73f Mon Sep 17 00:00:00 2001 From: nHackel Date: Thu, 2 May 2024 14:34:28 +0200 Subject: [PATCH 01/25] Improve frequency filter performance for frequency selection + large num freqs --- src/FrequencyFilter.jl | 59 ++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index 5b464807..33392023 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -36,11 +36,15 @@ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0, nFreq = rxNumFrequencies(f, numPeriodGrouping) nReceivers = rxNumChannels(f) nPeriods = 1 #acqNumPeriodsPerFrame(f) - freqIndices = collect(vec(CartesianIndices((nFreq, nReceivers)))) + freqs = measIsFrequencySelection(f) ? measFrequencySelection(f) : 1:nFreq - filterFrequenciesBySelection!(freqIndices, f) - filterFrequenciesByChannel!(freqIndices, recChannels) - + if numPeriodGrouping == 1 + freqIndices = vec([CartesianIndex{2}(i, j) for i in freqs, j in recChannels]) + else + freqIndices = collect(vec(CartesianIndices((nFreq, nReceivers)))) + filterFrequenciesBySelection!(freqIndices, f) + filterFrequenciesByChannel!(freqIndices, recChannels) + end if minFreq > 0 filterFrequenciesByMinFreq!(freqIndices, f, minFreq) end @@ -64,12 +68,9 @@ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0, SNRAll = nothing if SNRThresh > 0 || numUsedFreqs > 0 - SNR = zeros(nFreq, nReceivers) - idx = measIsFrequencySelection(f) ? measFrequencySelection(f) : idx = 1:nFreq - SNRAll = calibSNR(f) if !isnothing(SNRAll) - SNR[idx,:] = SNRAll[:,:,1] + SNR = Dict(CartesianIndex(freqs[i], j) => SNRAll[i, j, 1] for i = 1:length(freqs), j in recChannels) end end @@ -86,42 +87,42 @@ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0, filterFrequenciesByStepsize!(freqIndices, stepsize) end - return freqIndices + return collect(vec(freqIndices)) end export filterFrequenciesBySelection! -function filterFrequenciesBySelection!(indices::Vector{CartesianIndex{2}}, f::MPIFile) +function filterFrequenciesBySelection!(indices, f::MPIFile) if measIsFrequencySelection(f) return filterFrequenciesBySelection!(indices, measFrequencySelection(f)) end end -filterFrequenciesBySelection!(indices::Vector{CartesianIndex{2}}, selection::Vector{Int64}) = filter!(x -> in(x[1], selection), indices) +filterFrequenciesBySelection!(indices, selection::Vector{Int64}, sorted = issorted(selection)) = filter!(x -> sorted ? insorted(x[1], selection) : in(x[1], selection), indices) export filterFrequenciesByChannel! -filterFrequenciesByChannel!(indices::Vector{CartesianIndex{2}}, channels) = filter!(x-> in(x[2], channels), indices) +filterFrequenciesByChannel!(indices, channels) = filter!(x-> in(x[2], channels), indices) export filterFrequenciesByMinFreq! -function filterFrequenciesByMinFreq!(indices::Vector{CartesianIndex{2}}, f::MPIFile, minFreq; numPeriodGrouping = 1) +function filterFrequenciesByMinFreq!(indices, f::MPIFile, minFreq; numPeriodGrouping = 1) nFreq = rxNumFrequencies(f, numPeriodGrouping) minIdx = floor(Int, minFreq / rxBandwidth(f) * (nFreq-1) ) + 1 return filterFrequenciesByMinIdx!(indices, minIdx) end -filterFrequenciesByMinIdx!(indices::Vector{CartesianIndex{2}}, minIdx) = minIdx > 0 ? filter!(x -> x[1] > minIdx, indices) : indices +filterFrequenciesByMinIdx!(indices, minIdx) = minIdx > 0 ? filter!(x -> x[1] > minIdx, indices) : indices export filterFrequenciesByMaxFreq! -function filterFrequenciesByMaxFreq!(indices::Vector{CartesianIndex{2}}, f::MPIFile, maxFreq; numPeriodGrouping = 1) +function filterFrequenciesByMaxFreq!(indices, f::MPIFile, maxFreq; numPeriodGrouping = 1) nFreq = rxNumFrequencies(f, numPeriodGrouping) maxIdx = ceil(Int, maxFreq / rxBandwidth(f) * (nFreq-1) ) + 1 return filterFrequenciesByMaxIdx!(indices, maxIdx) end -filterFrequenciesByMaxIdx!(indices::Vector{CartesianIndex{2}}, maxIdx) = filter!(x-> x[1] < maxIdx, indices) +filterFrequenciesByMaxIdx!(indices, maxIdx) = filter!(x-> x[1] < maxIdx, indices) export filterFrequenciesByMaxMixingOrder! -filterFrequenciesByMaxMixingOrder!(indices::Vector{CartesianIndex{2}}, maxMixingOrder, f::MPIFile) = filterFrequenciesByMaxMixingOrder!(indices, maxMixingOrder, mixingFactors(f)) -filterFrequenciesByMaxMixingOrder!(indices::Vector{CartesianIndex{2}}, maxMixingOrder, mf::Matrix) = filter!(x-> mf[x[1], 4] <= maxMixingOrder, indices) +filterFrequenciesByMaxMixingOrder!(indices, maxMixingOrder, f::MPIFile) = filterFrequenciesByMaxMixingOrder!(indices, maxMixingOrder, mixingFactors(f)) +filterFrequenciesByMaxMixingOrder!(indices, maxMixingOrder, mf::Matrix) = filter!(x-> mf[x[1], 4] <= maxMixingOrder, indices) export filterFrequenciesByNumSidebandFreqs! -function filterFrequenciesByNumSidebandFreqs!(indices::Vector{CartesianIndex{2}}, numSidebandFreqs::Int64, f::MPIFile; numPeriodGrouping = 1) +function filterFrequenciesByNumSidebandFreqs!(indices, numSidebandFreqs::Int64, f::MPIFile; numPeriodGrouping = 1) # https://en.wikipedia.org/wiki/Sideband # Because of period grouping we have more frequencies than in original data @@ -140,17 +141,19 @@ function filterFrequenciesByNumSidebandFreqs!(indices::Vector{CartesianIndex{2}} end export filterFrequenciesBySNRThresh! -function filterFrequenciesBySNRThresh!(indices::Vector{CartesianIndex{2}}, f::MPIFile, snrthresh; numPeriodGrouping = 1) +function filterFrequenciesBySNRThresh!(indices, f::MPIFile, snrthresh; numPeriodGrouping = 1) SNR = getCalibSNR(f, numPeriodGrouping = numPeriodGrouping) return filterFrequenciesBySNRThresh!(indices, snrthresh, SNR) end -filterFrequenciesBySNRThresh!(indices::Vector{CartesianIndex{2}}, SNRThresh, SNR) = filter!(x-> SNR[x] >= SNRThresh , indices) +filterFrequenciesBySNRThresh!(indices, SNRThresh, SNR) = filter!(x-> SNR[x] >= SNRThresh , indices) +filterFrequenciesBySNRThresh!(indices, SNRThresh, SNR::Dict{CartesianIndex{2}, Float64}) = filter!(x-> get(SNR, x, 0.0) >= SNRThresh , indices) + export filterFrequenciesByNumUsedFrequencies! -function filterFrequenciesByNumUsedFrequencies!(indices::Vector{CartesianIndex{2}}, f::MPIFile, maxFreq) +function filterFrequenciesByNumUsedFrequencies!(indices, f::MPIFile, maxFreq) error("Not implemented") end -filterFrequenciesByNumUsedFrequencies!(indices::Vector{CartesianIndex{2}}, maxIdx) = error("not implemented") +filterFrequenciesByNumUsedFrequencies!(indices, maxIdx) = error("not implemented") #= numFreqsAlreadyFalse = sum(!freqMask) numFreqsFalse = round(Int,length(freqMask)* (1-numUsedFreqs)) @@ -168,12 +171,12 @@ filterFrequenciesByNumUsedFrequencies!(indices::Vector{CartesianIndex{2}}, maxId =# export filterFrequenciesByStepsize! -function filterFrequenciesByStepsize!(indices::Vector{CartesianIndex{2}}, stepsize) +function filterFrequenciesByStepsize!(indices, stepsize) stepIndices = 1:stepsize:maximum(map(x -> x[1], indices)) filter!(x -> in(x[1], stepIndices), indices) end -function sortFrequencies(indices::Vector{CartesianIndex{2}}, f::MPIFile; numPeriodGrouping = 1, stepsize = 1, sortBySNR = false, sortByMixFactors = false) +function sortFrequencies(indices, f::MPIFile; numPeriodGrouping = 1, stepsize = 1, sortBySNR = false, sortByMixFactors = false) if sortBySNR && !sortByMixFactors indices = sortFrequenciesBySNR(indices, f, numPeriodGrouping = numPeriodGrouping, stepsize = stepsize) elseif !sortBySNR && sortByMixFactors @@ -185,11 +188,11 @@ function sortFrequencies(indices::Vector{CartesianIndex{2}}, f::MPIFile; numPeri end export sortFrequenciesBySNR -function sortFrequenciesBySNR(indices::Vector{CartesianIndex{2}}, f::MPIFile; numPeriodGrouping = 1, stepsize = 1) +function sortFrequenciesBySNR(indices, f::MPIFile; numPeriodGrouping = 1, stepsize = 1) SNR = getCalibSNR(f, numPeriodGrouping = numPeriodGrouping, stepsize = stepsize) sortFrequenciesBySNR(indices, SNR) end -sortFrequenciesBySNR(indices::Vector{CartesianIndex{2}}, SNR::AbstractArray) = indices[reverse(sortperm(SNR[indices]),dims=1)] +sortFrequenciesBySNR(indices, SNR::AbstractArray) = indices[reverse(sortperm(SNR[indices]),dims=1)] export sortFrequenciesByMixFactors function sortFrequenciesByMixFactors(indices, f::MPIFile; numPeriodGrouping = 1) @@ -203,7 +206,7 @@ function sortFrequenciesByMixFactors(indices, f::MPIFile; numPeriodGrouping = 1) end sortFrequenciesByMixFactors(indices, mfNorm) end -sortFrequenciesByMixFactors(indices::Vector{CartesianIndex{2}}, mfNorm::AbstractArray) = indices[sortperm(mfNorm[indices])] +sortFrequenciesByMixFactors(indices, mfNorm::AbstractArray) = indices[sortperm(mfNorm[indices])] function rowsToSubsampledRows(f::MPIFile, rows) From 5dcca8aa99519eac6b278a37a3aabbafeb5139ff Mon Sep 17 00:00:00 2001 From: nHackel Date: Thu, 2 May 2024 15:43:46 +0200 Subject: [PATCH 02/25] Add permuteDims path to getSM with measIsFourierSpace --- src/MDFCommon.jl | 7 +++++-- src/SystemMatrix.jl | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/MDFCommon.jl b/src/MDFCommon.jl index 2c225356..8199c4e1 100644 --- a/src/MDFCommon.jl +++ b/src/MDFCommon.jl @@ -25,8 +25,7 @@ function measDataTDPeriods(f::Union{MDFFileV2, MDFv2InMemory}, periods=1:acqNumP end function systemMatrix(f::Union{MDFFileV2, MDFv2InMemory}, rows, bgCorrection=true) - if !experimentHasMeasurement(f) || !measIsFastFrameAxis(f) || - !measIsFourierTransformed(f) + if !experimentHasMeasurement(f) || !measIsFourierTransformed(f) return nothing end @@ -34,6 +33,10 @@ function systemMatrix(f::Union{MDFFileV2, MDFv2InMemory}, rows, bgCorrection=tru data_ = measDataRaw(f) + if !measIsFastFrameAxis(f) + data_ = permutedims(data_, [4, 1, 2, 3]) + end + data_ = data_[:, rows_, :] data = reshape(data_, Val(2)) diff --git a/src/SystemMatrix.jl b/src/SystemMatrix.jl index 8fcb489a..0ec9f4ea 100644 --- a/src/SystemMatrix.jl +++ b/src/SystemMatrix.jl @@ -19,7 +19,7 @@ function getSystemMatrix(f::MPIFile, frequencies=nothing; frequencies = filterFrequencies(f) end - if measIsFastFrameAxis(f) && measIsFourierTransformed(f) + if measIsFourierTransformed(f) && numPeriodGrouping == 1 # This is the fast path if the SM lays on disk in the required format data = systemMatrix(f, frequencies, bgCorrection) else From 230ca39bdb97052243352859b49eb9fd459b9edf Mon Sep 17 00:00:00 2001 From: nHackel Date: Thu, 2 May 2024 18:02:32 +0200 Subject: [PATCH 03/25] Remove io-read from hot loop in rowToSubsampledRows --- src/FrequencyFilter.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index 33392023..a0b5b727 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -215,7 +215,7 @@ function rowsToSubsampledRows(f::MPIFile, rows) tmp = Array{Union{Nothing, CartesianIndex{2}}}(undef, rxNumFrequencies(f), rxNumChannels(f)) idxAvailable = measFrequencySelection(f) for (i, idx) in enumerate(idxAvailable) - for d=1:rxNumChannels(f) + for d=1:size(tmp, 2) tmp[idx, d] = CartesianIndex(i, d) end end From 901f39c868e01a558a70472661e3033d622e9693 Mon Sep 17 00:00:00 2001 From: jonschumacher Date: Thu, 2 May 2024 21:34:11 +0200 Subject: [PATCH 04/25] Fix issues with optional frame permutation --- src/MDFCommon.jl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/MDFCommon.jl b/src/MDFCommon.jl index 8199c4e1..853e5329 100644 --- a/src/MDFCommon.jl +++ b/src/MDFCommon.jl @@ -51,7 +51,7 @@ function systemMatrix(f::Union{MDFFileV2, MDFv2InMemory}, rows, bgCorrection=tru subsamplingIndices_ = tmp[:, rows_, :] subsamplingIndices = reshape(subsamplingIndices_, Val(2)) - for l=1:size(fgdata,2) + for l ∈ axes(fgdata, 2) dataBackTrafo[:,l] .= 0.0 dataBackTrafo[subsamplingIndices[:,l],l] .= fgdata[:,l] dataBackTrafo[:,l] .= adjoint(B) * vec(dataBackTrafo[:,l]) @@ -62,7 +62,14 @@ function systemMatrix(f::Union{MDFFileV2, MDFv2InMemory}, rows, bgCorrection=tru if bgCorrection # this assumes equidistant bg frames @debug "Applying bg correction on system matrix (MDF)" bgdata = data[measBGFrameIdx(f),:] - blockLen = measBGFrameBlockLengths( invpermute!(deepcopy(measIsBGFrame(f)), measFramePermutation(f)) ) # Added deepcopy to be side-effect free in in-memory MDF + + if measIsFramePermutation(f) + mask = invpermute!(deepcopy(measIsBGFrame(f)), measFramePermutation(f)) # Added deepcopy to be side-effect free in in-memory MDF + else + mask = deepcopy(measIsBGFrame(f)) # Added deepcopy to be side-effect free in in-memory MDF + end + blockLen = measBGFrameBlockLengths(mask) + st = 1 for j=1:length(blockLen) bgdata[st:st+blockLen[j]-1,:] .= @@ -72,10 +79,10 @@ function systemMatrix(f::Union{MDFFileV2, MDFv2InMemory}, rows, bgCorrection=tru bgdataInterp = interpolate(bgdata, (BSpline(Linear()), NoInterp())) # Cubic does not work for complex numbers - origIndex = measFramePermutation(f) M = size(fgdata,1) K = size(bgdata,1) N = M + K + origIndex = measIsFramePermutation(f) ? measFramePermutation(f) : 1:N for m=1:M alpha = (origIndex[m]-1)/(N-1)*(K-1)+1 for k=1:size(fgdata,2) From e518496dc273b3bc97cd80987617f922eb91405e Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 3 May 2024 09:27:32 +0200 Subject: [PATCH 05/25] Use array for SNR instead of Dict again, avoids regression for small #freqs --- src/FrequencyFilter.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index a0b5b727..9a4a281a 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -68,9 +68,11 @@ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0, SNRAll = nothing if SNRThresh > 0 || numUsedFreqs > 0 + SNR = zeros(eltype(SNRThresh), nFreq, nReceivers) + SNRAll = calibSNR(f) if !isnothing(SNRAll) - SNR = Dict(CartesianIndex(freqs[i], j) => SNRAll[i, j, 1] for i = 1:length(freqs), j in recChannels) + SNR[freqs,:] = SNRAll[:,:,1] end end @@ -145,7 +147,7 @@ function filterFrequenciesBySNRThresh!(indices, f::MPIFile, snrthresh; numPeriod SNR = getCalibSNR(f, numPeriodGrouping = numPeriodGrouping) return filterFrequenciesBySNRThresh!(indices, snrthresh, SNR) end -filterFrequenciesBySNRThresh!(indices, SNRThresh, SNR) = filter!(x-> SNR[x] >= SNRThresh , indices) +filterFrequenciesBySNRThresh!(indices, SNRThresh, SNR::Matrix) = filter!(x-> SNR[x] >= SNRThresh , indices) filterFrequenciesBySNRThresh!(indices, SNRThresh, SNR::Dict{CartesianIndex{2}, Float64}) = filter!(x-> get(SNR, x, 0.0) >= SNRThresh , indices) From aac0b808d40f181b3c8acba4b505f9839dee036c Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 3 May 2024 09:29:46 +0200 Subject: [PATCH 06/25] Improve stepsize filtering --- src/FrequencyFilter.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index 9a4a281a..a7d150c8 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -175,7 +175,7 @@ filterFrequenciesByNumUsedFrequencies!(indices, maxIdx) = error("not implemented export filterFrequenciesByStepsize! function filterFrequenciesByStepsize!(indices, stepsize) stepIndices = 1:stepsize:maximum(map(x -> x[1], indices)) - filter!(x -> in(x[1], stepIndices), indices) + filter!(x -> insorted(x[1], stepIndices), indices) end function sortFrequencies(indices, f::MPIFile; numPeriodGrouping = 1, stepsize = 1, sortBySNR = false, sortByMixFactors = false) From 8b9fa10d3b723efa5108ecb1e1a882690826a258 Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 3 May 2024 10:06:18 +0200 Subject: [PATCH 07/25] Use insorted for filter stepsize --- src/FrequencyFilter.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index a7d150c8..65ffedfc 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -101,7 +101,7 @@ end filterFrequenciesBySelection!(indices, selection::Vector{Int64}, sorted = issorted(selection)) = filter!(x -> sorted ? insorted(x[1], selection) : in(x[1], selection), indices) export filterFrequenciesByChannel! -filterFrequenciesByChannel!(indices, channels) = filter!(x-> in(x[2], channels), indices) +filterFrequenciesByChannel!(indices, channels, sorted = issorted(channels)) = filter!(x-> sorted ? insorted(x[2], channels) : in(x[2], channels), indices) export filterFrequenciesByMinFreq! function filterFrequenciesByMinFreq!(indices, f::MPIFile, minFreq; numPeriodGrouping = 1) From 507fa2b84b635e0639ae050f2f5eb35294686234 Mon Sep 17 00:00:00 2001 From: jonschumacher Date: Wed, 22 May 2024 21:10:15 +0200 Subject: [PATCH 08/25] Prevent error when loading sf with `bgCorrection = true` when having no background frames --- src/MDFCommon.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/MDFCommon.jl b/src/MDFCommon.jl index 2c225356..585e0da7 100644 --- a/src/MDFCommon.jl +++ b/src/MDFCommon.jl @@ -56,7 +56,7 @@ function systemMatrix(f::Union{MDFFileV2, MDFv2InMemory}, rows, bgCorrection=tru fgdata = dataBackTrafo end - if bgCorrection # this assumes equidistant bg frames + if bgCorrection && length(measBGFrameIdx(f)) > 0 # this assumes equidistant bg frames @debug "Applying bg correction on system matrix (MDF)" bgdata = data[measBGFrameIdx(f),:] blockLen = measBGFrameBlockLengths( invpermute!(deepcopy(measIsBGFrame(f)), measFramePermutation(f)) ) # Added deepcopy to be side-effect free in in-memory MDF @@ -79,6 +79,8 @@ function systemMatrix(f::Union{MDFFileV2, MDFv2InMemory}, rows, bgCorrection=tru fgdata[m,k] -= bgdataInterp(alpha,k) end end + elseif bgCorrection && length(measBGFrameIdx(f)) == 0 + @warn "Ignoring parameter `bgCorrection` since there are no background frames in the file." end return fgdata end From 4de30362ff54a7e25011ff22b617d45a3b4a6563 Mon Sep 17 00:00:00 2001 From: Jonas Schumacher Date: Thu, 23 May 2024 09:21:54 +0200 Subject: [PATCH 09/25] Fix ambiguities --- src/FrequencyFilter.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index 65ffedfc..4db26b8c 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -143,12 +143,12 @@ function filterFrequenciesByNumSidebandFreqs!(indices, numSidebandFreqs::Int64, end export filterFrequenciesBySNRThresh! -function filterFrequenciesBySNRThresh!(indices, f::MPIFile, snrthresh; numPeriodGrouping = 1) - SNR = getCalibSNR(f, numPeriodGrouping = numPeriodGrouping) - return filterFrequenciesBySNRThresh!(indices, snrthresh, SNR) +function filterFrequenciesBySNRThresh!(indices, f::MPIFile, snrthresh::T; numPeriodGrouping = 1) where T <: Real + snr = getCalibSNR(f, numPeriodGrouping = numPeriodGrouping) + return filterFrequenciesBySNRThresh!(indices, snrthresh, snr) end -filterFrequenciesBySNRThresh!(indices, SNRThresh, SNR::Matrix) = filter!(x-> SNR[x] >= SNRThresh , indices) -filterFrequenciesBySNRThresh!(indices, SNRThresh, SNR::Dict{CartesianIndex{2}, Float64}) = filter!(x-> get(SNR, x, 0.0) >= SNRThresh , indices) +filterFrequenciesBySNRThresh!(indices, snrthresh::T, snr::Matrix) where T <: Real = filter!(x-> snr[x] >= snrthresh , indices) +filterFrequenciesBySNRThresh!(indices, snrthresh::T, snr::Dict{CartesianIndex{2}, Float64}) where T <: Real = filter!(x-> get(snr, x, 0.0) >= snrthresh , indices) export filterFrequenciesByNumUsedFrequencies! From e54dd97e9eca72f83d0e180e39e17ae3271ffef9 Mon Sep 17 00:00:00 2001 From: Jonas Schumacher Date: Thu, 23 May 2024 09:56:30 +0200 Subject: [PATCH 10/25] Fix tests --- src/FrequencyFilter.jl | 15 +++++++-------- test/General.jl | 24 ++++++++++++------------ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/FrequencyFilter.jl b/src/FrequencyFilter.jl index 4db26b8c..f5f60a1b 100644 --- a/src/FrequencyFilter.jl +++ b/src/FrequencyFilter.jl @@ -7,7 +7,7 @@ function getCalibSNR(f::MPIFile; numPeriodGrouping = 1, stepsize = 1) idx = measIsFrequencySelection(f) ? measFrequencySelection(f) : idx = 1:nFreq SNRAll = calibSNR(f) - if SNRAll != nothing + if !isnothing(SNRAll) SNR[idx,:] = SNRAll[:,:,1] end @@ -68,11 +68,11 @@ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0, SNRAll = nothing if SNRThresh > 0 || numUsedFreqs > 0 - SNR = zeros(eltype(SNRThresh), nFreq, nReceivers) + SNR = zeros(Float64, nFreq, nReceivers) SNRAll = calibSNR(f) if !isnothing(SNRAll) - SNR[freqs,:] = SNRAll[:,:,1] + SNR[freqs, :] = SNRAll[:, :, 1] end end @@ -84,7 +84,6 @@ function filterFrequencies(f::MPIFile; SNRThresh=-1, minFreq=0, error("It is not possible to use SNRThresh and SNRFactorUsedFreq similtaneously") end - if stepsize > 1 filterFrequenciesByStepsize!(freqIndices, stepsize) end @@ -144,11 +143,11 @@ end export filterFrequenciesBySNRThresh! function filterFrequenciesBySNRThresh!(indices, f::MPIFile, snrthresh::T; numPeriodGrouping = 1) where T <: Real - snr = getCalibSNR(f, numPeriodGrouping = numPeriodGrouping) - return filterFrequenciesBySNRThresh!(indices, snrthresh, snr) + SNR = getCalibSNR(f, numPeriodGrouping = numPeriodGrouping) + return filterFrequenciesBySNRThresh!(indices, snrthresh, SNR) end -filterFrequenciesBySNRThresh!(indices, snrthresh::T, snr::Matrix) where T <: Real = filter!(x-> snr[x] >= snrthresh , indices) -filterFrequenciesBySNRThresh!(indices, snrthresh::T, snr::Dict{CartesianIndex{2}, Float64}) where T <: Real = filter!(x-> get(snr, x, 0.0) >= snrthresh , indices) +filterFrequenciesBySNRThresh!(indices, SNRThresh::T, SNR::Matrix) where T <: Real = filter!(x-> SNR[x] >= SNRThresh , indices) +filterFrequenciesBySNRThresh!(indices, SNRThresh::T, SNR::Dict{CartesianIndex{2}, Float64}) where T <: Real = filter!(x-> get(SNR, x, 0.0) >= SNRThresh , indices) export filterFrequenciesByNumUsedFrequencies! diff --git a/test/General.jl b/test/General.jl index 9b24da47..f6c72293 100644 --- a/test/General.jl +++ b/test/General.jl @@ -165,27 +165,27 @@ #@test size(filterFrequencies(sm, numUsedFreqs = 100)) == (100,) # not working @test size(getSystemMatrix(sm,collect(vec(CartesianIndices((10, 1)))))) == (1936,10) - @test size(getSystemMatrix(sm,collect(vec(CartesianIndices((10, 1)))),loadasreal=true)) == (1936,20) - @test size(getSystemMatrix(sm,collect(vec(CartesianIndices((10, 1)))),bgCorrection=true)) == (1936,10) + @test size(getSystemMatrix(sm,collect(vec(CartesianIndices((10, 1)))), loadasreal=true)) == (1936,20) + @test size(getSystemMatrix(sm,collect(vec(CartesianIndices((10, 1)))), bgCorrection=true)) == (1936,10) # test on the data level if the conversion was successful SNRThresh = 2 - freq = filterFrequencies(smBruker,SNRThresh=SNRThresh) - SBruker = getSystemMatrix(smBruker,frequencies=freq) - S = getSystemMatrix(sm,frequencies=freq) + freq = filterFrequencies(smBruker, SNRThresh=SNRThresh) + SBruker = getSystemMatrix(smBruker, frequencies=freq) + S = getSystemMatrix(sm, frequencies=freq) - relativeDeviation = zeros(Float32,length(freq)) + relativeDeviation = zeros(Float32, length(freq)) for f in 1:length(map(f->f[1], freq)) - relativeDeviation[f] = norm(SBruker[:,f]-S[:,f])/norm(SBruker[:,f]) + relativeDeviation[f] = norm(SBruker[:,f]-S[:,f]) / norm(SBruker[:,f]) end # test if relative deviation for most of the frequency components is below 0.003 - @test quantile(relativeDeviation,0.95)<0.003 + @test quantile(relativeDeviation, 0.95) < 0.003 end # Next test checks if the cached system matrix is the same as the one loaded # from the raw data S_loadedfromraw = getMeasurementsFD(smBrukerPretendToBeMeas, - frames=1:acqNumFGFrames(smBrukerPretendToBeMeas),sortFrames=true, - spectralLeakageCorrection=false,transposed=true,tfCorrection=false) + frames=1:acqNumFGFrames(smBrukerPretendToBeMeas), sortFrames=true, + spectralLeakageCorrection=false, transposed=true, tfCorrection=false) S_loadedfromproc = systemMatrix(smBruker) @@ -232,7 +232,7 @@ relativeDeviation[f] = norm(S1[:,f]-S2[:,f])/norm(S2[:,f]) end # test if relative deviation for most of the frequency components is below 0.003 - @test quantile(relativeDeviation,0.95)<0.14 + @test quantile(relativeDeviation, 0.95)<0.14 # Calibration file 1D @@ -253,7 +253,7 @@ @info "Test $sm" @test size( systemMatrixWithBG(sm) ) == (67,52,3,1) - @test size( systemMatrix(sm,collect(vec(CartesianIndices((10, 1))))) ) == (60,10) + @test size( systemMatrix(sm, collect(vec(CartesianIndices((10, 1))))) ) == (60,10) @test size( systemMatrix(sm) ) == (60,52,3,1) @test size(calibSNR(sm)) == (52, 3, 1) From 2b1a7955b90227c2c0b931e2fd9dec324542dab1 Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 24 May 2024 11:06:05 +0200 Subject: [PATCH 11/25] Add bgFrames to select bgCorrection frames --- src/Measurements.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Measurements.jl b/src/Measurements.jl index 07b6efe4..3eea09f3 100644 --- a/src/Measurements.jl +++ b/src/Measurements.jl @@ -188,7 +188,7 @@ Supported keyword arguments: """ function getMeasurements(f::MPIFile, neglectBGFrames=true; frames=neglectBGFrames ? (1:acqNumFGFrames(f)) : (1:acqNumFrames(f)), - bgCorrection=false, interpolateBG=false, tfCorrection=rxHasTransferFunction(f), + bgCorrection=false, bgFrames = measBGFrameIdx(f), interpolateBG=false, tfCorrection=rxHasTransferFunction(f), sortFrames=false, numAverages=1, numPeriodGrouping=1, kargs...) if neglectBGFrames @@ -197,7 +197,7 @@ function getMeasurements(f::MPIFile, neglectBGFrames=true; data = getAveragedMeasurements(f; frames=idx[frames], numAverages=numAverages, kargs...) - idxBG = measBGFrameIdx(f) + idxBG = bgFrames hasBGFrames = length(idxBG) > 0 if bgCorrection && !hasBGFrames @warn "Background correction was selected but there are no background frames in the file." From 4ae397387157b4d72f04120b2f08b866a7542ffd Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 24 May 2024 11:25:54 +0200 Subject: [PATCH 12/25] Adapt bgFrame behaviour to be similar to frames --- src/Measurements.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Measurements.jl b/src/Measurements.jl index 3eea09f3..adc975eb 100644 --- a/src/Measurements.jl +++ b/src/Measurements.jl @@ -188,7 +188,7 @@ Supported keyword arguments: """ function getMeasurements(f::MPIFile, neglectBGFrames=true; frames=neglectBGFrames ? (1:acqNumFGFrames(f)) : (1:acqNumFrames(f)), - bgCorrection=false, bgFrames = measBGFrameIdx(f), interpolateBG=false, tfCorrection=rxHasTransferFunction(f), + bgCorrection=false, bgFrames = 1:length(measBGFrameIdx(f)), interpolateBG=false, tfCorrection=rxHasTransferFunction(f), sortFrames=false, numAverages=1, numPeriodGrouping=1, kargs...) if neglectBGFrames @@ -197,7 +197,7 @@ function getMeasurements(f::MPIFile, neglectBGFrames=true; data = getAveragedMeasurements(f; frames=idx[frames], numAverages=numAverages, kargs...) - idxBG = bgFrames + idxBG = measBGFrameIdx[bgFrames] hasBGFrames = length(idxBG) > 0 if bgCorrection && !hasBGFrames @warn "Background correction was selected but there are no background frames in the file." From 55dca32846fb5bbcdd9db047d33ce9fd7670e076 Mon Sep 17 00:00:00 2001 From: nHackel Date: Fri, 24 May 2024 11:41:51 +0000 Subject: [PATCH 13/25] Fix bug in bgFrame selection --- src/Measurements.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Measurements.jl b/src/Measurements.jl index adc975eb..19fcc84e 100644 --- a/src/Measurements.jl +++ b/src/Measurements.jl @@ -197,7 +197,7 @@ function getMeasurements(f::MPIFile, neglectBGFrames=true; data = getAveragedMeasurements(f; frames=idx[frames], numAverages=numAverages, kargs...) - idxBG = measBGFrameIdx[bgFrames] + idxBG = measBGFrameIdx(f)[bgFrames] hasBGFrames = length(idxBG) > 0 if bgCorrection && !hasBGFrames @warn "Background correction was selected but there are no background frames in the file." From 3139d697ff3f70e973f47e9547174d357e3b4f0f Mon Sep 17 00:00:00 2001 From: Jonas Schumacher Date: Mon, 27 May 2024 15:14:40 +0200 Subject: [PATCH 14/25] Bump patch release (for bg frame selection) --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 54d83a3a..64021d0d 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MPIFiles" uuid = "371237a9-e6c1-5201-9adb-3d8cfa78fa9f" authors = ["Tobias Knopp "] -version = "0.15.3" +version = "0.15.4" [deps] AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" From bfd1f0173a709f923b9c7f6ebc09ea4837de8cec Mon Sep 17 00:00:00 2001 From: Jonas Schumacher Date: Wed, 29 May 2024 11:01:12 +0200 Subject: [PATCH 15/25] Fix Codecov --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 838d24a0..dfff1bf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,8 @@ jobs: - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: file: lcov.info docs: From 9dcba9af7308b583a03734eb40f006163bcc3834 Mon Sep 17 00:00:00 2001 From: Jonas Schumacher Date: Thu, 30 May 2024 13:10:15 +0200 Subject: [PATCH 16/25] Fix build status (#82) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0615574e..91999336 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ MPIFiles.jl is a Julia package for loading and storing magnetic particle imaging [![](https://img.shields.io/badge/docs-latest-blue.svg)](https://magneticparticleimaging.github.io/MPIFiles.jl/dev) [![DOI](http://joss.theoj.org/papers/10.21105/joss.01331/status.svg)](https://doi.org/10.21105/joss.01331) -[![Build status](https://github.com/MagneticParticleImaging/MPIFiles.jl/workflows/CI/badge.svg)](https://github.com/MagneticParticleImaging/MPIFiles.jl/actions) +[![Build status](https://github.com/MagneticParticleImaging/MPIMeasurements.jl/actions/workflows/ci.yml/badge.svg)](https://github.com/MagneticParticleImaging/MPIMeasurements.jl/actions/workflows/ci.yml) [![codecov.io](http://codecov.io/github/MagneticParticleImaging/MPIFiles.jl/coverage.svg?branch=master)](http://codecov.io/github/MagneticParticleImaging/MPIFiles.jl?branch=master) [![License](https://img.shields.io/github/license/MagneticParticleImaging/MPIFiles.jl?color=green&style=flat)](https://github.com/MagneticParticleImaging/MPIFiles.jl/blob/master/LICENSE) -[![Aqua QA](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl) \ No newline at end of file +[![Aqua QA](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl) From 926e746a286fcb2cd0ffaa8e0b8552cf7009ecc6 Mon Sep 17 00:00:00 2001 From: Niklas Hackelberg Date: Fri, 31 May 2024 16:53:38 +0200 Subject: [PATCH 17/25] Update Project.toml --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 64021d0d..3c8cd589 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MPIFiles" uuid = "371237a9-e6c1-5201-9adb-3d8cfa78fa9f" authors = ["Tobias Knopp "] -version = "0.15.4" +version = "0.16.0" [deps] AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" From eaebf0b9a5eb3ef0543d9c6bb846fe91e5fb7682 Mon Sep 17 00:00:00 2001 From: nHackel Date: Wed, 5 Jun 2024 10:46:08 +0200 Subject: [PATCH 18/25] Init breakage CI --- .github/workflows/breakage.yml | 132 +++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 .github/workflows/breakage.yml diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml new file mode 100644 index 00000000..5c9f4a2b --- /dev/null +++ b/.github/workflows/breakage.yml @@ -0,0 +1,132 @@ +# Based on: https://github.com/JuliaSmoothOptimizers/LinearOperators.jl/blob/main/.github/workflows/Breakage.yml +name: Breakage + +# read-only repo token +# no access to secrets +on: + pull_request: + branches: + - master + + jobs: + break: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + pkg: [ + "MagneticParticleImaging/MPIReco.jl", + ] + pkgversion: [latest, stable] + + steps: + - uses: actions/checkout@v2 + + # Install Julia + - uses: julia-actions/setup-julia@v2 + with: + version: 1 + arch: x64 + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + + # Breakage test + - name: 'Breakage of ${{ matrix.pkg }}, ${{ matrix.pkgversion }} version' + env: + URL: ${{ matrix.pkg }} + VERSION: ${{ matrix.pkgversion }} + run: | + set -v + mkdir -p ./pr + echo "${{ github.event.number }}" > ./pr/NR + git clone https://github.com/$URL + export PKG=$(echo $URL | cut -f2 -d/) + cd $PKG + if [ $VERSION == "stable" ]; then + TAG=$(git tag -l "v*" --sort=-creatordate | head -n1) + if [ -z "$TAG" ]; then + TAG="no_tag" + else + git checkout $TAG + fi + else + TAG=$VERSION + fi + export TAG + julia -e 'using Pkg; + PKG, TAG, VERSION = ENV["PKG"], ENV["TAG"], ENV["VERSION"] + joburl = joinpath(ENV["GITHUB_SERVER_URL"], ENV["GITHUB_REPOSITORY"], "actions/runs", ENV["GITHUB_RUN_ID"]) + open("../pr/$PKG-$VERSION", "w") do io + try + TAG == "no_tag" && error("Not tag for $VERSION") + pkg"activate ."; + pkg"instantiate"; + pkg"dev ../"; + if TAG == "latest" + global TAG = chomp(read(`git rev-parse --short HEAD`, String)) + end + pkg"build"; + pkg"test"; + + print(io, "[![](https://img.shields.io/badge/$TAG-Pass-green)]($joburl)"); + catch e + @error e; + print(io, "[![](https://img.shields.io/badge/$TAG-Fail-red)]($joburl)"); + end; + end' + + - uses: actions/upload-artifact@v2 + with: + name: pr + path: pr/ + + upload: + needs: break + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: pr + path: pr/ + + - run: ls + - run: | + cd pr + echo "| Package name | latest | stable |" > MSG + echo "|--|--|--|" >> MSG + count=0 + for file in * + do + [ "$file" == "NR" ] && continue + [ "$file" == "MSG" ] && continue + if [ $count == "0" ]; then + name=$(echo $file | cut -f1 -d-) + echo -n "| $name | " + else + echo -n "| " + fi + cat $file + if [ $count == "0" ]; then + echo -n " " + count=1 + else + echo " |" + count=0 + fi + done >> MSG + + - uses: actions/upload-artifact@v2 + with: + name: pr + path: pr/ \ No newline at end of file From 3cd3aa7ec7174b6bc3dfac593db1ae1b40c93154 Mon Sep 17 00:00:00 2001 From: nHackel Date: Wed, 5 Jun 2024 10:50:04 +0200 Subject: [PATCH 19/25] Format breakage.yml --- .github/workflows/breakage.yml | 244 ++++++++++++++++----------------- 1 file changed, 120 insertions(+), 124 deletions(-) diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml index 5c9f4a2b..cbd820a3 100644 --- a/.github/workflows/breakage.yml +++ b/.github/workflows/breakage.yml @@ -1,132 +1,128 @@ -# Based on: https://github.com/JuliaSmoothOptimizers/LinearOperators.jl/blob/main/.github/workflows/Breakage.yml name: Breakage - -# read-only repo token -# no access to secrets on: pull_request: branches: - master - jobs: - break: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - pkg: [ - "MagneticParticleImaging/MPIReco.jl", - ] - pkgversion: [latest, stable] - - steps: - - uses: actions/checkout@v2 - - # Install Julia - - uses: julia-actions/setup-julia@v2 - with: - version: 1 - arch: x64 - - uses: actions/cache@v1 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- - - uses: julia-actions/julia-buildpkg@v1 - - # Breakage test - - name: 'Breakage of ${{ matrix.pkg }}, ${{ matrix.pkgversion }} version' - env: - URL: ${{ matrix.pkg }} - VERSION: ${{ matrix.pkgversion }} - run: | - set -v - mkdir -p ./pr - echo "${{ github.event.number }}" > ./pr/NR - git clone https://github.com/$URL - export PKG=$(echo $URL | cut -f2 -d/) - cd $PKG - if [ $VERSION == "stable" ]; then - TAG=$(git tag -l "v*" --sort=-creatordate | head -n1) - if [ -z "$TAG" ]; then - TAG="no_tag" - else - git checkout $TAG - fi +jobs: + break: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + pkg: [ + "MagneticParticleImaging/MPIReco.jl", + ] + pkgversion: [latest, stable] + + steps: + - uses: actions/checkout@v2 + + # Install Julia + - uses: julia-actions/setup-julia@v2 + with: + version: 1 + arch: x64 + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + + # Breakage test + - name: 'Breakage of ${{ matrix.pkg }}, ${{ matrix.pkgversion }} version' + env: + URL: ${{ matrix.pkg }} + VERSION: ${{ matrix.pkgversion }} + run: | + set -v + mkdir -p ./pr + echo "${{ github.event.number }}" > ./pr/NR + git clone https://github.com/$URL + export PKG=$(echo $URL | cut -f2 -d/) + cd $PKG + if [ $VERSION == "stable" ]; then + TAG=$(git tag -l "v*" --sort=-creatordate | head -n1) + if [ -z "$TAG" ]; then + TAG="no_tag" else - TAG=$VERSION + git checkout $TAG fi - export TAG - julia -e 'using Pkg; - PKG, TAG, VERSION = ENV["PKG"], ENV["TAG"], ENV["VERSION"] - joburl = joinpath(ENV["GITHUB_SERVER_URL"], ENV["GITHUB_REPOSITORY"], "actions/runs", ENV["GITHUB_RUN_ID"]) - open("../pr/$PKG-$VERSION", "w") do io - try - TAG == "no_tag" && error("Not tag for $VERSION") - pkg"activate ."; - pkg"instantiate"; - pkg"dev ../"; - if TAG == "latest" - global TAG = chomp(read(`git rev-parse --short HEAD`, String)) - end - pkg"build"; - pkg"test"; - - print(io, "[![](https://img.shields.io/badge/$TAG-Pass-green)]($joburl)"); - catch e - @error e; - print(io, "[![](https://img.shields.io/badge/$TAG-Fail-red)]($joburl)"); - end; - end' - - - uses: actions/upload-artifact@v2 - with: - name: pr - path: pr/ - - upload: - needs: break - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v2 - with: - name: pr - path: pr/ - - - run: ls - - run: | - cd pr - echo "| Package name | latest | stable |" > MSG - echo "|--|--|--|" >> MSG - count=0 - for file in * - do - [ "$file" == "NR" ] && continue - [ "$file" == "MSG" ] && continue - if [ $count == "0" ]; then - name=$(echo $file | cut -f1 -d-) - echo -n "| $name | " - else - echo -n "| " - fi - cat $file - if [ $count == "0" ]; then - echo -n " " - count=1 - else - echo " |" - count=0 - fi - done >> MSG - - - uses: actions/upload-artifact@v2 - with: - name: pr - path: pr/ \ No newline at end of file + else + TAG=$VERSION + fi + export TAG + julia -e 'using Pkg; + PKG, TAG, VERSION = ENV["PKG"], ENV["TAG"], ENV["VERSION"] + joburl = joinpath(ENV["GITHUB_SERVER_URL"], ENV["GITHUB_REPOSITORY"], "actions/runs", ENV["GITHUB_RUN_ID"]) + open("../pr/$PKG-$VERSION", "w") do io + try + TAG == "no_tag" && error("Not tag for $VERSION") + pkg"activate ."; + pkg"instantiate"; + pkg"dev ../"; + if TAG == "latest" + global TAG = chomp(read(`git rev-parse --short HEAD`, String)) + end + pkg"build"; + pkg"test"; + + print(io, "[![](https://img.shields.io/badge/$TAG-Pass-green)]($joburl)"); + catch e + @error e; + print(io, "[![](https://img.shields.io/badge/$TAG-Fail-red)]($joburl)"); + end; + end' + + - uses: actions/upload-artifact@v2 + with: + name: pr + path: pr/ + + upload: + needs: break + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/download-artifact@v2 + with: + name: pr + path: pr/ + + - run: ls + - run: | + cd pr + echo "| Package name | latest | stable |" > MSG + echo "|--|--|--|" >> MSG + count=0 + for file in * + do + [ "$file" == "NR" ] && continue + [ "$file" == "MSG" ] && continue + if [ $count == "0" ]; then + name=$(echo $file | cut -f1 -d-) + echo -n "| $name | " + else + echo -n "| " + fi + cat $file + if [ $count == "0" ]; then + echo -n " " + count=1 + else + echo " |" + count=0 + fi + done >> MSG + + - uses: actions/upload-artifact@v2 + with: + name: pr + path: pr/ \ No newline at end of file From 030d99c38771f5a685f9614053d65081b760c4d0 Mon Sep 17 00:00:00 2001 From: nHackel Date: Wed, 5 Jun 2024 11:16:12 +0200 Subject: [PATCH 20/25] Add reference to workflow from LinearOperators.jl --- .github/workflows/breakage.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml index cbd820a3..330236bb 100644 --- a/.github/workflows/breakage.yml +++ b/.github/workflows/breakage.yml @@ -1,4 +1,5 @@ -name: Breakage +name: Breakage +# Based on: https://github.com/JuliaSmoothOptimizers/LinearOperators.jl/blob/main/.github/workflows/Breakage.yml on: pull_request: branches: From 096ba0fc41baeefb6801597f113d8f21ba814237 Mon Sep 17 00:00:00 2001 From: nHackel Date: Wed, 5 Jun 2024 11:25:09 +0200 Subject: [PATCH 21/25] Remove upload job since we dont comment --- .github/workflows/breakage.yml | 42 ---------------------------------- 1 file changed, 42 deletions(-) diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml index 330236bb..847e6a7f 100644 --- a/.github/workflows/breakage.yml +++ b/.github/workflows/breakage.yml @@ -81,48 +81,6 @@ jobs: end; end' - - uses: actions/upload-artifact@v2 - with: - name: pr - path: pr/ - - upload: - needs: break - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions/download-artifact@v2 - with: - name: pr - path: pr/ - - - run: ls - - run: | - cd pr - echo "| Package name | latest | stable |" > MSG - echo "|--|--|--|" >> MSG - count=0 - for file in * - do - [ "$file" == "NR" ] && continue - [ "$file" == "MSG" ] && continue - if [ $count == "0" ]; then - name=$(echo $file | cut -f1 -d-) - echo -n "| $name | " - else - echo -n "| " - fi - cat $file - if [ $count == "0" ]; then - echo -n " " - count=1 - else - echo " |" - count=0 - fi - done >> MSG - - uses: actions/upload-artifact@v2 with: name: pr From e7a0d3d5a7fe605b81449b5bec0a3084c7c64222 Mon Sep 17 00:00:00 2001 From: nHackel Date: Wed, 5 Jun 2024 12:15:09 +0200 Subject: [PATCH 22/25] Let breaking CI fail if package tests fail --- .github/workflows/breakage.yml | 39 ++++++++++++---------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml index 847e6a7f..7efd1957 100644 --- a/.github/workflows/breakage.yml +++ b/.github/workflows/breakage.yml @@ -3,8 +3,11 @@ name: Breakage on: pull_request: branches: - - master - + - master + push: + branches: + - master + tags: '*' jobs: break: runs-on: ubuntu-latest @@ -62,26 +65,12 @@ jobs: julia -e 'using Pkg; PKG, TAG, VERSION = ENV["PKG"], ENV["TAG"], ENV["VERSION"] joburl = joinpath(ENV["GITHUB_SERVER_URL"], ENV["GITHUB_REPOSITORY"], "actions/runs", ENV["GITHUB_RUN_ID"]) - open("../pr/$PKG-$VERSION", "w") do io - try - TAG == "no_tag" && error("Not tag for $VERSION") - pkg"activate ."; - pkg"instantiate"; - pkg"dev ../"; - if TAG == "latest" - global TAG = chomp(read(`git rev-parse --short HEAD`, String)) - end - pkg"build"; - pkg"test"; - - print(io, "[![](https://img.shields.io/badge/$TAG-Pass-green)]($joburl)"); - catch e - @error e; - print(io, "[![](https://img.shields.io/badge/$TAG-Fail-red)]($joburl)"); - end; - end' - - - uses: actions/upload-artifact@v2 - with: - name: pr - path: pr/ \ No newline at end of file + TAG == "no_tag" && error("Not tag for $VERSION") + pkg"activate ."; + pkg"instantiate"; + pkg"dev ../"; + if TAG == "latest" + global TAG = chomp(read(`git rev-parse --short HEAD`, String)) + end + pkg"build"; + pkg"test";' \ No newline at end of file From 8f51b891723bd08dc8ccd9521554d40f2f2f1b9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:58:00 +0200 Subject: [PATCH 23/25] Bump actions/checkout from 2 to 4 (#86) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/breakage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml index 7efd1957..16ecec5d 100644 --- a/.github/workflows/breakage.yml +++ b/.github/workflows/breakage.yml @@ -20,7 +20,7 @@ jobs: pkgversion: [latest, stable] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Install Julia - uses: julia-actions/setup-julia@v2 From 9857bc2611a6592c2394d7bc1734a4816add876e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:58:14 +0200 Subject: [PATCH 24/25] Bump actions/cache from 1 to 4 (#87) Bumps [actions/cache](https://github.com/actions/cache) from 1 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v1...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/breakage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/breakage.yml b/.github/workflows/breakage.yml index 16ecec5d..495e05b4 100644 --- a/.github/workflows/breakage.yml +++ b/.github/workflows/breakage.yml @@ -27,7 +27,7 @@ jobs: with: version: 1 arch: x64 - - uses: actions/cache@v1 + - uses: actions/cache@v4 env: cache-name: cache-artifacts with: From dca896e8b3fb093d0d3115f30122472519f1ae75 Mon Sep 17 00:00:00 2001 From: Jonas Schumacher Date: Thu, 13 Jun 2024 11:12:55 +0200 Subject: [PATCH 25/25] Make CompatHelper.yml workflow manually triggerable --- .github/workflows/CompatHelper.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml index dadc755a..0305291d 100644 --- a/.github/workflows/CompatHelper.yml +++ b/.github/workflows/CompatHelper.yml @@ -2,6 +2,7 @@ name: CompatHelper on: schedule: - cron: 0 0 * * * + workflow_dispatch: jobs: build: runs-on: ubuntu-latest