From c9dbc25c417b1255b45144699b34022610ea6b21 Mon Sep 17 00:00:00 2001 From: ffreyer Date: Mon, 21 Oct 2024 21:16:16 +0200 Subject: [PATCH 1/5] add "update_from_previous_version" --- ReferenceUpdater/src/local_server.jl | 77 ++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/ReferenceUpdater/src/local_server.jl b/ReferenceUpdater/src/local_server.jl index 4496911129d..2e10960f3ed 100644 --- a/ReferenceUpdater/src/local_server.jl +++ b/ReferenceUpdater/src/local_server.jl @@ -98,7 +98,7 @@ function serve_update_page_from_dir(folder) end end -function serve_update_page(; commit = nothing, pr = nothing) +function download_artifacts(; commit = nothing, pr = nothing) authget(url) = HTTP.get(url, Dict("Authorization" => "token $(github_token())")) commit !== nothing && pr !== nothing && error("Keyword arguments `commit` and `pr` can't be set at once.") @@ -146,7 +146,7 @@ function serve_update_page(; commit = nothing, pr = nothing) @warn("Found multiple checkruns for \"Merge artifacts\". Using latest with timestamp: $datetime") check = checkruns[idx] else - check = only(checkruns) + check = only(checkruns) end job = JSON3.read(authget("https://api.github.com/repos/MakieOrg/Makie.jl/actions/jobs/$(check["id"])").body) @@ -171,11 +171,10 @@ function serve_update_page(; commit = nothing, pr = nothing) @info "$download_url cached at $tmpdir" end - @info "Serving update page from folder $tmpdir." - serve_update_page_from_dir(tmpdir) - return + return tmpdir end end + error(""" No \"ReferenceImages\" artifact found for commit $headsha and job id $(check["id"]). This could be because the job's workflow run ($(job["run_url"])) has not completed, yet. @@ -183,6 +182,74 @@ function serve_update_page(; commit = nothing, pr = nothing) """) end +function serve_update_page(; commit = nothing, pr = nothing) + tmpdir = download_artifacts(commit = commit, pr = pr) + @info "Serving update page from folder $tmpdir." + serve_update_page_from_dir(tmpdir) + return +end + +function update_from_previous_version(; + source_version::String, target_version::String = last_major_version(), + commit = nothing, pr = nothing, score_threshold = 0.03) + + tmpdir = download_artifacts(commit = commit, pr = pr) + + @info "Updating reference images in $target_version from $source_version" + @info "By score threshold:" + # missing & changed + folder = realpath(tmpdir) + changed_or_missing = open(joinpath(folder, "scores.tsv"), "r") do file + names = String[] + for line in eachline(file) + score_str, filename = split(line, '\t') + if parse(Float64, score_str) >= score_threshold + push!(names, filename) + @info " $filename" + end + end + return names + end + + @info "Missing:" + open(joinpath(folder, "new_files.txt"), "r") do file + for line in eachline(file) + if !isempty(line) + push!(changed_or_missing, line) + @info " $line" + end + end + end + + # Merge new and old + @info "Downloading new reference image set (target)" + new_version = download_refimages(target_version) + @info "Target path: $new_version" + @info "Downloading old reference image set (source)" + old_version = download_refimages(source_version) + + + @info "Replacing target with source files..." + for image in changed_or_missing + @info "Overwriting or adding $image" + cp(joinpath(old_version, image), joinpath(new_version, image), force = true) + end + + rm(old_version, recursive = true) + + @info "Uploading updated reference images under tag \"$target_version\"" + try + upload_reference_images(new_version, target_version) + @info "Upload successful. You can ctrl+c out now." + HTTP.Response(200, "Upload successful") + catch e + showerror(stdout, e, catch_backtrace()) + HTTP.Response(404) + end + + return +end + function unzip(file, exdir = "") fileFullPath = isabspath(file) ? file : joinpath(pwd(),file) basePath = dirname(fileFullPath) From a60b7ac370f6d50ccc04f3481b2424ef1f4a8507 Mon Sep 17 00:00:00 2001 From: ffreyer Date: Tue, 22 Oct 2024 17:46:24 +0200 Subject: [PATCH 2/5] don't default target_version seems unsafe to me --- ReferenceUpdater/src/local_server.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReferenceUpdater/src/local_server.jl b/ReferenceUpdater/src/local_server.jl index 2e10960f3ed..e4804e4cddf 100644 --- a/ReferenceUpdater/src/local_server.jl +++ b/ReferenceUpdater/src/local_server.jl @@ -190,7 +190,7 @@ function serve_update_page(; commit = nothing, pr = nothing) end function update_from_previous_version(; - source_version::String, target_version::String = last_major_version(), + source_version::String, target_version::String, commit = nothing, pr = nothing, score_threshold = 0.03) tmpdir = download_artifacts(commit = commit, pr = pr) From 91769ba1ec67d8b2dd6280560b15ffadb61b902a Mon Sep 17 00:00:00 2001 From: ffreyer Date: Tue, 22 Oct 2024 17:57:36 +0200 Subject: [PATCH 3/5] split up function for more manual usage --- ReferenceUpdater/src/local_server.jl | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ReferenceUpdater/src/local_server.jl b/ReferenceUpdater/src/local_server.jl index e4804e4cddf..e141f88a6a2 100644 --- a/ReferenceUpdater/src/local_server.jl +++ b/ReferenceUpdater/src/local_server.jl @@ -221,6 +221,17 @@ function update_from_previous_version(; end end + update_from_previous_version(changed_or_missing; + source_version = source_version, target_version = target_version) + + return +end + + +function update_from_previous_version( + changed_or_missing::Vector{String}; + source_version::String, target_version::String) + # Merge new and old @info "Downloading new reference image set (target)" new_version = download_refimages(target_version) @@ -232,7 +243,10 @@ function update_from_previous_version(; @info "Replacing target with source files..." for image in changed_or_missing @info "Overwriting or adding $image" - cp(joinpath(old_version, image), joinpath(new_version, image), force = true) + src = joinpath(old_version, image) + isfile(src) || error("Failed to find file $image in $source_version.") + trg = joinpath(new_version, image) + cp(src, trg, force = true) end rm(old_version, recursive = true) From 81da74a429750b1efc74fad29705fe59aac8d86a Mon Sep 17 00:00:00 2001 From: ffreyer Date: Tue, 22 Oct 2024 18:08:33 +0200 Subject: [PATCH 4/5] add some docs --- ReferenceUpdater/src/local_server.jl | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/ReferenceUpdater/src/local_server.jl b/ReferenceUpdater/src/local_server.jl index e141f88a6a2..cd6a5070290 100644 --- a/ReferenceUpdater/src/local_server.jl +++ b/ReferenceUpdater/src/local_server.jl @@ -182,6 +182,12 @@ function download_artifacts(; commit = nothing, pr = nothing) """) end +""" + serve_update_page(; commit, pr) + +Create a ReferenceUpdater page which shows the test differences from a CI run +of a given commit hash or pr. +""" function serve_update_page(; commit = nothing, pr = nothing) tmpdir = download_artifacts(commit = commit, pr = pr) @info "Serving update page from folder $tmpdir." @@ -189,6 +195,19 @@ function serve_update_page(; commit = nothing, pr = nothing) return end +""" + update_from_previous_version(; source_version, target_version, commit, pr[, score_threshold = 0.03]) + +Replaces every test that is failing (based on score_threshold) or missing in the +given commit or pr with the respective test from the given `source_version` and +uploads the result to `target_version`. + +This is useful for breaking pull requests, where new or changed tests from master +(previous version) should be added to the new, breaking verison. E.g.: +``` +update_from_previous_version(source_version = "v0.21.0", target_version = "v0.22.0", pr = 4477) +``` +""" function update_from_previous_version(; source_version::String, target_version::String, commit = nothing, pr = nothing, score_threshold = 0.03) @@ -227,7 +246,17 @@ function update_from_previous_version(; return end +""" + update_from_previous_version(changed_or_missing::Vector{String}; source_version, target_version) + +Replaces every test in `changed_or_missing` with the respective test from the +given `source_version` and uploads the result to `target_version`. This is the +more manual version of the other method. +Tests should be given as `"backend/name-of-test.png"` in `changed_or_missing`. +Source and target version are given as `"v0.00.0"` matching the respective +github release version. +""" function update_from_previous_version( changed_or_missing::Vector{String}; source_version::String, target_version::String) From c7dd50ef09caa93051d2d3ccfdc1559a9874c2ea Mon Sep 17 00:00:00 2001 From: ffreyer Date: Tue, 22 Oct 2024 18:09:16 +0200 Subject: [PATCH 5/5] remove ctrl-c mention --- ReferenceUpdater/src/local_server.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReferenceUpdater/src/local_server.jl b/ReferenceUpdater/src/local_server.jl index cd6a5070290..b90eed14bda 100644 --- a/ReferenceUpdater/src/local_server.jl +++ b/ReferenceUpdater/src/local_server.jl @@ -283,7 +283,7 @@ function update_from_previous_version( @info "Uploading updated reference images under tag \"$target_version\"" try upload_reference_images(new_version, target_version) - @info "Upload successful. You can ctrl+c out now." + @info "Upload successful." HTTP.Response(200, "Upload successful") catch e showerror(stdout, e, catch_backtrace())