From a30270e8f61ffaadbe927924e17d5f4c390c3ef0 Mon Sep 17 00:00:00 2001 From: Nehal Patel Date: Thu, 10 Nov 2022 06:59:45 -0500 Subject: [PATCH] Add documentation. --- README.md | 50 ++++++++++++++++++++++++++++++++++++++++- src/Downloads.jl | 58 +++++++++++++++++++++++++++++++++++++++++++----- test/runtests.jl | 43 ++++++++++++++++++----------------- 3 files changed, 123 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index b1cbb3b..4504e73 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,13 @@ Julia 1.3 through 1.5 as well. ## API -The public API of `Downloads` consists of three functions and three types: +The public API of `Downloads` consists of the following functions and types: - `download` — download a file from a URL, erroring if it can't be downloaded - `request` — request a URL, returning a `Response` object indicating success - `default_downloader!` - set the default `Downloader` object +- `pushhook!` — add a hook which allows for customizing downloading parameters +- `deletehook!` — remove a previously added parameter customization hook - `Response` — a type capturing the status and other metadata about a request - `RequestError` — an error type thrown by `download` and `request` on error - `Downloader` — an object encapsulating shared resources for downloading @@ -141,6 +143,52 @@ with getting a response at all, then a `RequestError` is thrown or returned. Set the default `Downloader`. If no argument is provided, resets the default downloader so that a fresh one is created the next time the default downloader is needed. +### pushhook! +```jl + pushhook!(hook) -> key +``` +- `hook :: Function` +- `key :: HookKey` + +Add a hook to customize download parameters for all downloads. + +The signature `hook` should be `(easy::Easy, info::Dict) -> Nothing`. +Multiple hooks can be added with repeated calls to `pushhook!`. Hooks are +applied in the order they were added. + +The returned `key` maybe used to remove a previously added `hook` cf. `deletehook!` + +Examples: +```jl +# define hook +hook = (easy, info) -> begin + # allow for long pauses during downloads + # (perhaps for malware scanning) + setopt(easy, Downloads.Curl.CURLOPT_LOW_SPEED_LIMIT, 1 #= bytes =#) + setopt(easy, Downloads.Curl.CURLOPT_LOW_SPEED_TIME , 200 #= seconds =#) + # other possibilities + # set ca_roots + # disable certificate verification + # block or rewrite URLs +end + +# add hook +key = pushhook!(hook) + +# would fail with default download parameters... +download("https://httpbingo.julialang.org/delay/40", "test.txt") + +# cleanup +deletehook!(key) +``` + +### deletehook! +```jl + deletehook!(key) +``` +- `key :: HookKey` + +Remove a hook previously added with `pushhook!. ### Response ```jl diff --git a/src/Downloads.jl b/src/Downloads.jl index 51a41ab..9beebf5 100644 --- a/src/Downloads.jl +++ b/src/Downloads.jl @@ -7,6 +7,8 @@ More generally, the module exports functions and types that provide lower-level for file downloading: - [`download`](@ref) — download a file from a URL, erroring if it can't be downloaded - [`request`](@ref) — request a URL, returning a `Response` object indicating success +- [`pushhook!`](@ref) — add a hook which allows for customizing downloading parameters +- [`deletehook!`](@ref) — remove a previously added parameter customization hook - [`Response`](@ref) — a type capturing the status and other metadata about a request - [`RequestError`](@ref) — an error type thrown by `download` and `request` on error - [`Downloader`](@ref) — an object encapsulating shared resources for downloading @@ -77,23 +79,67 @@ const EASY_HOOK = Ref{Union{Function, Nothing}}(nothing) ## Allow for a set of global hooks that can customize each download (via setting parameters on the ## `Easy` object associated with a request const HookKey = Int -current_key = 0 +CURRENT_KEY = 0 GlobalHookEntry = Tuple{HookKey, Function} const GLOBAL_HOOK_LOCK = ReentrantLock() const GLOBAL_HOOKS = Array{GlobalHookEntry,1}(undef, 0) ## Add hook +""" + pushhook!(hook) -> key + + hook :: Function + key :: HookKey +Add a hook to customize download parameters for all downloads. + +The signature `hook` should be `(easy::Easy, info::Dict) -> Nothing``. +Mulitple hooks can be added with repeated calls to `pushhook!`. Hooks are +applied in the order they were added. + +The returned `key` maybe used to remove a previously added `hook` cf. [deletehook!](@ref) + +Examples: +```jl +# define hook +hook = (easy, info) -> begin + # allow for long pauses during downloads (perhaps for malware scanning) + setopt(easy, Downloads.Curl.CURLOPT_LOW_SPEED_LIMIT, 1 #= bytes =#) + setopt(easy, Downloads.Curl.CURLOPT_LOW_SPEED_TIME , 200 #= seconds =#) + + # other possibilities + # set ca_roots + # disable certificate verification + # block or rewrite URLs + +end + +# add hook +key = pushhook!(hook) + +# would fail with default download parameters... +download("https://httpbingo.julialang.org/delay/40", "test.txt") + +# cleanup +deletehook!(key) +``` +""" function pushhook!(hook::Function) :: HookKey - global current_key + global CURRENT_KEY key = -1 lock(GLOBAL_HOOK_LOCK) do - key = current_key + key = CURRENT_KEY push!(GLOBAL_HOOKS, (key, hook)) - current_key += 1 + CURRENT_KEY += 1 end - return key + key end +""" + deletehook!(key) + key :: HookKey + +Remove a hook previously added with [`pushhook!`](@ref). +""" function deletehook!(key::HookKey) keep = x -> x[1] != key lock(GLOBAL_HOOK_LOCK) do @@ -101,6 +147,7 @@ function deletehook!(key::HookKey) warn("Downloads.jl: Hook key $(key) not found in global hooks") filter!(keep, GLOBAL_HOOKS) end + nothing end function apply_global_hooks(easy::Easy, info::NamedTuple) @@ -109,6 +156,7 @@ function apply_global_hooks(easy::Easy, info::NamedTuple) h(easy, info) end end + nothing end diff --git a/test/runtests.jl b/test/runtests.jl index 9bfbc3a..919f48d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,27 +1,6 @@ include("setup.jl") @testset "Downloads.jl" begin - @testset "Global easy hooks" begin - trip_wire = 0 - original_hook_count = length(Downloads.GLOBAL_HOOKS) - url = "$server/get" - hook = (easy, info) -> trip_wire += 1 - key1 = pushhook!(hook) - _ = download_body(url) - @test trip_wire == 1 - key2 = pushhook!(hook) - _ = download_body(url) - @test trip_wire == 3 - deletehook!(key1) - _ = download_body(url) - @test trip_wire == 4 - deletehook!(key2) - _ = download_body(url) - @test trip_wire == 4 - - @test length(Downloads.GLOBAL_HOOKS) == original_hook_count - end - #= @testset "libcurl configuration" begin julia = "$(VERSION.major).$(VERSION.minor)" @test Curl.USER_AGENT == "curl/$(Curl.CURL_VERSION) julia/$julia" @@ -607,7 +586,27 @@ include("setup.jl") @test Downloads.content_length(["Accept"=>"*/*",]) === nothing @test Downloads.content_length(["Accept"=>"*/*", "Content-Length"=>"100"]) == 100 end - =# + + @testset "Global easy hooks" begin + trip_wire = 0 + original_hook_count = length(Downloads.GLOBAL_HOOKS) + url = "$server/get" + hook = (easy, info) -> trip_wire += 1 + key1 = pushhook!(hook) + _ = download_body(url) + @test trip_wire == 1 + key2 = pushhook!(hook) + _ = download_body(url) + @test trip_wire == 3 + deletehook!(key1) + _ = download_body(url) + @test trip_wire == 4 + deletehook!(key2) + _ = download_body(url) + @test trip_wire == 4 + + @test length(Downloads.GLOBAL_HOOKS) == original_hook_count + end end Downloads.DOWNLOADER[] = nothing