diff --git a/NEWS.md b/NEWS.md index 2da8a7bad8b23b..fc3a2304ea72ae 100644 --- a/NEWS.md +++ b/NEWS.md @@ -73,6 +73,7 @@ Multi-threading changes ----------------------- * `Threads.@threads` now supports the `:greedy` scheduler, intended for non-uniform workloads ([#52096]). +* A new exported struct `Lockable{T, L<:AbstractLock}` makes it easy to bundle a resource and its lock together ([#52898]). Build system changes -------------------- @@ -96,6 +97,7 @@ New library features * `invmod(n)` is an abbreviation for `invmod(n, typeof(n))` for native integer types ([#52180]). * `replace(string, pattern...)` now supports an optional `IO` argument to write the output to a stream rather than returning a string ([#48625]). +* New methods `allequal(f, itr)` and `allunique(f, itr)` taking a predicate function ([#47679]). * `sizehint!(s, n)` now supports an optional `shrink` argument to disable shrinking ([#51929]). * New function `Docs.hasdoc(module, symbol)` tells whether a name has a docstring ([#52139]). * New function `Docs.undocumented_names(module)` returns a module's undocumented public names ([#52413]). @@ -109,6 +111,8 @@ New library features automatically. * `@timed` now additionally returns the elapsed compilation and recompilation time ([#52889]) * `filter` can now act on a `NamedTuple` ([#50795]). +* `Iterators.cycle(iter, n)` runs over `iter` a fixed number of times, instead of forever ([#47354]) +* `zero(::AbstractArray)` now applies recursively, so `zero([[1,2],[3,4,5]])` now produces the additive identity `[[0,0],[0,0,0]]` rather than erroring ([#38064]). Standard library changes ------------------------ diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 1af8e07584d0dc..90398d98465df5 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1100,9 +1100,9 @@ function copyto_unaliased!(deststyle::IndexStyle, dest::AbstractArray, srcstyle: iterdest, itersrc = eachindex(dest), eachindex(src) if iterdest == itersrc # Shared-iterator implementation - for I in iterdest + @inbounds for I in iterdest if isassigned(src, I) - @inbounds dest[I] = src[I] + dest[I] = src[I] else _unsetindex!(dest, I) end @@ -1217,7 +1217,8 @@ function copymutable(a::AbstractArray) end copymutable(itr) = collect(itr) -zero(x::AbstractArray{T}) where {T} = fill!(similar(x, typeof(zero(T))), zero(T)) +zero(x::AbstractArray{T}) where {T<:Number} = fill!(similar(x, typeof(zero(T))), zero(T)) +zero(x::AbstractArray) = map(zero, x) ## iteration support for arrays by iterating over `eachindex` in the array ## # Allows fast iteration by default for both IndexLinear and IndexCartesian arrays @@ -1989,24 +1990,91 @@ The keyword also accepts `Val(dims)`. For multiple dimensions `dims = Val(::Tuple)` was added in Julia 1.8. # Examples + +Concatenate two arrays in different dimensions: +```jldoctest +julia> a = [1 2 3] +1×3 Matrix{Int64}: + 1 2 3 + +julia> b = [4 5 6] +1×3 Matrix{Int64}: + 4 5 6 + +julia> cat(a, b; dims=1) +2×3 Matrix{Int64}: + 1 2 3 + 4 5 6 + +julia> cat(a, b; dims=2) +1×6 Matrix{Int64}: + 1 2 3 4 5 6 + +julia> cat(a, b; dims=(1, 2)) +2×6 Matrix{Int64}: + 1 2 3 0 0 0 + 0 0 0 4 5 6 +``` + +# Extended Help + +Concatenate 3D arrays: +```jldoctest +julia> a = ones(2, 2, 3); + +julia> b = ones(2, 2, 4); + +julia> c = cat(a, b; dims=3); + +julia> size(c) == (2, 2, 7) +true +``` + +Concatenate arrays of different sizes: ```jldoctest julia> cat([1 2; 3 4], [pi, pi], fill(10, 2,3,1); dims=2) # same as hcat 2×6×1 Array{Float64, 3}: [:, :, 1] = 1.0 2.0 3.14159 10.0 10.0 10.0 3.0 4.0 3.14159 10.0 10.0 10.0 +``` +Construct a block diagonal matrix: +``` julia> cat(true, trues(2,2), trues(4)', dims=(1,2)) # block-diagonal 4×7 Matrix{Bool}: 1 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 1 0 0 0 0 0 0 0 1 1 1 1 +``` +``` julia> cat(1, [2], [3;;]; dims=Val(2)) 1×3 Matrix{Int64}: 1 2 3 ``` + +!!! note + `cat` does not join two strings, you may want to use `*`. + +```jldoctest +julia> a = "aaa"; + +julia> b = "bbb"; + +julia> cat(a, b; dims=1) +2-element Vector{String}: + "aaa" + "bbb" + +julia> cat(a, b; dims=2) +1×2 Matrix{String}: + "aaa" "bbb" + +julia> a * b +"aaabbb" +``` """ @inline cat(A...; dims) = _cat(dims, A...) # `@constprop :aggressive` allows `catdims` to be propagated as constant improving return type inference diff --git a/base/abstractarraymath.jl b/base/abstractarraymath.jl index 6ed3fff14928fc..0c339fbc54e578 100644 --- a/base/abstractarraymath.jl +++ b/base/abstractarraymath.jl @@ -119,6 +119,7 @@ julia> A """ conj!(A::AbstractArray{<:Number}) = (@inbounds broadcast!(conj, A, A); A) conj!(x::AbstractArray{<:Real}) = x +conj!(A::AbstractArray) = (foreach(conj!, A); A) """ conj(A::AbstractArray) diff --git a/base/array.jl b/base/array.jl index 600ae73d2338ed..4b2cd34b2f0db3 100644 --- a/base/array.jl +++ b/base/array.jl @@ -672,30 +672,34 @@ _array_for(::Type{T}, itr, isz) where {T} = _array_for(T, isz, _similar_shape(it collect(collection) Return an `Array` of all items in a collection or iterator. For dictionaries, returns -`Vector{Pair{KeyType, ValType}}`. If the argument is array-like or is an iterator with the -[`HasShape`](@ref IteratorSize) trait, the result will have the same shape +a `Vector` of `key=>value` [Pair](@ref Pair)s. If the argument is array-like or is an iterator +with the [`HasShape`](@ref IteratorSize) trait, the result will have the same shape and number of dimensions as the argument. -Used by comprehensions to turn a generator into an `Array`. +Used by [comprehensions](@ref man-comprehensions) to turn a [generator expression](@ref man-generators) +into an `Array`. Thus, *on generators*, the square-brackets notation may be used instead of calling `collect`, +see second example. # Examples + +Collect items from a `UnitRange{Int64}` collection: + ```jldoctest -julia> collect(1:2:13) -7-element Vector{Int64}: - 1 - 3 - 5 - 7 - 9 - 11 - 13 +julia> collect(1:3) +3-element Vector{Int64}: + 1 + 2 + 3 +``` -julia> [x^2 for x in 1:8 if isodd(x)] -4-element Vector{Int64}: - 1 - 9 - 25 - 49 +Collect items from a generator (same output as `[x^2 for x in 1:3]`): + +```jldoctest +julia> collect(x^2 for x in 1:3) +3-element Vector{Int64}: + 1 + 4 + 9 ``` """ collect(itr) = _collect(1:1 #= Array =#, itr, IteratorEltype(itr), IteratorSize(itr)) @@ -3067,7 +3071,8 @@ of [`unsafe_wrap`](@ref) utilizing `Memory` or `MemoryRef` instead of raw pointe """ function wrap end -@eval @propagate_inbounds function wrap(::Type{Array}, ref::MemoryRef{T}, dims::NTuple{N, Integer}) where {T, N} +# validity checking for _wrap calls, separate from allocation of Array so that it can be more likely to inline into the caller +function _wrap(ref::MemoryRef{T}, dims::NTuple{N, Int}) where {T, N} mem = ref.mem mem_len = length(mem) + 1 - memoryrefoffset(ref) len = Core.checked_dims(dims...) @@ -3076,18 +3081,35 @@ function wrap end mem = ccall(:jl_genericmemory_slice, Memory{T}, (Any, Ptr{Cvoid}, Int), mem, ref.ptr_or_offset, len) ref = MemoryRef(mem) end - $(Expr(:new, :(Array{T, N}), :ref, :dims)) + return ref end @noinline invalid_wrap_err(len, dims, proddims) = throw(DimensionMismatch( "Attempted to wrap a MemoryRef of length $len with an Array of size dims=$dims, which is invalid because prod(dims) = $proddims > $len, so that the array would have more elements than the underlying memory can store.")) -function wrap(::Type{Array}, m::Memory{T}, dims::NTuple{N, Integer}) where {T, N} - wrap(Array, MemoryRef(m), dims) +@eval @propagate_inbounds function wrap(::Type{Array}, m::MemoryRef{T}, dims::NTuple{N, Integer}) where {T, N} + dims = convert(Dims, dims) + ref = _wrap(m, dims) + $(Expr(:new, :(Array{T, N}), :ref, :dims)) +end + +@eval @propagate_inbounds function wrap(::Type{Array}, m::Memory{T}, dims::NTuple{N, Integer}) where {T, N} + dims = convert(Dims, dims) + ref = _wrap(MemoryRef(m), dims) + $(Expr(:new, :(Array{T, N}), :ref, :dims)) +end +@eval @propagate_inbounds function wrap(::Type{Array}, m::MemoryRef{T}, l::Integer) where {T} + dims = (Int(l),) + ref = _wrap(m, dims) + $(Expr(:new, :(Array{T, 1}), :ref, :dims)) end -function wrap(::Type{Array}, m::MemoryRef{T}, l::Integer) where {T} - wrap(Array, m, (l,)) +@eval @propagate_inbounds function wrap(::Type{Array}, m::Memory{T}, l::Integer) where {T} + dims = (Int(l),) + ref = _wrap(MemoryRef(m), (l,)) + $(Expr(:new, :(Array{T, 1}), :ref, :dims)) end -function wrap(::Type{Array}, m::Memory{T}, l::Integer) where {T} - wrap(Array, MemoryRef(m), (l,)) +@eval @propagate_inbounds function wrap(::Type{Array}, m::Memory{T}) where {T} + ref = MemoryRef(m) + dims = (length(m),) + $(Expr(:new, :(Array{T, 1}), :ref, :dims)) end diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 9244ee5f3f8ec5..21303f527e3f2c 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -1221,45 +1221,53 @@ function semi_concrete_eval_call(interp::AbstractInterpreter, return nothing end +const_prop_result(inf_result::InferenceResult) = + ConstCallResults(inf_result.result, inf_result.exc_result, ConstPropResult(inf_result), + inf_result.ipo_effects, inf_result.linfo) + +# return cached constant analysis result +return_cached_result(::AbstractInterpreter, inf_result::InferenceResult, ::AbsIntState) = + const_prop_result(inf_result) + function const_prop_call(interp::AbstractInterpreter, mi::MethodInstance, result::MethodCallResult, arginfo::ArgInfo, sv::AbsIntState, concrete_eval_result::Union{Nothing, ConstCallResults}=nothing) inf_cache = get_inference_cache(interp) 𝕃ᵢ = typeinf_lattice(interp) inf_result = cache_lookup(𝕃ᵢ, mi, arginfo.argtypes, inf_cache) - if inf_result === nothing - # fresh constant prop' - argtypes = has_conditional(𝕃ᵢ, sv) ? ConditionalArgtypes(arginfo, sv) : SimpleArgtypes(arginfo.argtypes) - inf_result = InferenceResult(mi, argtypes, typeinf_lattice(interp)) - if !any(inf_result.overridden_by_const) - add_remark!(interp, sv, "[constprop] Could not handle constant info in matching_cache_argtypes") - return nothing - end - frame = InferenceState(inf_result, #=cache_mode=#:local, interp) - if frame === nothing - add_remark!(interp, sv, "[constprop] Could not retrieve the source") - return nothing # this is probably a bad generated function (unsound), but just ignore it - end - frame.parent = sv - if !typeinf(interp, frame) - add_remark!(interp, sv, "[constprop] Fresh constant inference hit a cycle") - return nothing - end - @assert inf_result.result !== nothing - if concrete_eval_result !== nothing - # override return type and effects with concrete evaluation result if available - inf_result.result = concrete_eval_result.rt - inf_result.ipo_effects = concrete_eval_result.effects - end - else + if inf_result !== nothing # found the cache for this constant prop' if inf_result.result === nothing add_remark!(interp, sv, "[constprop] Found cached constant inference in a cycle") return nothing end + @assert inf_result.linfo === mi "MethodInstance for cached inference result does not match" + return return_cached_result(interp, inf_result, sv) + end + # perform fresh constant prop' + argtypes = has_conditional(𝕃ᵢ, sv) ? ConditionalArgtypes(arginfo, sv) : SimpleArgtypes(arginfo.argtypes) + inf_result = InferenceResult(mi, argtypes, typeinf_lattice(interp)) + if !any(inf_result.overridden_by_const) + add_remark!(interp, sv, "[constprop] Could not handle constant info in matching_cache_argtypes") + return nothing + end + frame = InferenceState(inf_result, #=cache_mode=#:local, interp) + if frame === nothing + add_remark!(interp, sv, "[constprop] Could not retrieve the source") + return nothing # this is probably a bad generated function (unsound), but just ignore it + end + frame.parent = sv + if !typeinf(interp, frame) + add_remark!(interp, sv, "[constprop] Fresh constant inference hit a cycle") + return nothing + end + @assert inf_result.result !== nothing + if concrete_eval_result !== nothing + # override return type and effects with concrete evaluation result if available + inf_result.result = concrete_eval_result.rt + inf_result.ipo_effects = concrete_eval_result.effects end - return ConstCallResults(inf_result.result, inf_result.exc_result, - ConstPropResult(inf_result), inf_result.ipo_effects, mi) + return const_prop_result(inf_result) end # TODO implement MustAlias forwarding diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index e30c6e5f96fcb0..747fd6e3ef7055 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -818,23 +818,29 @@ struct EdgeCallResult end end +# return cached regular inference result +function return_cached_result(::AbstractInterpreter, codeinst::CodeInstance, caller::AbsIntState) + rt = cached_return_type(codeinst) + effects = ipo_effects(codeinst) + update_valid_age!(caller, WorldRange(min_world(codeinst), max_world(codeinst))) + return EdgeCallResult(rt, codeinst.exctype, codeinst.def, effects) +end + # compute (and cache) an inferred AST and return the current best estimate of the result type function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize(atype), sparams::SimpleVector, caller::AbsIntState) mi = specialize_method(method, atype, sparams)::MethodInstance - code = get(code_cache(interp), mi, nothing) + codeinst = get(code_cache(interp), mi, nothing) force_inline = is_stmt_inline(get_curr_ssaflag(caller)) - if code isa CodeInstance # return existing rettype if the code is already inferred - inferred = @atomic :monotonic code.inferred + if codeinst isa CodeInstance # return existing rettype if the code is already inferred + inferred = @atomic :monotonic codeinst.inferred if inferred === nothing && force_inline # we already inferred this edge before and decided to discard the inferred code, # nevertheless we re-infer it here again in order to propagate the re-inferred # source to the inliner as a volatile result cache_mode = CACHE_MODE_VOLATILE else - rt = cached_return_type(code) - effects = ipo_effects(code) - update_valid_age!(caller, WorldRange(min_world(code), max_world(code))) - return EdgeCallResult(rt, code.exctype, mi, effects) + @assert codeinst.def === mi "MethodInstance for cached edge does not match" + return return_cached_result(interp, codeinst, caller) end else cache_mode = CACHE_MODE_GLOBAL # cache edge targets globally by default diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index 383ae2c6831828..d4785df98ec21b 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -2028,7 +2028,21 @@ AbstractFloat """ Integer <: Real -Abstract supertype for all integers. +Abstract supertype for all integers (e.g. [`Signed`](@ref), [`Unsigned`](@ref), and [`Bool`](@ref)). + +See also [`isinteger`](@ref), [`trunc`](@ref), [`div`](@ref). + +# Examples +``` +julia> 42 isa Integer +true + +julia> 1.0 isa Integer +false + +julia> isinteger(1.0) +true +``` """ Integer @@ -2043,6 +2057,21 @@ Signed Unsigned <: Integer Abstract supertype for all unsigned integers. + +Built-in unsigned integers are printed in hexadecimal, with prefix `0x`, +and can be entered in the same way. + +# Examples +``` +julia> typemax(UInt8) +0xff + +julia> Int(0x00d) +13 + +julia> unsigned(true) +0x0000000000000001 +``` """ Unsigned @@ -2053,57 +2082,147 @@ Boolean type, containing the values `true` and `false`. `Bool` is a kind of number: `false` is numerically equal to `0` and `true` is numerically equal to `1`. -Moreover, `false` acts as a multiplicative "strong zero": +Moreover, `false` acts as a multiplicative "strong zero" +against [`NaN`](@ref) and [`Inf`](@ref): ```jldoctest -julia> false == 0 +julia> [true, false] == [1, 0] true -julia> true == 1 -true +julia> 42.0 + true +43.0 -julia> 0 * NaN -NaN +julia> 0 .* (NaN, Inf, -Inf) +(NaN, NaN, NaN) -julia> false * NaN -0.0 +julia> false .* (NaN, Inf, -Inf) +(0.0, 0.0, -0.0) ``` -See also: [`digits`](@ref), [`iszero`](@ref), [`NaN`](@ref). +Branches via [`if`](@ref) and other conditionals only accept `Bool`. +There are no "truthy" values in Julia. + +Comparisons typically return `Bool`, and broadcasted comparisons may +return [`BitArray`](@ref) instead of an `Array{Bool}`. + +```jldoctest +julia> [1 2 3 4 5] .< pi +1×5 BitMatrix: + 1 1 1 0 0 + +julia> map(>(pi), [1 2 3 4 5]) +1×5 Matrix{Bool}: + 0 0 0 1 1 +``` + +See also [`trues`](@ref), [`falses`](@ref), [`ifelse`](@ref). """ Bool -for (bit, sign, exp, frac) in ((16, 1, 5, 10), (32, 1, 8, 23), (64, 1, 11, 52)) - @eval begin - """ - Float$($bit) <: AbstractFloat +""" + Float64 <: AbstractFloat <: Real - $($bit)-bit floating point number type (IEEE 754 standard). +64-bit floating point number type (IEEE 754 standard). +Binary format is 1 sign, 11 exponent, 52 fraction bits. +See [`bitstring`](@ref), [`signbit`](@ref), [`exponent`](@ref), [`frexp`](@ref), +and [`significand`](@ref) to access various bits. - Binary format: $($sign) sign, $($exp) exponent, $($frac) fraction bits. - """ - $(Symbol("Float", bit)) - end -end +This is the default for floating point literals, `1.0 isa Float64`, +and for many operations such as `1/2, 2pi, log(2), range(0,90,length=4)`. +Unlike integers, this default does not change with `Sys.WORD_SIZE`. + +The exponent for scientific notation can be entered as `e` or `E`, +thus `2e3 === 2.0E3 === 2.0 * 10^3`. Doing so is strongly preferred over +`10^n` because integers overflow, thus `2.0 * 10^19 < 0` but `2e19 > 0`. + +See also [`Inf`](@ref), [`NaN`](@ref), [`floatmax`](@ref), [`Float32`](@ref), [`Complex`](@ref). +""" +Float64 + +""" + Float32 <: AbstractFloat <: Real + +32-bit floating point number type (IEEE 754 standard). +Binary format is 1 sign, 8 exponent, 23 fraction bits. + +The exponent for scientific notation should be entered as lower-case `f`, +thus `2f3 === 2.0f0 * 10^3 === Float32(2_000)`. +For array literals and comprehensions, the element type can be specified before +the square brackets: `Float32[1,4,9] == Float32[i^2 for i in 1:3]`. + +See also [`Inf32`](@ref), [`NaN32`](@ref), [`Float16`](@ref), [`exponent`](@ref), [`frexp`](@ref). +""" +Float32 + +""" + Float16 <: AbstractFloat <: Real + +16-bit floating point number type (IEEE 754 standard). +Binary format is 1 sign, 5 exponent, 10 fraction bits. +""" +Float16 for bit in (8, 16, 32, 64, 128) + type = Symbol(:Int, bit) + srange = bit > 31 ? "" : "Represents numbers `n ∈ " * repr(eval(:(typemin($type):typemax($type)))) * "`.\n" + unshow = repr(eval(Symbol(:UInt, bit))(bit-1)) + @eval begin """ - Int$($bit) <: Signed + Int$($bit) <: Signed <: Integer $($bit)-bit signed integer type. + + $($(srange))Note that such integers overflow without warning, + thus `typemax($($type)) + $($type)(1) < 0`. + + See also [`Int`](@ref $Int), [`widen`](@ref), [`BigInt`](@ref). """ $(Symbol("Int", bit)) """ - UInt$($bit) <: Unsigned + UInt$($bit) <: Unsigned <: Integer $($bit)-bit unsigned integer type. + + Printed in hexadecimal, thus $($(unshow)) == $($(bit-1)). """ $(Symbol("UInt", bit)) end end +""" + Int + +Sys.WORD_SIZE-bit signed integer type, `Int <: Signed <: Integer <: Real`. + +This is the default type of most integer literals and is an alias for either `Int32` +or `Int64`, depending on `Sys.WORD_SIZE`. It is the type returned by functions such as +[`length`](@ref), and the standard type for indexing arrays. + +Note that integers overflow without warning, thus `typemax(Int) + 1 < 0` and `10^19 < 0`. +Overflow can be avoided by using [`BigInt`](@ref). +Very large integer literals will use a wider type, for instance `10_000_000_000_000_000_000 isa Int128`. + +Integer division is [`div`](@ref) alias `÷`, +whereas [`/`](@ref) acting on integers returns [`Float64`](@ref). + +See also [`$(Symbol("Int", Sys.WORD_SIZE))`](@ref), [`widen`](@ref), [`typemax`](@ref), [`bitstring`](@ref). +""" +Int + +""" + UInt + +Sys.WORD_SIZE-bit unsigned integer type, `UInt <: Unsigned <: Integer`. + +Like [`Int`](@ref Int), the alias `UInt` may point to either `UInt32` or `UInt64`, +according to the value of `Sys.WORD_SIZE` on a given computer. + +Printed and parsed in hexadecimal: `UInt(15) === $(repr(UInt(15)))`. +""" +UInt + """ Symbol @@ -2796,7 +2915,13 @@ Ptr{T}() """ +(x, y...) -Addition operator. `x+y+z+...` calls this function with all arguments, i.e. `+(x, y, z, ...)`. +Addition operator. + +Infix `x+y+z+...` calls this function with all arguments, i.e. `+(x, y, z, ...)`, +which by default then calls `(x+y) + z + ...` starting from the left. + +Note that overflow is possible for most integer types, including the +default `Int`, when adding large numbers. # Examples ```jldoctest @@ -2805,6 +2930,14 @@ julia> 1 + 20 + 4 julia> +(1, 20, 4) 25 + +julia> [1,2] + [3,4] +2-element Vector{Int64}: + 4 + 6 + +julia> typemax(Int) + 1 < 0 +true ``` """ (+)(x, y...) @@ -2828,6 +2961,12 @@ julia> -[1 2; 3 4] 2×2 Matrix{Int64}: -1 -2 -3 -4 + +julia> -(true) # promotes to Int +-1 + +julia> -(0x003) +0xfffd ``` """ -(x) @@ -2851,7 +2990,18 @@ julia> -(2, 4.5) """ *(x, y...) -Multiplication operator. `x*y*z*...` calls this function with all arguments, i.e. `*(x, y, z, ...)`. +Multiplication operator. + +Infix `x*y*z*...` calls this function with all arguments, i.e. `*(x, y, z, ...)`, +which by default then calls `(x*y) * z * ...` starting from the left. + +Juxtaposition such as `2pi` also calls `*(2, pi)`. Note that this operation +has higher precedence than a literal `*`. Note also that juxtaposition "0x..." +(integer zero times a variable whose name starts with `x`) is forbidden as +it clashes with unsigned integer literals: `0x01 isa UInt8`. + +Note that overflow is possible for most integer types, including the default `Int`, +when multiplying large numbers. # Examples ```jldoctest @@ -2860,6 +3010,17 @@ julia> 2 * 7 * 8 julia> *(2, 7, 8) 112 + +julia> [2 0; 0 3] * [1, 10] # matrix * vector +2-element Vector{Int64}: + 2 + 30 + +julia> 1/2pi, 1/2*pi # juxtaposition has higher precedence +(0.15915494309189535, 1.5707963267948966) + +julia> x = [1, 2]; x'x # adjoint vector * vector +5 ``` """ (*)(x, y...) @@ -2867,8 +3028,10 @@ julia> *(2, 7, 8) """ /(x, y) -Right division operator: multiplication of `x` by the inverse of `y` on the right. Gives -floating-point results for integer arguments. +Right division operator: multiplication of `x` by the inverse of `y` on the right. + +Gives floating-point results for integer arguments. +See [`÷`](@ref div) for integer division, or [`//`](@ref) for [`Rational`](@ref) results. # Examples ```jldoctest diff --git a/base/env.jl b/base/env.jl index 3092efbe372e03..2928a8bdfede29 100644 --- a/base/env.jl +++ b/base/env.jl @@ -3,19 +3,18 @@ if Sys.iswindows() const ERROR_ENVVAR_NOT_FOUND = UInt32(203) - const env_dict = Dict{String, Vector{Cwchar_t}}() - const env_lock = ReentrantLock() + const env_dict = Lockable(Dict{String, Vector{Cwchar_t}}()) function memoized_env_lookup(str::AbstractString) # Windows environment variables have a different format from Linux / MacOS, and previously # incurred allocations because we had to convert a String to a Vector{Cwchar_t} each time # an environment variable was looked up. This function memoizes that lookup process, storing # the String => Vector{Cwchar_t} pairs in env_dict - @lock env_lock begin - var = get(env_dict, str, nothing) + @lock env_dict begin + var = get(env_dict[], str, nothing) if isnothing(var) var = cwstring(str) - env_dict[str] = var + env_dict[][str] = var end return var end diff --git a/base/essentials.jl b/base/essentials.jl index 9c156fb1b8f6df..fdc66e94d1a00d 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -673,7 +673,7 @@ Change the type-interpretation of the binary data in the isbits value `x` to that of the isbits type `Out`. The size (ignoring padding) of `Out` has to be the same as that of the type of `x`. For example, `reinterpret(Float32, UInt32(7))` interprets the 4 bytes corresponding to `UInt32(7)` as a -[`Float32`](@ref). +[`Float32`](@ref). Note that `reinterpret(In, reinterpret(Out, x)) === x` ```jldoctest julia> reinterpret(Float32, UInt32(7)) @@ -689,11 +689,16 @@ julia> reinterpret(Tuple{UInt16, UInt8}, (0x01, 0x0203)) (0x0301, 0x02) ``` +!!! note + + The treatment of padding differs from reinterpret(::DataType, ::AbstractArray). + !!! warning Use caution if some combinations of bits in `Out` are not considered valid and would otherwise be prevented by the type's constructors and methods. Unexpected behavior may result without additional validation. + """ function reinterpret(::Type{Out}, x) where {Out} if isprimitivetype(Out) && isprimitivetype(typeof(x)) diff --git a/base/file.jl b/base/file.jl index e63ed67ae249bc..6347f042e3422e 100644 --- a/base/file.jl +++ b/base/file.jl @@ -480,13 +480,26 @@ function tempdir() rc = ccall(:uv_os_tmpdir, Cint, (Ptr{UInt8}, Ptr{Csize_t}), buf, sz) if rc == 0 resize!(buf, sz[]) - return String(buf) + break elseif rc == Base.UV_ENOBUFS resize!(buf, sz[] - 1) # space for null-terminator implied by StringVector else uv_error("tempdir()", rc) end end + tempdir = String(buf) + try + s = stat(tempdir) + if !ispath(s) + @warn "tempdir path does not exist" tempdir + elseif !isdir(s) + @warn "tempdir path is not a directory" tempdir + end + catch ex + ex isa IOError || ex isa SystemError || rethrow() + @warn "accessing tempdir path failed" _exception=ex + end + return tempdir end """ @@ -504,13 +517,19 @@ function prepare_for_deletion(path::AbstractString) return end - try chmod(path, filemode(path) | 0o333) - catch; end + try + chmod(path, filemode(path) | 0o333) + catch ex + ex isa IOError || ex isa SystemError || rethrow() + end for (root, dirs, files) in walkdir(path; onerror=x->()) for dir in dirs dpath = joinpath(root, dir) - try chmod(dpath, filemode(dpath) | 0o333) - catch; end + try + chmod(dpath, filemode(dpath) | 0o333) + catch ex + ex isa IOError || ex isa SystemError || rethrow() + end end end end diff --git a/base/iobuffer.jl b/base/iobuffer.jl index 895205549bc7e2..dadb13e1f1e6a3 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -4,32 +4,45 @@ # Stateful string mutable struct GenericIOBuffer{T<:AbstractVector{UInt8}} <: IO - data::T # T should support: getindex, setindex!, length, copyto!, and resize! + data::T # T should support: getindex, setindex!, length, copyto!, similar, and (optionally) resize! reinit::Bool # if true, data needs to be re-allocated (after take!) readable::Bool writable::Bool seekable::Bool # if not seekable, implementation is free to destroy (compact) past read data append::Bool # add data at end instead of at pointer - size::Int # end pointer (and write pointer if append == true) + size::Int # end pointer (and write pointer if append == true) + offset maxsize::Int # fixed array size (typically pre-allocated) - ptr::Int # read (and maybe write) pointer + ptr::Int # read (and maybe write) pointer + offset + offset::Int # offset of ptr and size from actual start of data and actual size mark::Int # reset mark location for ptr (or <0 for no mark) function GenericIOBuffer{T}(data::T, readable::Bool, writable::Bool, seekable::Bool, append::Bool, maxsize::Integer) where T<:AbstractVector{UInt8} require_one_based_indexing(data) - new(data,false,readable,writable,seekable,append,length(data),maxsize,1,-1) + return new(data, false, readable, writable, seekable, append, length(data), maxsize, 1, 0, -1) end end -const IOBuffer = GenericIOBuffer{Vector{UInt8}} + +const IOBuffer = GenericIOBuffer{Memory{UInt8}} function GenericIOBuffer(data::T, readable::Bool, writable::Bool, seekable::Bool, append::Bool, maxsize::Integer) where T<:AbstractVector{UInt8} GenericIOBuffer{T}(data, readable, writable, seekable, append, maxsize) end +function GenericIOBuffer(data::Vector{UInt8}, readable::Bool, writable::Bool, seekable::Bool, append::Bool, + maxsize::Integer) + ref = data.ref + buf = GenericIOBuffer(ref.mem, readable, writable, seekable, append, maxsize) + offset = memoryrefoffset(ref) - 1 + buf.ptr += offset + buf.size = length(data) + offset + buf.offset = offset + return buf +end # allocate Vector{UInt8}s for IOBuffer storage that can efficiently become Strings -StringVector(n::Integer) = unsafe_wrap(Vector{UInt8}, _string_n(n)) +StringMemory(n::Integer) = unsafe_wrap(Memory{UInt8}, _string_n(n)) +StringVector(n::Integer) = wrap(Array, StringMemory(n)) # IOBuffers behave like Files. They are typically readable and writable. They are seekable. (They can be appendable). @@ -98,7 +111,7 @@ function IOBuffer( flags = open_flags(read=read, write=write, append=append, truncate=truncate) buf = GenericIOBuffer(data, flags.read, flags.write, true, flags.append, Int(maxsize)) if flags.truncate - buf.size = 0 + buf.size = buf.offset end return buf end @@ -113,7 +126,7 @@ function IOBuffer(; size = sizehint !== nothing ? Int(sizehint) : maxsize != typemax(Int) ? Int(maxsize) : 32 flags = open_flags(read=read, write=write, append=append, truncate=truncate) buf = IOBuffer( - StringVector(size), + StringMemory(size), read=flags.read, write=flags.write, append=flags.append, @@ -135,12 +148,12 @@ See [`IOBuffer`](@ref) for the available constructors. If `data` is given, creates a `PipeBuffer` to operate on a data vector, optionally specifying a size beyond which the underlying `Array` may not be grown. """ -PipeBuffer(data::AbstractVector{UInt8}=UInt8[]; maxsize::Int = typemax(Int)) = +PipeBuffer(data::AbstractVector{UInt8}=Memory{UInt8}(); maxsize::Int = typemax(Int)) = GenericIOBuffer(data, true, true, false, true, maxsize) -PipeBuffer(maxsize::Integer) = (x = PipeBuffer(StringVector(maxsize), maxsize = maxsize); x.size=0; x) +PipeBuffer(maxsize::Integer) = (x = PipeBuffer(StringMemory(maxsize), maxsize = maxsize); x.size = 0; x) _similar_data(b::GenericIOBuffer, len::Int) = similar(b.data, len) -_similar_data(b::IOBuffer, len::Int) = StringVector(len) +_similar_data(b::IOBuffer, len::Int) = StringMemory(len) function copy(b::GenericIOBuffer) ret = typeof(b)(b.reinit ? _similar_data(b, 0) : b.writable ? @@ -148,6 +161,8 @@ function copy(b::GenericIOBuffer) b.readable, b.writable, b.seekable, b.append, b.maxsize) ret.size = b.size ret.ptr = b.ptr + ret.mark = b.mark + ret.offset = b.offset return ret end @@ -156,9 +171,9 @@ show(io::IO, b::GenericIOBuffer) = print(io, "IOBuffer(data=UInt8[...], ", "writable=", b.writable, ", ", "seekable=", b.seekable, ", ", "append=", b.append, ", ", - "size=", b.size, ", ", + "size=", b.size - b.offset, ", ", "maxsize=", b.maxsize == typemax(Int) ? "Inf" : b.maxsize, ", ", - "ptr=", b.ptr, ", ", + "ptr=", b.ptr - b.offset, ", ", "mark=", b.mark, ")") @noinline function _throw_not_readable() @@ -240,11 +255,9 @@ read(from::GenericIOBuffer, ::Type{Ptr{T}}) where {T} = convert(Ptr{T}, read(fro isreadable(io::GenericIOBuffer) = io.readable iswritable(io::GenericIOBuffer) = io.writable -# TODO: GenericIOBuffer is not iterable, so doesn't really have a length. -# This should maybe be sizeof() instead. -#length(io::GenericIOBuffer) = (io.seekable ? io.size : bytesavailable(io)) +filesize(io::GenericIOBuffer) = (io.seekable ? io.size - io.offset : bytesavailable(io)) bytesavailable(io::GenericIOBuffer) = io.size - io.ptr + 1 -position(io::GenericIOBuffer) = io.ptr-1 +position(io::GenericIOBuffer) = io.ptr - io.offset - 1 function skip(io::GenericIOBuffer, n::Integer) seekto = io.ptr + n @@ -262,7 +275,7 @@ function seek(io::GenericIOBuffer, n::Integer) # of an GenericIOBuffer), so that would need to be fixed in order to throw an error here #(n < 0 || n > io.size) && throw(ArgumentError("Attempted to seek outside IOBuffer boundaries.")) #io.ptr = n+1 - io.ptr = max(min(n+1, io.size+1), 1) + io.ptr = min(max(0, n)+io.offset, io.size)+1 return io end @@ -271,32 +284,66 @@ function seekend(io::GenericIOBuffer) return io end +# choose a resize strategy based on whether `resize!` is defined: +# for a Vector, we use `resize!`, but for most other types, +# this calls `similar`+copy +function _resize!(io::GenericIOBuffer, sz::Int) + a = io.data + offset = io.offset + if applicable(resize!, a, sz) + if offset != 0 + size = io.size + size > offset && copyto!(a, 1, a, offset + 1, min(sz, size - offset)) + io.ptr -= offset + io.size -= offset + io.offset = 0 + end + resize!(a, sz) + else + size = io.size + if size >= sz && sz != 0 + b = a + else + b = _similar_data(io, sz == 0 ? 0 : max(overallocation(size - io.offset), sz)) + end + size > offset && copyto!(b, 1, a, offset + 1, min(sz, size - offset)) + io.data = b + io.ptr -= offset + io.size -= offset + io.offset = 0 + end + return io +end + function truncate(io::GenericIOBuffer, n::Integer) io.writable || throw(ArgumentError("truncate failed, IOBuffer is not writeable")) io.seekable || throw(ArgumentError("truncate failed, IOBuffer is not seekable")) n < 0 && throw(ArgumentError("truncate failed, n bytes must be ≥ 0, got $n")) n > io.maxsize && throw(ArgumentError("truncate failed, $(n) bytes is exceeds IOBuffer maxsize $(io.maxsize)")) + n = Int(n) if io.reinit io.data = _similar_data(io, n) io.reinit = false - elseif n > length(io.data) - resize!(io.data, n) + elseif n > length(io.data) + io.offset + _resize!(io, n) end + ismarked(io) && io.mark > n && unmark(io) + n += io.offset io.data[io.size+1:n] .= 0 io.size = n io.ptr = min(io.ptr, n+1) - ismarked(io) && io.mark > n && unmark(io) return io end function compact(io::GenericIOBuffer) io.writable || throw(ArgumentError("compact failed, IOBuffer is not writeable")) io.seekable && throw(ArgumentError("compact failed, IOBuffer is seekable")) + io.reinit && return local ptr::Int, bytes_to_move::Int - if ismarked(io) && io.mark < io.ptr - if io.mark == 0 return end - ptr = io.mark - bytes_to_move = bytesavailable(io) + (io.ptr-io.mark) + if ismarked(io) && io.mark < position(io) + io.mark == 0 && return + ptr = io.mark + io.offset + bytes_to_move = bytesavailable(io) + (io.ptr - ptr) else ptr = io.ptr bytes_to_move = bytesavailable(io) @@ -304,19 +351,24 @@ function compact(io::GenericIOBuffer) copyto!(io.data, 1, io.data, ptr, bytes_to_move) io.size -= ptr - 1 io.ptr -= ptr - 1 - io.mark -= ptr - 1 - return io + io.offset = 0 + return end @noinline function ensureroom_slowpath(io::GenericIOBuffer, nshort::UInt) io.writable || throw(ArgumentError("ensureroom failed, IOBuffer is not writeable")) + if io.reinit + io.data = _similar_data(io, nshort % Int) + io.reinit = false + end if !io.seekable - if !ismarked(io) && io.ptr > 1 && io.size <= io.ptr - 1 + if !ismarked(io) && io.ptr > io.offset+1 && io.size <= io.ptr - 1 io.ptr = 1 io.size = 0 + io.offset = 0 else - datastart = ismarked(io) ? io.mark : io.ptr - if (io.size+nshort > io.maxsize) || + datastart = (ismarked(io) ? io.mark : io.ptr - io.offset) + if (io.size-io.offset+nshort > io.maxsize) || (datastart > 4096 && datastart > io.size - io.ptr) || (datastart > 262144) # apply somewhat arbitrary heuristics to decide when to destroy @@ -330,23 +382,18 @@ end @inline ensureroom(io::GenericIOBuffer, nshort::Int) = ensureroom(io, UInt(nshort)) @inline function ensureroom(io::GenericIOBuffer, nshort::UInt) - if !io.writable || (!io.seekable && io.ptr > 1) + if !io.writable || (!io.seekable && io.ptr > io.offset+1) || io.reinit ensureroom_slowpath(io, nshort) end - n = min((nshort % Int) + (io.append ? io.size : io.ptr-1), io.maxsize) - if io.reinit - io.data = _similar_data(io, n) - io.reinit = false - else - l = length(io.data) - if n > l - _growend!(io.data, (n - l) % UInt) - end + n = min((nshort % Int) + (io.append ? io.size : io.ptr-1) - io.offset, io.maxsize) + l = length(io.data) + io.offset + if n > l + _resize!(io, Int(n)) end return io end -eof(io::GenericIOBuffer) = (io.ptr-1 == io.size) +eof(io::GenericIOBuffer) = (io.ptr - 1 >= io.size) function closewrite(io::GenericIOBuffer) io.writable = false @@ -358,11 +405,12 @@ end io.writable = false io.seekable = false io.size = 0 + io.offset = 0 io.maxsize = 0 io.ptr = 1 io.mark = -1 - if io.writable - resize!(io.data, 0) + if io.writable && !io.reinit + io.data = _resize!(io, 0) end nothing end @@ -388,45 +436,45 @@ julia> String(take!(io)) function take!(io::GenericIOBuffer) ismarked(io) && unmark(io) if io.seekable - nbytes = io.size - data = copyto!(StringVector(nbytes), 1, io.data, 1, nbytes) + nbytes = io.size - io.offset + data = copyto!(StringVector(nbytes), 1, io.data, io.offset + 1, nbytes) else nbytes = bytesavailable(io) - data = read!(io,StringVector(nbytes)) + data = read!(io, StringVector(nbytes)) end if io.writable io.ptr = 1 io.size = 0 + io.offset = 0 end return data end function take!(io::IOBuffer) ismarked(io) && unmark(io) if io.seekable - if io.writable - if io.reinit - data = StringVector(0) - else - data = resize!(io.data, io.size) - io.reinit = true - end + nbytes = filesize(io) + if nbytes == 0 || io.reinit + data = StringVector(0) + elseif io.writable + data = wrap(Array, MemoryRef(io.data, io.offset + 1), nbytes) else - data = copyto!(StringVector(io.size), 1, io.data, 1, io.size) + data = copyto!(StringVector(io.size), 1, io.data, io.offset + 1, nbytes) end else nbytes = bytesavailable(io) - if io.writable - data = io.data - io.reinit = true - _deletebeg!(data, io.ptr-1) - resize!(data, nbytes) + if nbytes == 0 + data = StringVector(0) + elseif io.writable + data = wrap(Array, MemoryRef(io.data, io.ptr), nbytes) else - data = read!(io, StringVector(nbytes)) + data = read!(io, data) end end if io.writable + io.reinit = true io.ptr = 1 io.size = 0 + io.offset = 0 end return data end @@ -440,17 +488,23 @@ state. This should only be used internally for performance-critical `String` routines that immediately discard `io` afterwards, and it *assumes* that `io` is writable and seekable. -It saves no allocations compared to `take!`, it just omits some checks. +It might save an allocation compared to `take!` (if the compiler elides the +Array allocation), as well as omits some checks. """ -_unsafe_take!(io::IOBuffer) = resize!(io.data, io.size) +_unsafe_take!(io::IOBuffer) = + wrap(Array, io.size == io.offset ? + MemoryRef(Memory{UInt8}()) : + MemoryRef(io.data, io.offset + 1), + io.size - io.offset) function write(to::IO, from::GenericIOBuffer) + written::Int = bytesavailable(from) if to === from from.ptr = from.size + 1 - return 0 + else + written = GC.@preserve from unsafe_write(to, pointer(from.data, from.ptr), UInt(written)) + from.ptr += written end - written::Int = GC.@preserve from unsafe_write(to, pointer(from.data, from.ptr), UInt(bytesavailable(from))) - from.ptr += written return written end @@ -497,13 +551,13 @@ function readbytes!(io::GenericIOBuffer, b::Array{UInt8}, nb::Int) read_sub(io, b, 1, nr) return nr end -read(io::GenericIOBuffer) = read!(io,StringVector(bytesavailable(io))) +read(io::GenericIOBuffer) = read!(io, StringVector(bytesavailable(io))) readavailable(io::GenericIOBuffer) = read(io) -read(io::GenericIOBuffer, nb::Integer) = read!(io,StringVector(min(nb, bytesavailable(io)))) +read(io::GenericIOBuffer, nb::Integer) = read!(io, StringVector(min(nb, bytesavailable(io)))) function occursin(delim::UInt8, buf::IOBuffer) p = pointer(buf.data, buf.ptr) - q = GC.@preserve buf ccall(:memchr,Ptr{UInt8},(Ptr{UInt8},Int32,Csize_t),p,delim,bytesavailable(buf)) + q = GC.@preserve buf ccall(:memchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p, delim, bytesavailable(buf)) return q != C_NULL end @@ -532,8 +586,8 @@ end function copyline(out::GenericIOBuffer, s::IO; keep::Bool=false) copyuntil(out, s, 0x0a, keep=true) line = out.data - i = out.size - if keep || i == 0 || line[i] != 0x0a + i = out.size # XXX: this is only correct for appended data. if the data was inserted, only ptr should change + if keep || i == out.offset || line[i] != 0x0a return out elseif i < 2 || line[i-1] != 0x0d i -= 1 diff --git a/base/iostream.jl b/base/iostream.jl index ba422cd692fcd8..5d972945e00e0f 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -455,26 +455,24 @@ end function copyuntil(out::IOBuffer, s::IOStream, delim::UInt8; keep::Bool=false) ensureroom(out, 1) # make sure we can read at least 1 byte, for iszero(n) check below - ptr = (out.append ? out.size+1 : out.ptr) - d = out.data - len = length(d) while true + d = out.data + len = length(d) + ptr = (out.append ? out.size+1 : out.ptr) GC.@preserve d @_lock_ios s n= Int(ccall(:jl_readuntil_buf, Csize_t, (Ptr{Cvoid}, UInt8, Ptr{UInt8}, Csize_t), s.ios, delim, pointer(d, ptr), (len - ptr + 1) % Csize_t)) iszero(n) && break ptr += n - if d[ptr-1] == delim - keep || (ptr -= 1) - break - end + found = (d[ptr - 1] == delim) + found && !keep && (ptr -= 1) + out.size = max(out.size, ptr - 1) + out.append || (out.ptr = ptr) + found && break (eof(s) || len == out.maxsize) && break len = min(2len + 64, out.maxsize) - resize!(d, len) - end - out.size = max(out.size, ptr - 1) - if !out.append - out.ptr = ptr + ensureroom(out, len) + @assert length(out.data) >= len end return out end diff --git a/base/iterators.jl b/base/iterators.jl index b51920cdddb687..a0b3a6cd4672d7 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -953,12 +953,17 @@ struct Cycle{I} end """ - cycle(iter) + cycle(iter[, n::Int]) An iterator that cycles through `iter` forever. -If `iter` is empty, so is `cycle(iter)`. +If `n` is specified, then it cycles through `iter` that many times. +When `iter` is empty, so are `cycle(iter)` and `cycle(iter, n)`. -See also: [`Iterators.repeated`](@ref), [`Base.repeat`](@ref). +`Iterators.cycle(iter, n)` is the lazy equivalent of [`Base.repeat`](@ref)`(vector, n)`, +while [`Iterators.repeated`](@ref)`(iter, n)` is the lazy [`Base.fill`](@ref)`(item, n)`. + +!!! compat "Julia 1.11" + The method `cycle(iter, n)` was added in Julia 1.11. # Examples ```jldoctest @@ -967,9 +972,19 @@ julia> for (i, v) in enumerate(Iterators.cycle("hello")) i > 10 && break end hellohelloh + +julia> foreach(print, Iterators.cycle(['j', 'u', 'l', 'i', 'a'], 3)) +juliajuliajulia + +julia> repeat([1,2,3], 4) == collect(Iterators.cycle([1,2,3], 4)) +true + +julia> fill([1,2,3], 4) == collect(Iterators.repeated([1,2,3], 4)) +true ``` """ cycle(xs) = Cycle(xs) +cycle(xs, n::Integer) = flatten(repeated(xs, n)) eltype(::Type{Cycle{I}}) where {I} = eltype(I) IteratorEltype(::Type{Cycle{I}}) where {I} = IteratorEltype(I) @@ -1000,7 +1015,7 @@ repeated(x) = Repeated(x) An iterator that generates the value `x` forever. If `n` is specified, generates `x` that many times (equivalent to `take(repeated(x), n)`). -See also: [`Iterators.cycle`](@ref), [`Base.repeat`](@ref). +See also [`fill`](@ref Base.fill), and compare [`Iterators.cycle`](@ref). # Examples ```jldoctest @@ -1012,6 +1027,12 @@ julia> collect(a) [1 2] [1 2] [1 2] + +julia> ans == fill([1 2], 4) +true + +julia> Iterators.cycle([1 2], 4) |> collect |> println +[1, 2, 1, 2, 1, 2, 1, 2] ``` """ repeated(x, n::Integer) = take(repeated(x), Int(n)) diff --git a/base/loading.jl b/base/loading.jl index 5f0ea1859bc0f6..8bcaa4b2ab5b36 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -776,15 +776,30 @@ end function entry_point_and_project_file(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}} path = normpath(joinpath(dir, "$name.jl")) isfile_casesensitive(path) && return path, nothing - dir = joinpath(dir, name) - path, project_file = entry_point_and_project_file_inside(dir, name) + dir_name = joinpath(dir, name) + path, project_file = entry_point_and_project_file_inside(dir_name, name) path === nothing || return path, project_file - dir = dir * ".jl" - path, project_file = entry_point_and_project_file_inside(dir, name) + dir_jl = dir_name * ".jl" + path, project_file = entry_point_and_project_file_inside(dir_jl, name) path === nothing || return path, project_file return nothing, nothing end +# Find the project file for the extension `ext` in the implicit env `dir`` +function implicit_env_project_file_extension(dir::String, ext::PkgId) + for pkg in readdir(dir; join=true) + project_file = env_project_file(pkg) + project_file isa String || continue + proj = project_file_name_uuid(project_file, "") + uuid5(proj.uuid, ext.name) == ext.uuid || continue + path = project_file_ext_path(project_file, ext.name) + if path !== nothing + return path, project_file + end + end + return nothing, nothing +end + # given a path and a name, return the entry point function entry_path(path::String, name::String)::Union{Nothing,String} isfile_casesensitive(path) && return normpath(path) @@ -796,11 +811,12 @@ end ## explicit project & manifest API ## # find project file root or deps `name => uuid` mapping +# `ext` is the name of the extension if `name` is loaded from one # return `nothing` if `name` is not found -function explicit_project_deps_get(project_file::String, name::String)::Union{Nothing,UUID} +function explicit_project_deps_get(project_file::String, name::String, ext::Union{String,Nothing}=nothing)::Union{Nothing,UUID} d = parsed_toml(project_file) - root_uuid = dummy_uuid(project_file) if get(d, "name", nothing)::Union{String, Nothing} === name + root_uuid = dummy_uuid(project_file) uuid = get(d, "uuid", nothing)::Union{String, Nothing} return uuid === nothing ? root_uuid : UUID(uuid) end @@ -809,6 +825,19 @@ function explicit_project_deps_get(project_file::String, name::String)::Union{No uuid = get(deps, name, nothing)::Union{String, Nothing} uuid === nothing || return UUID(uuid) end + if ext !== nothing + extensions = get(d, "extensions", nothing) + extensions === nothing && return nothing + ext_data = get(extensions, ext, nothing) + ext_data === nothing && return nothing + if (ext_data isa String && name == ext_data) || (ext_data isa Vector{String} && name in ext_data) + weakdeps = get(d, "weakdeps", nothing)::Union{Dict{String, Any}, Nothing} + weakdeps === nothing && return nothing + wuuid = get(weakdeps, name, nothing)::Union{String, Nothing} + wuuid === nothing && return nothing + return UUID(wuuid) + end + end return nothing end @@ -996,11 +1025,28 @@ end function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Union{Nothing,PkgId} @assert where.uuid !== nothing project_file = entry_point_and_project_file(dir, where.name)[2] - project_file === nothing && return nothing # a project file is mandatory for a package with a uuid + if project_file === nothing + # `where` could be an extension + project_file = implicit_env_project_file_extension(dir, where)[2] + project_file === nothing && return nothing + end proj = project_file_name_uuid(project_file, where.name) - proj == where || return nothing # verify that this is the correct project file + ext = nothing + if proj !== where + # `where` could be an extension in `proj` + d = parsed_toml(project_file) + exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing} + if exts !== nothing && where.name in keys(exts) + if where.uuid !== uuid5(proj.uuid, where.name) + return nothing + end + ext = where.name + else + return nothing + end + end # this is the correct project, so stop searching here - pkg_uuid = explicit_project_deps_get(project_file, name) + pkg_uuid = explicit_project_deps_get(project_file, name, ext) return PkgId(pkg_uuid, name) end @@ -2447,7 +2493,7 @@ end const PRECOMPILE_TRACE_COMPILE = Ref{String}() function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String}, - concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout) + concrete_deps::typeof(_concrete_dependencies), flags::Cmd=``, internal_stderr::IO = stderr, internal_stdout::IO = stdout) @nospecialize internal_stderr internal_stdout rm(output, force=true) # Remove file if it exists output_o === nothing || rm(output_o, force=true) @@ -2503,6 +2549,7 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) $(opts) --startup-file=no --history-file=no --warn-overwrite=yes --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no") + $flags $trace -`, "OPENBLAS_NUM_THREADS" => 1, @@ -2570,17 +2617,17 @@ This can be used to reduce package load times. Cache files are stored in `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref) for important notes. """ -function compilecache(pkg::PkgId, internal_stderr::IO = stderr, internal_stdout::IO = stdout; reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}()) +function compilecache(pkg::PkgId, internal_stderr::IO = stderr, internal_stdout::IO = stdout; flags::Cmd=``, reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}()) @nospecialize internal_stderr internal_stdout path = locate_package(pkg) path === nothing && throw(ArgumentError("$(repr("text/plain", pkg)) not found during precompilation")) - return compilecache(pkg, path, internal_stderr, internal_stdout; reasons) + return compilecache(pkg, path, internal_stderr, internal_stdout; flags, reasons) end const MAX_NUM_PRECOMPILE_FILES = Ref(10) function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, internal_stdout::IO = stdout, - keep_loaded_modules::Bool = true; reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}()) + keep_loaded_modules::Bool = true; flags::Cmd=``, reasons::Union{Dict{String,Int},Nothing}=Dict{String,Int}()) @nospecialize internal_stderr internal_stdout # decide where to put the resulting cache file @@ -2618,7 +2665,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in close(tmpio_o) close(tmpio_so) end - p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, internal_stderr, internal_stdout) + p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, flags, internal_stderr, internal_stdout) if success(p) if cache_objects @@ -3627,5 +3674,5 @@ end precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing)) precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) -precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), IO, IO)) -precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), IO, IO)) +precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), IO, IO, Cmd)) +precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), IO, IO, Cmd)) diff --git a/base/lock.jl b/base/lock.jl index eb76a3b30e6a4c..7cbb023a78ee4a 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -294,6 +294,63 @@ macro lock_nofail(l, expr) end end +""" + Lockable(value, lock = ReentrantLock()) + +Creates a `Lockable` object that wraps `value` and +associates it with the provided `lock`. This object +supports [`@lock`](@ref), [`lock`](@ref), [`trylock`](@ref), +[`unlock`](@ref). To access the value, index the lockable object while +holding the lock. + +!!! compat "Julia 1.11" + Requires at least Julia 1.11. + +## Example + +```jldoctest +julia> locked_list = Base.Lockable(Int[]); + +julia> @lock(locked_list, push!(locked_list[], 1)) # must hold the lock to access the value +1-element Vector{Int64}: + 1 + +julia> lock(summary, locked_list) +"1-element Vector{Int64}" +``` +""" +struct Lockable{T, L <: AbstractLock} + value::T + lock::L +end + +Lockable(value) = Lockable(value, ReentrantLock()) +getindex(l::Lockable) = (assert_havelock(l.lock); l.value) + +""" + lock(f::Function, l::Lockable) + +Acquire the lock associated with `l`, execute `f` with the lock held, +and release the lock when `f` returns. `f` will receive one positional +argument: the value wrapped by `l`. If the lock is already locked by a +different task/thread, wait for it to become available. +When this function returns, the `lock` has been released, so the caller should +not attempt to `unlock` it. + +!!! compat "Julia 1.11" + Requires at least Julia 1.11. +""" +function lock(f, l::Lockable) + lock(l.lock) do + f(l.value) + end +end + +# implement the rest of the Lock interface on Lockable +lock(l::Lockable) = lock(l.lock) +trylock(l::Lockable) = trylock(l.lock) +unlock(l::Lockable) = unlock(l.lock) + @eval Threads begin """ Threads.Condition([lock]) diff --git a/base/logging.jl b/base/logging.jl index 436d00335acf3c..d0f612c31eeae2 100644 --- a/base/logging.jl +++ b/base/logging.jl @@ -132,6 +132,11 @@ isless(a::LogLevel, b::LogLevel) = isless(a.level, b.level) -(level::LogLevel, inc::Integer) = LogLevel(level.level-inc) convert(::Type{LogLevel}, level::Integer) = LogLevel(level) +""" + BelowMinLevel + +Alias for [`LogLevel(-1_000_001)`](@ref LogLevel). +""" const BelowMinLevel = LogLevel(-1000001) """ Debug @@ -157,6 +162,11 @@ const Warn = LogLevel( 1000) Alias for [`LogLevel(2000)`](@ref LogLevel). """ const Error = LogLevel( 2000) +""" + AboveMaxLevel + +Alias for [`LogLevel(1_000_001)`](@ref LogLevel). +""" const AboveMaxLevel = LogLevel( 1000001) # Global log limiting mechanism for super fast but inflexible global log limiting. diff --git a/base/math.jl b/base/math.jl index 0071d427fbbacc..6ed69188371dd1 100644 --- a/base/math.jl +++ b/base/math.jl @@ -949,9 +949,11 @@ end Compute ``x \\times 2^n``. +See also [`frexp`](@ref), [`exponent`](@ref). + # Examples ```jldoctest -julia> ldexp(5., 2) +julia> ldexp(5.0, 2) 20.0 ``` """ @@ -1008,8 +1010,7 @@ Returns the largest integer `y` such that `2^y ≤ abs(x)`. Throws a `DomainError` when `x` is zero, infinite, or [`NaN`](@ref). For any other non-subnormal floating-point number `x`, this corresponds to the exponent bits of `x`. -See also [`signbit`](@ref), [`significand`](@ref), [`frexp`](@ref), [`issubnormal`](@ref), [`log2`](@ref). - +See also [`signbit`](@ref), [`significand`](@ref), [`frexp`](@ref), [`issubnormal`](@ref), [`log2`](@ref), [`ldexp`](@ref). # Examples ```jldoctest julia> exponent(8) @@ -1021,8 +1022,16 @@ julia> exponent(6.5) julia> exponent(-1//4) -2 +julia> exponent(3.142e-4) +-12 + julia> exponent(floatmin(Float32)), exponent(nextfloat(0.0f0)) (-126, -149) + +julia> exponent(0.0) +ERROR: DomainError with 0.0: +Cannot be ±0.0. +[...] ``` """ function exponent(x::T) where T<:IEEEFloat @@ -1062,6 +1071,8 @@ a non-zero finite number, then the result will be a number of the same type and sign as `x`, and whose absolute value is on the interval ``[1,2)``. Otherwise `x` is returned. +See also [`frexp`](@ref), [`exponent`](@ref). + # Examples ```jldoctest julia> significand(15.2) @@ -1096,10 +1107,19 @@ end Return `(x,exp)` such that `x` has a magnitude in the interval ``[1/2, 1)`` or 0, and `val` is equal to ``x \\times 2^{exp}``. + +See also [`significand`](@ref), [`exponent`](@ref), [`ldexp`](@ref). + # Examples ```jldoctest -julia> frexp(12.8) -(0.8, 4) +julia> frexp(6.0) +(0.75, 3) + +julia> significand(6.0), exponent(6.0) # interval [1, 2) instead +(1.5, 2) + +julia> frexp(0.0), frexp(NaN), frexp(-Inf) # exponent would give an error +((0.0, 0), (NaN, 0), (-Inf, 0)) ``` """ function frexp(x::T) where T<:IEEEFloat diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 4b462c8e2079b5..ed0a48b4f23cf1 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -879,6 +879,28 @@ _maybe_linear_logical_index(::IndexLinear, A, i) = LogicalIndex{Int}(i) uncolon(::Tuple{}) = Slice(OneTo(1)) uncolon(inds::Tuple) = Slice(inds[1]) +""" + _prechecked_iterate(iter[, state]) + +Internal function used to eliminate the dead branch in `iterate`. +Fallback to `iterate` by default, but optimized for indices type in `Base`. +""" +@propagate_inbounds _prechecked_iterate(iter) = iterate(iter) +@propagate_inbounds _prechecked_iterate(iter, state) = iterate(iter, state) + +_prechecked_iterate(iter::AbstractUnitRange, i = first(iter)) = i, convert(eltype(iter), i + step(iter)) +_prechecked_iterate(iter::LinearIndices, i = first(iter)) = i, i + 1 +_prechecked_iterate(iter::CartesianIndices) = first(iter), first(iter) +function _prechecked_iterate(iter::CartesianIndices, i) + i′ = IteratorsMD.inc(i.I, iter.indices) + return i′, i′ +end +_prechecked_iterate(iter::SCartesianIndices2) = first(iter), first(iter) +function _prechecked_iterate(iter::SCartesianIndices2{K}, (;i, j)) where {K} + I = i < K ? SCartesianIndex2{K}(i + 1, j) : SCartesianIndex2{K}(1, j + 1) + return I, I +end + ### From abstractarray.jl: Internal multidimensional indexing definitions ### getindex(x::Union{Number,AbstractChar}, ::CartesianIndex{0}) = x getindex(t::Tuple, i::CartesianIndex{1}) = getindex(t, i.I[1]) @@ -910,14 +932,11 @@ function _generate_unsafe_getindex!_body(N::Int) quote @inline D = eachindex(dest) - Dy = iterate(D) + Dy = _prechecked_iterate(D) @inbounds @nloops $N j d->I[d] begin - # This condition is never hit, but at the moment - # the optimizer is not clever enough to split the union without it - Dy === nothing && return dest - (idx, state) = Dy + (idx, state) = Dy::NTuple{2,Any} dest[idx] = @ncall $N getindex src j - Dy = iterate(D, state) + Dy = _prechecked_iterate(D, state) end return dest end @@ -953,14 +972,12 @@ function _generate_unsafe_setindex!_body(N::Int) @nexprs $N d->(I_d = unalias(A, I[d])) idxlens = @ncall $N index_lengths I @ncall $N setindex_shape_check x′ (d->idxlens[d]) - Xy = iterate(x′) + X = eachindex(x′) + Xy = _prechecked_iterate(X) @inbounds @nloops $N i d->I_d begin - # This is never reached, but serves as an assumption for - # the optimizer that it does not need to emit error paths - Xy === nothing && break - (val, state) = Xy - @ncall $N setindex! A val i - Xy = iterate(x′, state) + (idx, state) = Xy::NTuple{2,Any} + @ncall $N setindex! A x′[idx] i + Xy = _prechecked_iterate(X, state) end A end diff --git a/base/promotion.jl b/base/promotion.jl index 16769f8566b92c..1d4fea8c404eb9 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -430,7 +430,11 @@ end """ ^(x, y) -Exponentiation operator. If `x` is a matrix, computes matrix exponentiation. +Exponentiation operator. + +If `x` and `y` are integers, the result may overflow. +To enter numbers in scientific notation, use [`Float64`](@ref) literals +such as `1.2e3` rather than `1.2 * 10^3`. If `y` is an `Int` literal (e.g. `2` in `x^2` or `-3` in `x^-3`), the Julia code `x^y` is transformed by the compiler to `Base.literal_pow(^, x, Val(y))`, to @@ -440,20 +444,31 @@ where usually `^ == Base.^` unless `^` has been defined in the calling namespace.) If `y` is a negative integer literal, then `Base.literal_pow` transforms the operation to `inv(x)^-y` by default, where `-y` is positive. +See also [`exp2`](@ref), [`<<`](@ref). + # Examples ```jldoctest julia> 3^5 243 -julia> A = [1 2; 3 4] -2×2 Matrix{Int64}: - 1 2 - 3 4 +julia> 3^-1 # uses Base.literal_pow +0.3333333333333333 + +julia> p = -1; + +julia> 3^p +ERROR: DomainError with -1: +Cannot raise an integer x to a negative power -1. +[...] + +julia> 3.0^p +0.3333333333333333 + +julia> 10^19 > 0 # integer overflow +false -julia> A^3 -2×2 Matrix{Int64}: - 37 54 - 81 118 +julia> big(10)^19 == 1e19 +true ``` """ ^(x::Number, y::Number) = ^(promote(x,y)...) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 23bfb38a286540..ff674690a5c66b 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -46,6 +46,23 @@ struct ReinterpretArray{T,N,S,A<:AbstractArray{S},IsReshaped} <: AbstractArray{T 3 + 4im 5 + 6im ``` + + If the location of padding bits does not line up between `T` and `eltype(A)`, the resulting array will be + read-only or write-only, to prevent invalid bits from being written to or read from, respectively. + + ```jldoctest + julia> a = reinterpret(Tuple{UInt8, UInt32}, UInt32[1, 2]) + 1-element reinterpret(Tuple{UInt8, UInt32}, ::Vector{UInt32}): + (0x01, 0x00000002) + + julia> a[1] = 3 + ERROR: Padding of type Tuple{UInt8, UInt32} is not compatible with type UInt32. + + julia> b = reinterpret(UInt32, Tuple{UInt8, UInt32}[(0x01, 0x00000002)]); # showing will error + + julia> b[1] + ERROR: Padding of type UInt32 is not compatible with type Tuple{UInt8, UInt32}. + ``` """ function reinterpret(::Type{T}, a::A) where {T,N,S,A<:AbstractArray{S, N}} function thrownonint(S::Type, T::Type, dim) diff --git a/base/set.jl b/base/set.jl index 6b0c933f79faac..2f96cef626b6fb 100644 --- a/base/set.jl +++ b/base/set.jl @@ -479,13 +479,21 @@ end """ allunique(itr) -> Bool + allunique(f, itr) -> Bool Return `true` if all values from `itr` are distinct when compared with [`isequal`](@ref). +Or if all of `[f(x) for x in itr]` are distinct, for the second method. + +Note that `allunique(f, itr)` may call `f` fewer than `length(itr)` times. +The precise number of calls is regarded as an implementation detail. `allunique` may use a specialized implementation when the input is sorted. See also: [`unique`](@ref), [`issorted`](@ref), [`allequal`](@ref). +!!! compat "Julia 1.11" + The method `allunique(f, itr)` requires at least Julia 1.11. + # Examples ```jldoctest julia> allunique([1, 2, 3]) @@ -499,6 +507,9 @@ false julia> allunique([NaN, 2.0, NaN, 4.0]) false + +julia> allunique(abs, [1, -1, 2]) +false ``` """ function allunique(C) @@ -509,8 +520,10 @@ function allunique(C) return _hashed_allunique(C) end +allunique(f, xs) = allunique(Generator(f, xs)) + function _hashed_allunique(C) - seen = Set{eltype(C)}() + seen = Set{@default_eltype(C)}() x = iterate(C) if haslength(C) && length(C) > 1000 for i in OneTo(1000) @@ -582,16 +595,30 @@ function allunique(t::Tuple) end allunique(t::Tuple{}) = true +function allunique(f::F, t::Tuple) where {F} + length(t) < 2 && return true + length(t) < 32 || return _hashed_allunique(Generator(f, t)) + return allunique(map(f, t)) +end + """ allequal(itr) -> Bool + allequal(f, itr) -> Bool Return `true` if all values from `itr` are equal when compared with [`isequal`](@ref). +Or if all of `[f(x) for x in itr]` are equal, for the second method. + +Note that `allequal(f, itr)` may call `f` fewer than `length(itr)` times. +The precise number of calls is regarded as an implementation detail. See also: [`unique`](@ref), [`allunique`](@ref). !!! compat "Julia 1.8" The `allequal` function requires at least Julia 1.8. +!!! compat "Julia 1.11" + The method `allequal(f, itr)` requires at least Julia 1.11. + # Examples ```jldoctest julia> allequal([]) @@ -608,14 +635,36 @@ false julia> allequal(Dict(:a => 1, :b => 1)) false + +julia> allequal(abs2, [1, -1]) +true ``` """ -allequal(itr) = isempty(itr) ? true : all(isequal(first(itr)), itr) +function allequal(itr) + if haslength(itr) + length(itr) <= 1 && return true + end + pl = Iterators.peel(itr) + isnothing(pl) && return true + a, rest = pl + return all(isequal(a), rest) +end allequal(c::Union{AbstractSet,AbstractDict}) = length(c) <= 1 allequal(r::AbstractRange) = iszero(step(r)) || length(r) <= 1 +allequal(f, xs) = allequal(Generator(f, xs)) + +function allequal(f, xs::Tuple) + length(xs) <= 1 && return true + f1 = f(xs[1]) + for x in tail(xs) + isequal(f1, f(x)) || return false + end + return true +end + filter!(f, s::Set) = unsafe_filter!(f, s) const hashs_seed = UInt === UInt64 ? 0x852ada37cfe8e0ce : 0xcfe8e0ce diff --git a/base/stream.jl b/base/stream.jl index 14621c464ce1ba..3de61181e978d4 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -608,7 +608,7 @@ end function alloc_request(buffer::IOBuffer, recommended_size::UInt) ensureroom(buffer, Int(recommended_size)) ptr = buffer.append ? buffer.size + 1 : buffer.ptr - nb = min(length(buffer.data), buffer.maxsize) - ptr + 1 + nb = min(length(buffer.data)-buffer.offset, buffer.maxsize) + buffer.offset - ptr + 1 return (Ptr{Cvoid}(pointer(buffer.data, ptr)), nb) end @@ -932,7 +932,7 @@ function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb::Int) nread = readbytes!(sbuf, a, nb) else newbuf = PipeBuffer(a, maxsize=nb) - newbuf.size = 0 # reset the write pointer to the beginning + newbuf.size = newbuf.offset # reset the write pointer to the beginning nread = try s.buffer = newbuf write(newbuf, sbuf) @@ -979,7 +979,7 @@ function unsafe_read(s::LibuvStream, p::Ptr{UInt8}, nb::UInt) unsafe_read(sbuf, p, nb) else newbuf = PipeBuffer(unsafe_wrap(Array, p, nb), maxsize=Int(nb)) - newbuf.size = 0 # reset the write pointer to the beginning + newbuf.size = newbuf.offset # reset the write pointer to the beginning try s.buffer = newbuf write(newbuf, sbuf) diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index d67ac70d8c6845..1eeaaa668d9ee7 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -200,34 +200,36 @@ julia> annotatedstring(AnnotatedString("annotated", [(1:9, :label => 1)]), ", an function annotatedstring(xs...) isempty(xs) && return AnnotatedString("") size = mapreduce(_str_sizehint, +, xs) - s = IOContext(IOBuffer(sizehint=size), :color => true) + buf = IOBuffer(sizehint=size) + s = IOContext(buf, :color => true) annotations = Vector{Tuple{UnitRange{Int}, Pair{Symbol, Any}}}() for x in xs + size = filesize(s.io) if x isa AnnotatedString for (region, annot) in x.annotations - push!(annotations, (s.io.size .+ (region), annot)) + push!(annotations, (size .+ (region), annot)) end print(s, x.string) elseif x isa SubString{<:AnnotatedString} for (region, annot) in x.string.annotations start, stop = first(region), last(region) if start <= x.offset + x.ncodeunits && stop > x.offset - rstart = s.io.size + max(0, start - x.offset - 1) + 1 - rstop = s.io.size + min(stop, x.offset + x.ncodeunits) - x.offset + rstart = size + max(0, start - x.offset - 1) + 1 + rstop = size + min(stop, x.offset + x.ncodeunits) - x.offset push!(annotations, (rstart:rstop, annot)) end end print(s, SubString(x.string.string, x.offset, x.ncodeunits, Val(:noshift))) elseif x isa AnnotatedChar for annot in x.annotations - push!(annotations, (1+s.io.size:1+s.io.size, annot)) + push!(annotations, (1+size:1+size, annot)) end print(s, x.char) else print(s, x) end end - str = String(resize!(s.io.data, s.io.size)) + str = String(take!(buf)) AnnotatedString(str, annotations) end @@ -400,7 +402,8 @@ AnnotatedIOBuffer() = AnnotatedIOBuffer(IOBuffer()) function show(io::IO, aio::AnnotatedIOBuffer) show(io, AnnotatedIOBuffer) - print(io, '(', aio.io.size, " byte", ifelse(aio.io.size == 1, "", "s"), ", ", + size = filesize(aio.io) + print(io, '(', size, " byte", ifelse(size == 1, "", "s"), ", ", length(aio.annotations), " annotation", ifelse(length(aio.annotations) == 1, "", "s"), ")") end @@ -410,7 +413,7 @@ pipe_writer(io::AnnotatedIOBuffer) = io.io # Useful `IOBuffer` methods that we don't get from `AbstractPipe` position(io::AnnotatedIOBuffer) = position(io.io) seek(io::AnnotatedIOBuffer, n::Integer) = (seek(io.io, n); io) -seekend(io::AnnotatedIOBuffer) = seekend(io.io) +seekend(io::AnnotatedIOBuffer) = (seekend(io.io); io) skip(io::AnnotatedIOBuffer, n::Integer) = (skip(io.io, n); io) copy(io::AnnotatedIOBuffer) = AnnotatedIOBuffer(copy(io.io), copy(io.annotations)) diff --git a/base/strings/io.jl b/base/strings/io.jl index 6db737532637d9..94b5a1e4e4614f 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -353,28 +353,19 @@ function join(io::IO, iterator, delim="") end end -# TODO: If/when we have `AnnotatedIO`, we can revisit this and -# implement it more nicely. -function join_annotated(iterator, delim="", last=delim) - xs = zip(iterator, Iterators.repeated(delim)) |> Iterators.flatten |> collect - xs = xs[1:end-1] - if length(xs) > 1 - xs[end-1] = last - end - annotatedstring(xs...)::AnnotatedString{String} -end - -function _join_maybe_annotated(args...) - if any(_isannotated ∘ eltype, args) - join_annotated(args...) +function _join_preserve_annotations(iterator, args...) + if _isannotated(eltype(iterator)) || any(_isannotated, args) + io = AnnotatedIOBuffer() + join(io, iterator, args...) + read(seekstart(io), AnnotatedString{String}) else - sprint(join, args...) + sprint(join, iterator, args...) end end -join(iterator) = _join_maybe_annotated(iterator) -join(iterator, delim) = _join_maybe_annotated(iterator, delim) -join(iterator, delim, last) = _join_maybe_annotated(iterator, delim, last) +join(iterator) = _join_preserve_annotations(iterator) +join(iterator, delim) = _join_preserve_annotations(iterator, delim) +join(iterator, delim, last) = _join_preserve_annotations(iterator, delim, last) ## string escaping & unescaping ## diff --git a/base/strings/string.jl b/base/strings/string.jl index 29216ae97aa37b..b2afce897a937f 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -63,8 +63,27 @@ by [`take!`](@ref) on a writable [`IOBuffer`](@ref) and by calls to In other cases, `Vector{UInt8}` data may be copied, but `v` is truncated anyway to guarantee consistent behavior. """ -String(v::AbstractVector{UInt8}) = String(copyto!(StringVector(length(v)), v)) -String(v::Vector{UInt8}) = ccall(:jl_array_to_string, Ref{String}, (Any,), v) +String(v::AbstractVector{UInt8}) = String(copyto!(StringMemory(length(v)), v)) +function String(v::Memory{UInt8}) + len = length(v) + len == 0 && return "" + return ccall(:jl_genericmemory_to_string, Ref{String}, (Any, Int), v, len) +end +function String(v::Vector{UInt8}) + #return ccall(:jl_array_to_string, Ref{String}, (Any,), v) + len = length(v) + len == 0 && return "" + ref = v.ref + if ref.ptr_or_offset == ref.mem.ptr + str = ccall(:jl_genericmemory_to_string, Ref{String}, (Any, Int), ref.mem, len) + else + str = ccall(:jl_pchar_to_string, Ref{String}, (Ptr{UInt8}, Int), ref, len) + end + # optimized empty!(v); sizehint!(v, 0) calls + setfield!(v, :size, (0,)) + setfield!(v, :ref, MemoryRef(Memory{UInt8}())) + return str +end """ unsafe_string(p::Ptr{UInt8}, [length::Integer]) @@ -97,8 +116,8 @@ Create a new `String` from an existing `AbstractString`. String(s::AbstractString) = print_to_string(s) @assume_effects :total String(s::Symbol) = unsafe_string(unsafe_convert(Ptr{UInt8}, s)) -unsafe_wrap(::Type{Vector{UInt8}}, s::String) = ccall(:jl_string_to_array, Ref{Vector{UInt8}}, (Any,), s) -unsafe_wrap(::Type{Vector{UInt8}}, s::FastContiguousSubArray{UInt8,1,Vector{UInt8}}) = unsafe_wrap(Vector{UInt8}, pointer(s), size(s)) +unsafe_wrap(::Type{Memory{UInt8}}, s::String) = ccall(:jl_string_to_genericmemory, Ref{Memory{UInt8}}, (Any,), s) +unsafe_wrap(::Type{Vector{UInt8}}, s::String) = wrap(Array, unsafe_wrap(Memory{UInt8}, s)) Vector{UInt8}(s::CodeUnits{UInt8,String}) = copyto!(Vector{UInt8}(undef, length(s)), s) Vector{UInt8}(s::String) = Vector{UInt8}(codeunits(s)) diff --git a/base/subarray.jl b/base/subarray.jl index 6f1e20af2b966c..eca06fa3eacff3 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -338,7 +338,9 @@ function getindex(V::FastContiguousSubArray, i::Int) end # parents of FastContiguousSubArrays may support fast indexing with AbstractUnitRanges, # so we may just forward the indexing to the parent -function getindex(V::FastContiguousSubArray, i::AbstractUnitRange{Int}) +# This may only be done for non-offset ranges, as the result would otherwise have offset axes +const OneBasedRanges = Union{OneTo{Int}, UnitRange{Int}, Slice{OneTo{Int}}, IdentityUnitRange{OneTo{Int}}} +function getindex(V::FastContiguousSubArray, i::OneBasedRanges) @inline @boundscheck checkbounds(V, i) @inbounds r = V.parent[V.offset1 .+ i] @@ -541,3 +543,6 @@ end function replace_in_print_matrix(S::SubArray{<:Any,1,<:AbstractVector}, i::Integer, j::Integer, s::AbstractString) replace_in_print_matrix(S.parent, to_indices(S.parent, reindex(S.indices, (i,)))..., j, s) end + +# XXX: this is considerably more unsafe than the other similarly named methods +unsafe_wrap(::Type{Vector{UInt8}}, s::FastContiguousSubArray{UInt8,1,Vector{UInt8}}) = unsafe_wrap(Vector{UInt8}, pointer(s), size(s)) diff --git a/base/sysinfo.jl b/base/sysinfo.jl index 69ee93864aadc6..97e5e2a71bcbca 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -358,7 +358,7 @@ free_memory() = ccall(:uv_get_available_memory, UInt64, ()) Get the total memory in RAM (including that which is currently used) in bytes. This amount may be constrained, e.g., by Linux control groups. For the unconstrained -amount, see `Sys.physical_memory()`. +amount, see `Sys.total_physical_memory()`. """ function total_memory() constrained = ccall(:uv_get_constrained_memory, UInt64, ()) diff --git a/deps/llvm.mk b/deps/llvm.mk index 18dab5dbceb307..3e3ea4e79c24ea 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -210,6 +210,9 @@ LLVM_CMAKE += -DCMAKE_EXE_LINKER_FLAGS="$(LLVM_LDFLAGS)" \ LLVM_CMAKE += -DLLVM_VERSION_SUFFIX:STRING="jl" LLVM_CMAKE += -DLLVM_SHLIB_SYMBOL_VERSION:STRING="JL_LLVM_$(LLVM_VER_SHORT)" +# Change the default bug report URL to Julia's issue tracker +LLVM_CMAKE += -DBUG_REPORT_URL="https://github.com/julialang/julia" + # Apply version-specific LLVM patches sequentially LLVM_PATCH_PREV := define LLVM_PATCH diff --git a/deps/unwind.mk b/deps/unwind.mk index 66607845428c4e..f8e8260b431fa2 100644 --- a/deps/unwind.mk +++ b/deps/unwind.mk @@ -73,7 +73,10 @@ check-unwind: $(BUILDDIR)/libunwind-$(UNWIND_VER)/build-checked ## LLVM libunwind ## -LLVMUNWIND_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=MinSizeRel -DLIBUNWIND_ENABLE_PEDANTIC=OFF -DLLVM_CONFIG_PATH=$(build_depsbindir)/llvm-config +LLVMUNWIND_OPTS := $(CMAKE_COMMON) \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DLIBUNWIND_ENABLE_PEDANTIC=OFF \ + -DLLVM_PATH=$(SRCCACHE)/$(LLVM_SRC_DIR)/llvm $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER).tar.xz: | $(SRCCACHE) $(JLDOWNLOAD) $@ https://github.com/llvm/llvm-project/releases/download/llvmorg-$(LLVMUNWIND_VER)/libunwind-$(LLVMUNWIND_VER).src.tar.xz diff --git a/doc/make.jl b/doc/make.jl index cf266370acb018..1f654649d92379 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -126,6 +126,7 @@ generate_markdown("NEWS") Manual = [ "manual/getting-started.md", + "manual/installation.md", "manual/variables.md", "manual/integers-and-floating-point-numbers.md", "manual/mathematical-operations.md", diff --git a/doc/man/julia.1 b/doc/man/julia.1 index a060c6dffb6b8b..c58b00120b7ec5 100644 --- a/doc/man/julia.1 +++ b/doc/man/julia.1 @@ -229,8 +229,8 @@ fallbacks to the latest compatible BugReporting.jl if not. For more information, .TP --heap-size-hint= -Forces garbage collection if memory usage is higher than that value. The memory hint might be -specified in megabytes (500M) or gigabytes (1.5G) +Forces garbage collection if memory usage is higher than that value. The value can be +specified in units of K, M, G, T, or % of physical memory. .TP --compile={yes*|no|all|min} diff --git a/doc/src/base/base.md b/doc/src/base/base.md index 64c635ed3043bf..3fd2bbb12bb1f4 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -1,4 +1,4 @@ -#replacefield Essentials +# Essentials ## Introduction @@ -469,7 +469,6 @@ Core.setglobalonce! Core.replaceglobal! ``` - ## Documentation (See also the [documentation](@ref man-documentation) chapter.) ```@docs diff --git a/doc/src/base/numbers.md b/doc/src/base/numbers.md index 8167650ac17d14..aad4e949010543 100644 --- a/doc/src/base/numbers.md +++ b/doc/src/base/numbers.md @@ -63,6 +63,8 @@ Core.Int64 Core.UInt64 Core.Int128 Core.UInt128 +Base.Int +Base.UInt Base.BigInt Base.Complex Base.Rational diff --git a/doc/src/base/parallel.md b/doc/src/base/parallel.md index c3106b8caf8c75..5798da54a5cde0 100644 --- a/doc/src/base/parallel.md +++ b/doc/src/base/parallel.md @@ -51,6 +51,7 @@ Base.trylock Base.islocked Base.ReentrantLock Base.@lock +Base.Lockable ``` ## Channels diff --git a/doc/src/devdocs/probes.md b/doc/src/devdocs/probes.md index d15723e945462f..e75e7f6bb01565 100644 --- a/doc/src/devdocs/probes.md +++ b/doc/src/devdocs/probes.md @@ -137,8 +137,8 @@ fib(x) = x <= 1 ? 1 : fib(x-1) + fib(x-2) beaver = @spawn begin while true fib(30) - # This safepoint is necessary until #41616, since otherwise this - # loop will never yield to GC. + # A manual safepoint is necessary since otherwise this loop + # may never yield to GC. GC.safepoint() end end diff --git a/doc/src/manual/arrays.md b/doc/src/manual/arrays.md index 6aa3a7a724e0e3..34e5a231d22de7 100644 --- a/doc/src/manual/arrays.md +++ b/doc/src/manual/arrays.md @@ -398,7 +398,7 @@ the result in single precision by writing: Float32[ 0.25*x[i-1] + 0.5*x[i] + 0.25*x[i+1] for i=2:length(x)-1 ] ``` -## Generator Expressions +## [Generator Expressions](@id man-generators) Comprehensions can also be written without the enclosing square brackets, producing an object known as a generator. This object can be iterated to produce values on demand, instead of allocating diff --git a/doc/src/manual/constructors.md b/doc/src/manual/constructors.md index f8b9dd9673082c..9f9afca3e076c0 100644 --- a/doc/src/manual/constructors.md +++ b/doc/src/manual/constructors.md @@ -575,3 +575,53 @@ This constructor will be invoked by the syntax `SummedArray(a)`. The syntax `new specifying parameters for the type to be constructed, i.e. this call will return a `SummedArray{T,S}`. `new{T,S}` can be used in any constructor definition, but for convenience the parameters to `new{}` are automatically derived from the type being constructed when possible. + +## Constructors are just callable objects + +An object of any type may be [made callable](@ref "Function-like objects") by defining a +method. This includes types, i.e., objects of type [`Type`](@ref); and constructors may, +in fact, be viewed as just callable type objects. For example, there are many methods +defined on `Bool` and various supertypes of it: + +```julia-repl +julia> methods(Bool) +# 10 methods for type constructor: + [1] Bool(x::BigFloat) + @ Base.MPFR mpfr.jl:393 + [2] Bool(x::Float16) + @ Base float.jl:338 + [3] Bool(x::Rational) + @ Base rational.jl:138 + [4] Bool(x::Real) + @ Base float.jl:233 + [5] (dt::Type{<:Integer})(ip::Sockets.IPAddr) + @ Sockets ~/tmp/jl/jl/julia-nightly-assert/share/julia/stdlib/v1.11/Sockets/src/IPAddr.jl:11 + [6] (::Type{T})(x::Enum{T2}) where {T<:Integer, T2<:Integer} + @ Base.Enums Enums.jl:19 + [7] (::Type{T})(z::Complex) where T<:Real + @ Base complex.jl:44 + [8] (::Type{T})(x::Base.TwicePrecision) where T<:Number + @ Base twiceprecision.jl:265 + [9] (::Type{T})(x::T) where T<:Number + @ boot.jl:894 + [10] (::Type{T})(x::AbstractChar) where T<:Union{AbstractChar, Number} + @ char.jl:50 +``` + +The usual constructor syntax is exactly equivalent to the function-like object +syntax, so trying to define a method with each syntax will cause the first method +to be overwritten by the next one: + +```jldoctest +julia> struct S + f::Int + end + +julia> S() = S(7) +S + +julia> (::Type{S})() = S(8) # overwrites the previous constructor method + +julia> S() +S(8) +``` diff --git a/doc/src/manual/installation.md b/doc/src/manual/installation.md new file mode 100644 index 00000000000000..07acfd1c62681b --- /dev/null +++ b/doc/src/manual/installation.md @@ -0,0 +1,126 @@ +# [Installation](@id man-installation) + +There are many ways to install Julia. The following sections highlight the +recommended method for each of the main supported platforms, and then present +alternative ways that might be useful in specialized situations. + +The current installation recommendation is a solution based on Juliaup. If you +installed Julia previously with a method that is _not_ based on Juliaup and want +to switch your system to an installation that is based on Juliaup, we recommend +that you uninstall all previous Julia versions, ensure that you remove anything +Julia related from your `PATH` variable and then install Julia with one of the +methods described below. + +## Windows + +On Windows Julia can be installed directly from the Windows store +[here](https://www.microsoft.com/store/apps/9NJNWW8PVKMN). One can also install +exactly the same version by executing + +``` +winget install julia -s msstore +``` + +in any shell. + +## Mac and Linux + +Julia can be installed on Linux or Mac by executing + +``` +curl -fsSL https://install.julialang.org | sh +``` + +in a shell. + +### Command line arguments + +One can pass various command line arguments to the Julia installer. The syntax +for installer arguments is + +```bash +curl -fsSL https://install.julialang.org | sh -s -- +``` + +Here `` should be replaced with one or more of the following arguments: +- `--yes` (or `-y`): Run the installer in a non-interactive mode. All +configuration values use their default or a value supplied as a command line +argument. +- `--default-channel=`: Configure the default Juliaup channel. For +example `--default-channel lts` would install the `lts` channel and configure it +as the default. +- `--add-to-path=`: Configure whether Julia should be added to the `PATH` +environment variable. Valid values are `yes` (default) and `no`. +- `--background-selfupdate=`: Configure an optional CRON job that +auto-updates Juliaup if `` has a value larger than 0. The actual value +controls how often the CRON job will run to check for a new Juliaup version in +seconds. The default value is 0, i.e. no CRON job will be created. +- `--startup-selfupdate=`: Configure how often Julia will check for new +versions of Juliaup when Julia is started. The default is every 1440 minutes. +- `-p=` (or `--path`): Configure where the Julia and Juliaup binaries are +installed. The default is `~/.juliaup`. + +## Alternative installation methods + +Note that we recommend the following methods _only_ if none of the installation +methods described above work for your system. + +Some of the installation methods described below recommend installing a package +called `juliaup`. Note that this nevertheless installs a fully functional +Julia system, not just Juliaup. + +### App Installer (Windows) + +If the Windows Store is blocked on a system, we have an alternative +[MSIX App Installer](https://learn.microsoft.com/en-us/windows/msix/app-installer/app-installer-file-overview) +based setup. To use the App Installer version, download +[this](https://install.julialang.org/Julia.appinstaller) file and open it by +double clicking on it. + +### MSI Installer (Windows) + +If neither the Windows Store nor the App Installer version work on your Windows +system, you can also use a MSI based installer. Note that this installation +methods comes with serious limitations and is generally not recommended unless +no other method works. For example, there is no automatic update mechanism for +Juliaup with this installation method. The 64 bit version of the MSI installer +can be downloaded from [here](https://install.julialang.org/Julia-x64.msi) and +the 32 bit version from [here](https://install.julialang.org/Julia-x86.msi). + + By default the install will be a per-user install that does not require + elevation. You can also do a system install by running the following command + from a shell: + +``` +msiexec /i ALLUSERS=1 +``` + +### [Homebrew](https://brew.sh) (Mac and Linux) + +On systems with brew, you can install Julia by running +``` +brew install juliaup +``` +in a shell. Note that you will have to update Juliaup with standard brew +commands. + +### [Arch Linux - AUR](https://aur.archlinux.org/packages/juliaup/) (Linux) + +On Arch Linux, Juliaup is available [in the Arch User Repository (AUR)](https://aur.archlinux.org/packages/juliaup/). + +### [openSUSE Tumbleweed](https://get.opensuse.org/tumbleweed/) (Linux) + +On openSUSE Tumbleweed, you can install Julia by running + +```sh +zypper install juliaup +``` +in a shell with root privileges. + +### [cargo](https://crates.io/crates/juliaup/) (Windows, Mac and Linux) + +To install Julia via Rust's cargo, run: + +```sh +cargo install juliaup +``` diff --git a/doc/src/manual/metaprogramming.md b/doc/src/manual/metaprogramming.md index b1623ff8591b0b..b619021fcef92a 100644 --- a/doc/src/manual/metaprogramming.md +++ b/doc/src/manual/metaprogramming.md @@ -629,6 +629,15 @@ julia> @showarg(1+1) julia> @showarg(println("Yo!")) :(println("Yo!")) + +julia> @showarg(1) # Numeric literal +1 + +julia> @showarg("Yo!") # String literal +"Yo!" + +julia> @showarg("Yo! $("hello")") # String with interpolation is an Expr rather than a String +:("Yo! $("hello")") ``` In addition to the given argument list, every macro is passed extra arguments named `__source__` and `__module__`. diff --git a/doc/src/manual/strings.md b/doc/src/manual/strings.md index ec146125024b84..f4acffb2ae91bc 100644 --- a/doc/src/manual/strings.md +++ b/doc/src/manual/strings.md @@ -832,7 +832,7 @@ of the substring that matches, but perhaps we want to capture any non-blank text character. We could do the following: ```jldoctest -julia> m = match(r"^\s*(?:#\s*(.*?)\s*$|$)", "# a comment ") +julia> m = match(r"^\s*(?:#\s*(.*?)\s*$)", "# a comment ") RegexMatch("# a comment ", 1="a comment") ``` @@ -981,10 +981,10 @@ x Tells the regular expression parser to ignore most whitespace For example, the following regex has all three flags turned on: ```jldoctest -julia> r"a+.*b+.*?d$"ism -r"a+.*b+.*?d$"ims +julia> r"a+.*b+.*d$"ism +r"a+.*b+.*d$"ims -julia> match(r"a+.*b+.*?d$"ism, "Goodbye,\nOh, angry,\nBad world\n") +julia> match(r"a+.*b+.*d$"ism, "Goodbye,\nOh, angry,\nBad world\n") RegexMatch("angry,\nBad world") ``` diff --git a/doc/src/manual/style-guide.md b/doc/src/manual/style-guide.md index 19b4908927187c..92516623724b11 100644 --- a/doc/src/manual/style-guide.md +++ b/doc/src/manual/style-guide.md @@ -262,6 +262,29 @@ Splicing function arguments can be addictive. Instead of `[a..., b...]`, use sim which already concatenates arrays. [`collect(a)`](@ref) is better than `[a...]`, but since `a` is already iterable it is often even better to leave it alone, and not convert it to an array. +## Ensure constructors return an instance of their own type + +When a method `T(x)` is called on a type `T`, it is generally expected to return a value of type T. +Defining a [constructor](@ref man-constructors) that returns an unexpected type can lead to confusing and unpredictable behavior: + +```jldoctest +julia> struct Foo{T} + x::T + end + +julia> Base.Float64(foo::Foo) = Foo(Float64(foo.x)) # Do not define methods like this + +julia> Float64(Foo(3)) # Should return `Float64` +Foo{Float64}(3.0) + +julia> Foo{Int}(x) = Foo{Float64}(x) # Do not define methods like this + +julia> Foo{Int}(3) # Should return `Foo{Int}` +Foo{Float64}(3.0) +``` + +To maintain code clarity and ensure type consistency, always design constructors to return an instance of the type they are supposed to construct. + ## Don't use unnecessary static parameters A function signature: diff --git a/src/array.c b/src/array.c index 198215831497df..979772e649727f 100644 --- a/src/array.c +++ b/src/array.c @@ -99,21 +99,6 @@ jl_genericmemory_t *_new_genericmemory_(jl_value_t *mtype, size_t nel, int8_t is JL_DLLEXPORT jl_genericmemory_t *jl_string_to_genericmemory(jl_value_t *str); -JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str) -{ - jl_task_t *ct = jl_current_task; - jl_genericmemory_t *mem = jl_string_to_genericmemory(str); - JL_GC_PUSH1(&mem); - int ndimwords = 1; - int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t); - jl_array_t *a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, jl_array_uint8_type); - a->ref.mem = mem; - a->ref.ptr_or_offset = mem->ptr; - a->dimsize[0] = mem->length; - JL_GC_POP(); - return a; -} - JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer) { diff --git a/src/gc.c b/src/gc.c index 044dcf8113e9f0..a920958307c48c 100644 --- a/src/gc.c +++ b/src/gc.c @@ -4037,14 +4037,22 @@ void jl_gc_init(void) gc_num.max_pause = 0; gc_num.max_memory = 0; + uint64_t mem_reserve = 250*1024*1024; // LLVM + other libraries need some amount of memory + uint64_t min_heap_size_hint = mem_reserve + 1*1024*1024; + uint64_t hint = jl_options.heap_size_hint; #ifdef _P64 total_mem = uv_get_total_memory(); - uint64_t constrained_mem = uv_get_constrained_memory(); - if (constrained_mem > 0 && constrained_mem < total_mem) - jl_gc_set_max_memory(constrained_mem - 250*1024*1024); // LLVM + other libraries need some amount of memory + if (hint == 0) { + uint64_t constrained_mem = uv_get_constrained_memory(); + if (constrained_mem > 0 && constrained_mem < total_mem) + hint = constrained_mem; + } #endif - if (jl_options.heap_size_hint) - jl_gc_set_max_memory(jl_options.heap_size_hint - 250*1024*1024); + if (hint) { + if (hint < min_heap_size_hint) + hint = min_heap_size_hint; + jl_gc_set_max_memory(hint - mem_reserve); + } t_start = jl_hrtime(); } diff --git a/src/genericmemory.c b/src/genericmemory.c index 0bd4db30fd6902..f0e7b695f11229 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -101,6 +101,8 @@ JL_DLLEXPORT jl_genericmemory_t *jl_alloc_genericmemory(jl_value_t *mtype, size_ JL_DLLEXPORT jl_genericmemory_t *jl_string_to_genericmemory(jl_value_t *str) { + if (jl_string_len(str) == 0) + return (jl_genericmemory_t*)((jl_datatype_t*)jl_memory_uint8_type)->instance; jl_task_t *ct = jl_current_task; int tsz = sizeof(jl_genericmemory_t) + sizeof(void*); jl_genericmemory_t *m = (jl_genericmemory_t*)jl_gc_alloc(ct->ptls, tsz, jl_memory_uint8_type); diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index a72058f10f42de..91f989f611b223 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -436,7 +436,6 @@ XX(jl_stdout_stream) \ XX(jl_stored_inline) \ XX(jl_string_ptr) \ - XX(jl_string_to_array) \ XX(jl_subtype) \ XX(jl_subtype_env) \ XX(jl_subtype_env_size) \ diff --git a/src/jloptions.c b/src/jloptions.c index fa5d4ff43ee041..4b3f7209dc56d4 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -189,7 +189,7 @@ static const char opts[] = " --bug-report=help.\n\n" " --heap-size-hint= Forces garbage collection if memory usage is higher than that value.\n" - " The memory hint might be specified in megabytes(500M) or gigabytes(1G)\n\n" + " The value can be specified in units of K, M, G, T, or % of physical memory.\n\n" ; static const char opts_hidden[] = @@ -823,14 +823,24 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) case 'T': multiplier <<= 40; break; + case '%': + if (value > 100) + jl_errorf("julia: invalid percentage specified in --heap-size-hint"); + uint64_t mem = uv_get_total_memory(); + uint64_t cmem = uv_get_constrained_memory(); + if (cmem > 0 && cmem < mem) + mem = cmem; + multiplier = mem/100; + break; default: + jl_errorf("julia: invalid unit specified in --heap-size-hint"); break; } jl_options.heap_size_hint = (uint64_t)(value * multiplier); } } if (jl_options.heap_size_hint == 0) - jl_errorf("julia: invalid argument to --heap-size-hint without memory size specified"); + jl_errorf("julia: invalid memory size specified in --heap-size-hint"); break; case opt_gc_threads: diff --git a/src/jltypes.c b/src/jltypes.c index bf7561fc29e1b0..9dac534259f734 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1154,6 +1154,7 @@ static void cache_insert_type_set(jl_datatype_t *val, uint_t hv) jl_svec_t *cache_rehash_set(jl_svec_t *a, size_t newsz) { + newsz = newsz ? next_power_of_two(newsz) : 0; jl_value_t **ol = jl_svec_data(a); size_t sz = jl_svec_len(a); while (1) { diff --git a/src/staticdata.c b/src/staticdata.c index 0d38dddb79aafb..62c0b27cb8eabc 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -2305,6 +2305,7 @@ static void jl_reinit_ccallable(arraylist_t *ccallable_list, char *base, void *s static jl_svec_t *jl_prune_type_cache_hash(jl_svec_t *cache) JL_GC_DISABLED { size_t l = jl_svec_len(cache), i; + size_t sz = 0; if (l == 0) return cache; for (i = 0; i < l; i++) { @@ -2313,11 +2314,16 @@ static jl_svec_t *jl_prune_type_cache_hash(jl_svec_t *cache) JL_GC_DISABLED continue; if (ptrhash_get(&serialization_order, ti) == HT_NOTFOUND) jl_svecset(cache, i, jl_nothing); + else + sz += 1; } + if (sz < HT_N_INLINE) + sz = HT_N_INLINE; + void *idx = ptrhash_get(&serialization_order, cache); assert(idx != HT_NOTFOUND && idx != (void*)(uintptr_t)-1); assert(serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] == cache); - cache = cache_rehash_set(cache, l); + cache = cache_rehash_set(cache, sz); // redirect all references to the old cache to relocate to the new cache object ptrhash_put(&serialization_order, cache, idx); serialization_queue.items[(char*)idx - 1 - (char*)HT_NOTFOUND] = cache; diff --git a/stdlib/FileWatching/test/runtests.jl b/stdlib/FileWatching/test/runtests.jl index b9ad8d9bc77776..2592aea0243861 100644 --- a/stdlib/FileWatching/test/runtests.jl +++ b/stdlib/FileWatching/test/runtests.jl @@ -161,7 +161,7 @@ test2_12992() ####################################################################### # This section tests file watchers. # ####################################################################### -F_GETPATH = Sys.islinux() || Sys.iswindows() || Sys.isapple() || Sys.isfreebsd() # platforms where F_GETPATH is available +F_GETPATH = Sys.islinux() || Sys.iswindows() || Sys.isapple() # platforms where F_GETPATH is available F_PATH = F_GETPATH ? "afile.txt" : "" dir = mktempdir() file = joinpath(dir, "afile.txt") @@ -276,7 +276,7 @@ function test_dirmonitor_wait(tval) end end fname, events = wait(fm)::Pair - @test fname == F_PATH + @test fname == basename(file) @test events.changed && !events.timedout && !events.renamed close(fm) end diff --git a/stdlib/LinearAlgebra/docs/src/index.md b/stdlib/LinearAlgebra/docs/src/index.md index 138e505c79a464..da756f56e12b81 100644 --- a/stdlib/LinearAlgebra/docs/src/index.md +++ b/stdlib/LinearAlgebra/docs/src/index.md @@ -404,6 +404,35 @@ generally broadcasting over elements in the matrix representation fail because t be highly inefficient. For such use cases, consider computing the matrix representation up front and cache it for future reuse. +## [Pivoting Strategies](@id man-linalg-pivoting-strategies) + +Several of Julia's [matrix factorizations](@ref man-linalg-factorizations) support +[pivoting](https://en.wikipedia.org/wiki/Pivot_element), which can be used to improve their +numerical stability. In fact, some matrix factorizations, such as the LU +factorization, may fail without pivoting. + +In pivoting, first, a [pivot element](https://en.wikipedia.org/wiki/Pivot_element) +with good numerical properties is chosen based on a pivoting strategy. Next, the rows and +columns of the original matrix are permuted to bring the chosen element in place for +subsequent computation. Furthermore, the process is repeated for each stage of the factorization. + +Consequently, besides the conventional matrix factors, the outputs of +pivoted factorization schemes also include permutation matrices. + +In the following, the pivoting strategies implemented in Julia are briefly described. Note +that not all matrix factorizations may support them. Consult the documentation of the +respective [matrix factorization](@ref man-linalg-factorizations) for details on the +supported pivoting strategies. + +See also [`LinearAlgebra.ZeroPivotException`](@ref). + +```@docs +LinearAlgebra.NoPivot +LinearAlgebra.RowNonZero +LinearAlgebra.RowMaximum +LinearAlgebra.ColumnNorm +``` + ## Standard functions Linear algebra functions in Julia are largely implemented by calling functions from [LAPACK](https://www.netlib.org/lapack/). @@ -412,11 +441,14 @@ Other sparse solvers are available as Julia packages. ```@docs Base.:*(::AbstractMatrix, ::AbstractMatrix) +Base.:*(::AbstractMatrix, ::AbstractMatrix, ::AbstractVector) Base.:\(::AbstractMatrix, ::AbstractVecOrMat) Base.:/(::AbstractVecOrMat, ::AbstractVecOrMat) LinearAlgebra.SingularException LinearAlgebra.PosDefException LinearAlgebra.ZeroPivotException +LinearAlgebra.RankDeficientException +LinearAlgebra.LAPACKException LinearAlgebra.dot LinearAlgebra.dot(::Any, ::Any, ::Any) LinearAlgebra.cross @@ -570,6 +602,8 @@ LinearAlgebra.checksquare LinearAlgebra.peakflops LinearAlgebra.hermitianpart LinearAlgebra.hermitianpart! +LinearAlgebra.copy_adjoint! +LinearAlgebra.copy_transpose! ``` ## Low-level matrix operations @@ -761,7 +795,7 @@ LinearAlgebra.BLAS.trsm! LinearAlgebra.BLAS.trsm ``` -## LAPACK functions +## [LAPACK functions](@id man-linalg-lapack-functions) `LinearAlgebra.LAPACK` provides wrappers for some of the LAPACK functions for linear algebra. Those functions that overwrite one of the input arrays have names ending in `'!'`. diff --git a/stdlib/LinearAlgebra/src/LinearAlgebra.jl b/stdlib/LinearAlgebra/src/LinearAlgebra.jl index be46a700ab44c2..793775992fdea7 100644 --- a/stdlib/LinearAlgebra/src/LinearAlgebra.jl +++ b/stdlib/LinearAlgebra/src/LinearAlgebra.jl @@ -78,6 +78,7 @@ export cholesky, cond, condskeel, + copy_adjoint!, copy_transpose!, copyto!, copytrito!, @@ -190,10 +191,54 @@ abstract type Algorithm end struct DivideAndConquer <: Algorithm end struct QRIteration <: Algorithm end +# Pivoting strategies for matrix factorization algorithms. abstract type PivotingStrategy end + +""" + NoPivot + +Pivoting is not performed. Matrix factorizations such as the LU factorization +may fail without pivoting, and may also be numerically unstable for floating-point matrices in the face of roundoff error. +This pivot strategy is mainly useful for pedagogical purposes. +""" struct NoPivot <: PivotingStrategy end + +""" + RowNonZero + +First non-zero element in the remaining rows is chosen as the pivot element. + +Beware that for floating-point matrices, the resulting LU algorithm is numerically unstable — this strategy +is mainly useful for comparison to hand calculations (which typically use this strategy) or for other +algebraic types (e.g. rational numbers) not susceptible to roundoff errors. Otherwise, the default +`RowMaximum` pivoting strategy should be generally preferred in Gaussian elimination. + +Note that the [element type](@ref eltype) of the matrix must admit an [`iszero`](@ref) +method. +""" struct RowNonZero <: PivotingStrategy end + +""" + RowMaximum + +The maximum-magnitude element in the remaining rows is chosen as the pivot element. +This is the default strategy for LU factorization of floating-point matrices, and is sometimes +referred to as the "partial pivoting" algorithm. + +Note that the [element type](@ref eltype) of the matrix must admit an [`abs`](@ref) method, +whose result type must admit a [`<`](@ref) method. +""" struct RowMaximum <: PivotingStrategy end + +""" + ColumnNorm + +The column with the maximum norm is used for subsequent computation. This +is used for pivoted QR factorization. + +Note that the [element type](@ref eltype) of the matrix must admit [`norm`](@ref) and +[`abs`](@ref) methods, whose respective result types must admit a [`<`](@ref) method. +""" struct ColumnNorm <: PivotingStrategy end # Check that stride of matrix/vector is 1 diff --git a/stdlib/LinearAlgebra/src/exceptions.jl b/stdlib/LinearAlgebra/src/exceptions.jl index 574decf79fc079..7791b1ddef4161 100644 --- a/stdlib/LinearAlgebra/src/exceptions.jl +++ b/stdlib/LinearAlgebra/src/exceptions.jl @@ -6,6 +6,13 @@ export LAPACKException, RankDeficientException, ZeroPivotException +""" + LAPACKException + +Generic LAPACK exception thrown either during direct calls to the [LAPACK functions](@ref man-linalg-lapack-functions) +or during calls to other functions that use the LAPACK functions internally but lack specialized error handling. The `info` field +contains additional information on the underlying error and depends on the LAPACK function that was invoked. +""" struct LAPACKException <: Exception info::BlasInt end @@ -41,6 +48,13 @@ function Base.showerror(io::IO, ex::PosDefException) print(io, "; Factorization failed.") end +""" + RankDeficientException + +Exception thrown when the input matrix is [rank deficient](https://en.wikipedia.org/wiki/Rank_(linear_algebra)). Some +linear algebra functions, such as the Cholesky decomposition, are only applicable to matrices that are not rank +deficient. The `info` field indicates the computed rank of the matrix. +""" struct RankDeficientException <: Exception info::BlasInt end diff --git a/stdlib/LinearAlgebra/src/matmul.jl b/stdlib/LinearAlgebra/src/matmul.jl index c84a85ebc7c818..95a24b0d798eaf 100644 --- a/stdlib/LinearAlgebra/src/matmul.jl +++ b/stdlib/LinearAlgebra/src/matmul.jl @@ -682,19 +682,60 @@ end lapack_size(t::AbstractChar, M::AbstractVecOrMat) = (size(M, t=='N' ? 1 : 2), size(M, t=='N' ? 2 : 1)) +""" + copyto!(B::AbstractMatrix, ir_dest::AbstractUnitRange, jr_dest::AbstractUnitRange, + tM::AbstractChar, + M::AbstractVecOrMat, ir_src::AbstractUnitRange, jr_src::AbstractUnitRange) -> B + +Efficiently copy elements of matrix `M` to `B` conditioned on the character +parameter `tM` as follows: + +| `tM` | Destination | Source | +| --- | :--- | :--- | +| `'N'` | `B[ir_dest, jr_dest]` | `M[ir_src, jr_src]` | +| `'T'` | `B[ir_dest, jr_dest]` | `transpose(M)[ir_src, jr_src]` | +| `'C'` | `B[ir_dest, jr_dest]` | `adjoint(M)[ir_src, jr_src]` | + +The elements `B[ir_dest, jr_dest]` are overwritten. Furthermore, the index range +parameters must satisfy `length(ir_dest) == length(ir_src)` and +`length(jr_dest) == length(jr_src)`. + +See also [`copy_transpose!`](@ref) and [`copy_adjoint!`](@ref). +""" function copyto!(B::AbstractVecOrMat, ir_dest::AbstractUnitRange{Int}, jr_dest::AbstractUnitRange{Int}, tM::AbstractChar, M::AbstractVecOrMat, ir_src::AbstractUnitRange{Int}, jr_src::AbstractUnitRange{Int}) if tM == 'N' copyto!(B, ir_dest, jr_dest, M, ir_src, jr_src) + elseif tM == 'T' + copy_transpose!(B, ir_dest, jr_dest, M, jr_src, ir_src) else - LinearAlgebra.copy_transpose!(B, ir_dest, jr_dest, M, jr_src, ir_src) - tM == 'C' && conj!(@view B[ir_dest, jr_dest]) + copy_adjoint!(B, ir_dest, jr_dest, M, jr_src, ir_src) end B end +""" + copy_transpose!(B::AbstractMatrix, ir_dest::AbstractUnitRange, jr_dest::AbstractUnitRange, + tM::AbstractChar, + M::AbstractVecOrMat, ir_src::AbstractUnitRange, jr_src::AbstractUnitRange) -> B + +Efficiently copy elements of matrix `M` to `B` conditioned on the character +parameter `tM` as follows: + +| `tM` | Destination | Source | +| --- | :--- | :--- | +| `'N'` | `B[ir_dest, jr_dest]` | `transpose(M)[jr_src, ir_src]` | +| `'T'` | `B[ir_dest, jr_dest]` | `M[jr_src, ir_src]` | +| `'C'` | `B[ir_dest, jr_dest]` | `conj(M)[jr_src, ir_src]` | + +The elements `B[ir_dest, jr_dest]` are overwritten. Furthermore, the index +range parameters must satisfy `length(ir_dest) == length(jr_src)` and +`length(jr_dest) == length(ir_src)`. + +See also [`copyto!`](@ref) and [`copy_adjoint!`](@ref). +""" function copy_transpose!(B::AbstractMatrix, ir_dest::AbstractUnitRange{Int}, jr_dest::AbstractUnitRange{Int}, tM::AbstractChar, M::AbstractVecOrMat, ir_src::AbstractUnitRange{Int}, jr_src::AbstractUnitRange{Int}) if tM == 'N' - LinearAlgebra.copy_transpose!(B, ir_dest, jr_dest, M, ir_src, jr_src) + copy_transpose!(B, ir_dest, jr_dest, M, ir_src, jr_src) else copyto!(B, ir_dest, jr_dest, M, jr_src, ir_src) tM == 'C' && conj!(@view B[ir_dest, jr_dest]) diff --git a/stdlib/LinearAlgebra/src/transpose.jl b/stdlib/LinearAlgebra/src/transpose.jl index afc3494fc97260..8aa04f7d34b484 100644 --- a/stdlib/LinearAlgebra/src/transpose.jl +++ b/stdlib/LinearAlgebra/src/transpose.jl @@ -178,8 +178,41 @@ copy(::Union{Transpose,Adjoint}) Base.copy(A::TransposeAbsMat) = transpose!(similar(A.parent, reverse(axes(A.parent))), A.parent) Base.copy(A::AdjointAbsMat) = adjoint!(similar(A.parent, reverse(axes(A.parent))), A.parent) -function copy_transpose!(B::AbstractVecOrMat, ir_dest::AbstractRange{Int}, jr_dest::AbstractRange{Int}, - A::AbstractVecOrMat, ir_src::AbstractRange{Int}, jr_src::AbstractRange{Int}) +""" + copy_transpose!(B::AbstractVecOrMat, ir_dest::AbstractRange{Int}, jr_dest::AbstractRange{Int}, + A::AbstractVecOrMat, ir_src::AbstractRange{Int}, jr_src::AbstractRange{Int}) -> B + +Efficiently copy elements of matrix `A` to `B` with transposition as follows: + + B[ir_dest, jr_dest] = transpose(A)[jr_src, ir_src] + +The elements `B[ir_dest, jr_dest]` are overwritten. Furthermore, +the index range parameters must satisfy `length(ir_dest) == length(jr_src)` and +`length(jr_dest) == length(ir_src)`. +""" +copy_transpose!(B::AbstractVecOrMat, ir_dest::AbstractRange{Int}, jr_dest::AbstractRange{Int}, + A::AbstractVecOrMat, ir_src::AbstractRange{Int}, jr_src::AbstractRange{Int}) = + _copy_adjtrans!(B, ir_dest, jr_dest, A, ir_src, jr_src, transpose) + +""" + copy_adjoint!(B::AbstractVecOrMat, ir_dest::AbstractRange{Int}, jr_dest::AbstractRange{Int}, + A::AbstractVecOrMat, ir_src::AbstractRange{Int}, jr_src::AbstractRange{Int}) -> B + +Efficiently copy elements of matrix `A` to `B` with adjunction as follows: + + B[ir_dest, jr_dest] = adjoint(A)[jr_src, ir_src] + +The elements `B[ir_dest, jr_dest]` are overwritten. Furthermore, +the index range parameters must satisfy `length(ir_dest) == length(jr_src)` and +`length(jr_dest) == length(ir_src)`. +""" +copy_adjoint!(B::AbstractVecOrMat, ir_dest::AbstractRange{Int}, jr_dest::AbstractRange{Int}, + A::AbstractVecOrMat, ir_src::AbstractRange{Int}, jr_src::AbstractRange{Int}) = + _copy_adjtrans!(B, ir_dest, jr_dest, A, ir_src, jr_src, adjoint) + +function _copy_adjtrans!(B::AbstractVecOrMat, ir_dest::AbstractRange{Int}, jr_dest::AbstractRange{Int}, + A::AbstractVecOrMat, ir_src::AbstractRange{Int}, jr_src::AbstractRange{Int}, + tfun::T) where {T} if length(ir_dest) != length(jr_src) throw(ArgumentError(LazyString("source and destination must have same size (got ", length(jr_src)," and ",length(ir_dest),")"))) @@ -194,7 +227,7 @@ function copy_transpose!(B::AbstractVecOrMat, ir_dest::AbstractRange{Int}, jr_de for jsrc in jr_src jdest = first(jr_dest) for isrc in ir_src - B[idest,jdest] = transpose(A[isrc,jsrc]) + B[idest,jdest] = tfun(A[isrc,jsrc]) jdest += step(jr_dest) end idest += step(ir_dest) @@ -202,13 +235,10 @@ function copy_transpose!(B::AbstractVecOrMat, ir_dest::AbstractRange{Int}, jr_de return B end -function copy_similar(A::AdjointAbsMat, ::Type{T}) where {T} - C = similar(A, T, size(A)) - adjoint!(C, parent(A)) -end -function copy_similar(A::TransposeAbsMat, ::Type{T}) where {T} - C = similar(A, T, size(A)) - transpose!(C, parent(A)) +function copy_similar(A::AdjOrTransAbsMat, ::Type{T}) where {T} + Ap = parent(A) + f! = inplace_adj_or_trans(A) + return f!(similar(Ap, T, reverse(axes(Ap))), Ap) end function Base.copyto_unaliased!(deststyle::IndexStyle, dest::AbstractMatrix, srcstyle::IndexCartesian, src::AdjOrTransAbsMat) diff --git a/stdlib/LinearAlgebra/test/matmul.jl b/stdlib/LinearAlgebra/test/matmul.jl index 4b0181f079a6fc..5bcbb99a314fdd 100644 --- a/stdlib/LinearAlgebra/test/matmul.jl +++ b/stdlib/LinearAlgebra/test/matmul.jl @@ -873,6 +873,16 @@ end @test Xv1' * Xv3' ≈ XcXc end +@testset "copyto! for matrices of matrices" begin + A = [randn(ComplexF64, 2,3) for _ in 1:2, _ in 1:3] + for (tfun, tM) in ((identity, 'N'), (transpose, 'T'), (adjoint, 'C')) + At = copy(tfun(A)) + B = zero.(At) + copyto!(B, axes(B, 1), axes(B, 2), tM, A, axes(A, tM == 'N' ? 1 : 2), axes(A, tM == 'N' ? 2 : 1)) + @test B == At + end +end + @testset "method ambiguity" begin # Ambiguity test is run inside a clean process. # https://github.com/JuliaLang/julia/issues/28804 diff --git a/stdlib/LinearAlgebra/test/runtests.jl b/stdlib/LinearAlgebra/test/runtests.jl index 55bc119756ce83..d64da9899ca860 100644 --- a/stdlib/LinearAlgebra/test/runtests.jl +++ b/stdlib/LinearAlgebra/test/runtests.jl @@ -6,7 +6,5 @@ for file in readlines(joinpath(@__DIR__, "testgroups")) end @testset "Docstrings" begin - undoc = Docs.undocumented_names(LinearAlgebra) - @test_broken isempty(undoc) - @test undoc == [:ColumnNorm, :LAPACKException, :NoPivot, :RankDeficientException, :RowMaximum, :RowNonZero, :copy_transpose!] + @test isempty(Docs.undocumented_names(LinearAlgebra)) end diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index 01daef9d668990..9f1dd76b168af1 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -813,9 +813,9 @@ end # returns the removed portion as a String function edit_splice!(s::BufferLike, r::Region=region(s), ins::String = ""; rigid_mark::Bool=true) A, B = first(r), last(r) - A >= B && isempty(ins) && return String(ins) + A >= B && isempty(ins) && return ins buf = buffer(s) - pos = position(buf) + pos = position(buf) # n.b. position(), etc, are 0-indexed adjust_pos = true if A <= pos < B seek(buf, A) @@ -824,18 +824,29 @@ function edit_splice!(s::BufferLike, r::Region=region(s), ins::String = ""; rigi else adjust_pos = false end - if A < buf.mark < B || A == buf.mark == B - # rigid_mark is used only if the mark is strictly "inside" - # the region, or the region is empty and the mark is at the boundary - buf.mark = rigid_mark ? A : A + sizeof(ins) - elseif buf.mark >= B - buf.mark += sizeof(ins) - B + A - end - ensureroom(buf, B) # handle !buf.reinit from take! - ret = splice!(buf.data, A+1:B, codeunits(String(ins))) # position(), etc, are 0-indexed - buf.size = buf.size + sizeof(ins) - B + A - adjust_pos && seek(buf, position(buf) + sizeof(ins)) - return String(copy(ret)) + mark = buf.mark + if mark != -1 + if A < mark < B || A == mark == B + # rigid_mark is used only if the mark is strictly "inside" + # the region, or the region is empty and the mark is at the boundary + mark = rigid_mark ? A : A + sizeof(ins) + elseif mark >= B + mark += sizeof(ins) - B + A + end + buf.mark = -1 + end + # Implement ret = splice!(buf.data, A+1:B, codeunits(ins)) for a stream + pos = position(buf) + seek(buf, A) + ret = read(buf, A >= B ? 0 : B - A) + trail = read(buf) + seek(buf, A) + write(buf, ins) + write(buf, trail) + truncate(buf, position(buf)) + seek(buf, pos + (adjust_pos ? sizeof(ins) : 0)) + buf.mark = mark + return String(ret) end edit_splice!(s::MIState, ins::AbstractString) = edit_splice!(s, region(s), ins) diff --git a/stdlib/REPL/test/precompilation.jl b/stdlib/REPL/test/precompilation.jl index bf0c4e924a3c04..2dcf78c114d9a1 100644 --- a/stdlib/REPL/test/precompilation.jl +++ b/stdlib/REPL/test/precompilation.jl @@ -27,7 +27,7 @@ if !Sys.iswindows() tracecompile_out = read(f, String) close(ptm) # close after reading so we don't get precompiles from error shutdown - expected_precompiles = 0 + expected_precompiles = 1 n_precompiles = count(r"precompile\(", tracecompile_out) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 099ffc653e8d68..12b6b88d3f6f13 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -1950,3 +1950,60 @@ end @test repeat(f, 2, 3, 4) === FillArrays.Fill(3, (8, 6, 4)) @test repeat(f, inner=(1,2), outer=(3,1)) === FillArrays.Fill(3, (12, 4)) end + +@testset "zero" begin + @test zero([1 2; 3 4]) isa Matrix{Int} + @test zero([1 2; 3 4]) == [0 0; 0 0] + + @test zero([1.0]) isa Vector{Float64} + @test zero([1.0]) == [0.0] + + @test zero([[2,2], [3,3,3]]) isa Vector{Vector{Int}} + @test zero([[2,2], [3,3,3]]) == [[0,0], [0, 0, 0]] +end + +@testset "`_prechecked_iterate` optimization" begin + function test_prechecked_iterate(iter) + Js = Base._prechecked_iterate(iter) + for I in iter + J, s = Js::NTuple{2,Any} + @test J === I + Js = Base._prechecked_iterate(iter, s) + end + end + test_prechecked_iterate(1:10) + test_prechecked_iterate(Base.OneTo(10)) + test_prechecked_iterate(CartesianIndices((3, 3))) + test_prechecked_iterate(CartesianIndices(())) + test_prechecked_iterate(LinearIndices((3, 3))) + test_prechecked_iterate(LinearIndices(())) + test_prechecked_iterate(Base.SCartesianIndices2{3}(1:3)) +end + +@testset "IndexStyles in copyto!" begin + A = rand(3,2) + B = zeros(size(A)) + colons = ntuple(_->:, ndims(B)) + # Ensure that the AbstractArray methods are hit + # by using views instead of Arrays + @testset "IndexLinear - IndexLinear" begin + B .= 0 + copyto!(view(B, colons...), A) + @test B == A + end + @testset "IndexLinear - IndexCartesian" begin + B .= 0 + copyto!(view(B, colons...), view(A, axes(A)...)) + @test B == A + end + @testset "IndexCartesian - IndexLinear" begin + B .= 0 + copyto!(view(B, axes(B)...), A) + @test B == A + end + @testset "IndexCartesian - IndexCartesian" begin + B .= 0 + copyto!(view(B, axes(B)...), view(A, axes(A)...)) + @test B == A + end +end diff --git a/test/arrayops.jl b/test/arrayops.jl index 62df7550e90e59..a07e631e639f92 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -2612,7 +2612,7 @@ end end @testset "sign, conj[!], ~" begin - local A, B, C + local A, B, C, D, E A = [-10,0,3] B = [-10.0,0.0,3.0] C = [1,im,0] @@ -2629,6 +2629,11 @@ end @test typeof(conj(A)) == Vector{Int} @test typeof(conj(B)) == Vector{Float64} @test typeof(conj(C)) == Vector{Complex{Int}} + D = [C copy(C); copy(C) copy(C)] + @test conj(D) == conj!(copy(D)) + E = [D, copy(D)] + @test conj(E) == conj!(copy(E)) + @test (@allocations conj!(E)) == 0 @test .~A == [9,-1,-4] @test typeof(.~A) == Vector{Int} diff --git a/test/buildkitetestjson.jl b/test/buildkitetestjson.jl index 15bd8b8194a931..2751122e1a5fee 100644 --- a/test/buildkitetestjson.jl +++ b/test/buildkitetestjson.jl @@ -8,7 +8,7 @@ module BuildKiteTestJSON using Test using Dates -export write_testset_json +export write_testset_json_files # Bootleg JSON writer @@ -133,10 +133,17 @@ function collect_results!(results::Vector{Dict{String, Any}}, testset::Test.Defa results end -function write_testset_json(io::IO, testset::Test.DefaultTestSet) +function write_testset_json_files(dir::String, testset::Test.DefaultTestSet) data = Dict{String, Any}[] collect_results!(data, testset) - json_repr(io, data) + files = String[] + # Buildkite is limited to 5000 results per file https://buildkite.com/docs/test-analytics/importing-json + for (i, chunk) in enumerate(Iterators.partition(data, 5000)) + res_file = joinpath(dir, "results_$i.json") + open(io -> json_repr(io, chunk), res_file, "w") + push!(files, res_file) + end + return files end end diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 92bada23cb2582..434f6be0d8b059 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -1043,8 +1043,26 @@ end @test lines[3] == "foo" @test lines[4] == "bar" end -#heap-size-hint, we reserve 250 MB for non GC memory (llvm, etc.) -@test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=500M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$((500-250)*1024*1024)" +end + +@testset "heap size hint" begin + #heap-size-hint, we reserve 250 MB for non GC memory (llvm, etc.) + @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=500M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$((500-250)*1024*1024)" + + mem = ccall(:uv_get_total_memory, UInt64, ()) + cmem = ccall(:uv_get_constrained_memory, UInt64, ()) + if cmem > 0 && cmem < mem + mem = cmem + end + maxmem = parse(UInt64, readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=25% -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`)) + hint = max(mem÷4, 251*1024*1024) - 250*1024*1024 + MAX32HEAP = 1536 * 1024 * 1024 + if Int === Int32 && hint > MAX32HEAP + hint = MAX32HEAP + end + @test abs(Float64(maxmem) - hint)/maxmem < 0.05 + + @test readchomp(`$(Base.julia_cmd()) --startup-file=no --heap-size-hint=10M -e "println(@ccall jl_gc_get_max_memory()::UInt64)"`) == "$(1*1024*1024)" end ## `Main.main` entrypoint diff --git a/test/iobuffer.jl b/test/iobuffer.jl index ec77903b4a5b85..6151f90f297ee5 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -120,6 +120,7 @@ end Base.compact(io) @test position(io) == 0 @test ioslength(io) == 0 + Base._resize!(io,0) Base.ensureroom(io,50) @test position(io) == 0 @test ioslength(io) == 0 diff --git a/test/iterators.jl b/test/iterators.jl index 6fd308a31d746e..3616a17b31d31d 100644 --- a/test/iterators.jl +++ b/test/iterators.jl @@ -254,6 +254,22 @@ let i = 0 @test !Base.isdone(cycle(0:3), 1) end +@testset "cycle(iter, n)" begin + @test collect(cycle(0:3, 2)) == [0, 1, 2, 3, 0, 1, 2, 3] + @test collect(cycle(Iterators.filter(iseven, 1:4), 2)) == [2, 4, 2, 4] + @test collect(take(cycle(countfrom(11), 3), 4)) == 11:14 + + @test isempty(cycle(1:0)) == isempty(cycle(1:0, 3)) == true + @test isempty(cycle(1:5, 0)) + @test isempty(cycle(Iterators.filter(iseven, 1:4), 0)) + + @test eltype(cycle(0:3, 2)) === Int + @test Base.IteratorEltype(cycle(0:3, 2)) == Base.HasEltype() + + Base.haslength(cycle(0:3, 2)) == false # but not sure we should test these + Base.IteratorSize(cycle(0:3, 2)) == Base.SizeUnknown() +end + # repeated # -------- let i = 0 diff --git a/test/loading.jl b/test/loading.jl index 3740babcd17ed8..53e05a4a76bbce 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1131,6 +1131,18 @@ end cmd = `$(Base.julia_cmd()) --startup-file=no -e $sysimg_ext_test_code` cmd = addenv(cmd, "JULIA_LOAD_PATH" => join([proj, "@stdlib"], sep)) run(cmd) + + + # Extensions in implicit environments + old_load_path = copy(LOAD_PATH) + try + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "Extensions", "ImplicitEnv")) + pkgid_B = Base.PkgId(Base.uuid5(Base.identify_package("A").uuid, "BExt"), "BExt") + @test Base.identify_package(pkgid_B, "B") isa Base.PkgId + finally + copy!(LOAD_PATH, old_load_path) + end finally try rm(depot_path, force=true, recursive=true) @@ -1329,11 +1341,11 @@ end mkpath(project_path) # Create fake `Foo.jl` package with two files: - foo_path = joinpath(depot, "dev", "Foo") + foo_path = joinpath(depot, "dev", "Foo51989") mkpath(joinpath(foo_path, "src")) - open(joinpath(foo_path, "src", "Foo.jl"); write=true) do io + open(joinpath(foo_path, "src", "Foo51989.jl"); write=true) do io println(io, """ - module Foo + module Foo51989 include("internal.jl") end """) @@ -1343,7 +1355,7 @@ end end open(joinpath(foo_path, "Project.toml"); write=true) do io println(io, """ - name = "Foo" + name = "Foo51989" uuid = "00000000-0000-0000-0000-000000000001" version = "1.0.0" """) @@ -1351,27 +1363,27 @@ end # In our depot, `dev` and then `precompile` this `Foo` package. @test success(addenv( - `$(Base.julia_cmd()) --project=$project_path --startup-file=no -e 'import Pkg; Pkg.develop("Foo"); Pkg.precompile(); exit(0)'`, + `$(Base.julia_cmd()) --project=$project_path --startup-file=no -e 'import Pkg; Pkg.develop("Foo51989"); Pkg.precompile(); exit(0)'`, "JULIA_DEPOT_PATH" => depot)) # Get the size of the generated `.ji` file so that we can ensure that it gets altered - foo_compiled_path = joinpath(depot, "compiled", "v$(VERSION.major).$(VERSION.minor)", "Foo") + foo_compiled_path = joinpath(depot, "compiled", "v$(VERSION.major).$(VERSION.minor)", "Foo51989") cache_path = joinpath(foo_compiled_path, only(filter(endswith(".ji"), readdir(foo_compiled_path)))) cache_size = filesize(cache_path) # Next, remove the dependence on `internal.jl` and delete it: rm(joinpath(foo_path, "src", "internal.jl")) - open(joinpath(foo_path, "src", "Foo.jl"); write=true) do io + open(joinpath(foo_path, "src", "Foo51989.jl"); write=true) do io truncate(io, 0) println(io, """ - module Foo + module Foo51989 end """) end # Try to load `Foo`; this should trigger recompilation, not an error! @test success(addenv( - `$(Base.julia_cmd()) --project=$project_path --startup-file=no -e 'using Foo; exit(0)'`, + `$(Base.julia_cmd()) --project=$project_path --startup-file=no -e 'using Foo51989; exit(0)'`, "JULIA_DEPOT_PATH" => depot, )) diff --git a/test/misc.jl b/test/misc.jl index d2a8fabd119c5a..6597ecf8a84289 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -129,6 +129,36 @@ let l = ReentrantLock() @test_throws ErrorException unlock(l) end +# Lockable{T, L<:AbstractLock} +using Base: Lockable +let + @test_broken Base.isexported(Base, :Lockable) + lockable = Lockable(Dict("foo" => "hello"), ReentrantLock()) + # note field access is non-public + @test lockable.value["foo"] == "hello" + @test @lock(lockable, lockable[]["foo"]) == "hello" + lock(lockable) do d + @test d["foo"] == "hello" + end + lock(lockable) do d + d["foo"] = "goodbye" + end + @test lockable.value["foo"] == "goodbye" + @lock lockable begin + @test lockable[]["foo"] == "goodbye" + end + l = trylock(lockable) + try + @test l + finally + unlock(lockable) + end + # Test 1-arg constructor + lockable2 = Lockable(Dict("foo" => "hello")) + @test lockable2.lock isa ReentrantLock + @test @lock(lockable2, lockable2[]["foo"]) == "hello" +end + for l in (Threads.SpinLock(), ReentrantLock()) @test get_finalizers_inhibited() == 0 @test lock(get_finalizers_inhibited, l) == 1 diff --git a/test/offsetarray.jl b/test/offsetarray.jl index e2924ac0a8ca4f..9d6a8b08c0b1f7 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -864,3 +864,8 @@ end # this is fixed in #40038, so the evaluation of its CartesianIndices should work @test CartesianIndices(A) == CartesianIndices(B) end + +@testset "indexing views (#53249)" begin + v = view([1,2,3,4], :) + @test v[Base.IdentityUnitRange(2:3)] == OffsetArray(2:3, 2:3) +end diff --git a/test/precompile.jl b/test/precompile.jl index 90086ff7b7385a..129f909dcf6e21 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1986,6 +1986,23 @@ precompile_test_harness("Issue #50538") do load_path @test !isdefined(I50538, :undefglobal) end +precompile_test_harness("Test flags") do load_path + write(joinpath(load_path, "TestFlags.jl"), + """ + module TestFlags + end + """) + ji, ofile = Base.compilecache(Base.PkgId("TestFlags"); flags=`--check-bounds=no -O3`) + @show ji, ofile + open(ji, "r") do io + Base.isvalid_cache_header(io) + _, _, _, _, _, _, _, flags = Base.parse_cache_header(io, ji) + cacheflags = Base.CacheFlags(flags) + @test cacheflags.check_bounds == 2 + @test cacheflags.opt_level == 3 + end +end + empty!(Base.DEPOT_PATH) append!(Base.DEPOT_PATH, original_depot_path) empty!(Base.LOAD_PATH) diff --git a/test/project/Extensions/ImplicitEnv/A/Project.toml b/test/project/Extensions/ImplicitEnv/A/Project.toml new file mode 100644 index 00000000000000..043272d4bd0154 --- /dev/null +++ b/test/project/Extensions/ImplicitEnv/A/Project.toml @@ -0,0 +1,9 @@ +name = "A" +uuid = "299a509a-2181-4868-8714-15151945d902" +version = "0.1.0" + +[weakdeps] +B = "c2c18cb0-3543-497c-ac2a-523c527589e5" + +[extensions] +BExt = "B" diff --git a/test/project/Extensions/ImplicitEnv/A/ext/BExt.jl b/test/project/Extensions/ImplicitEnv/A/ext/BExt.jl new file mode 100644 index 00000000000000..70be6435bcbe8b --- /dev/null +++ b/test/project/Extensions/ImplicitEnv/A/ext/BExt.jl @@ -0,0 +1,3 @@ +module BExt + +end diff --git a/test/project/Extensions/ImplicitEnv/A/src/A.jl b/test/project/Extensions/ImplicitEnv/A/src/A.jl new file mode 100644 index 00000000000000..ab16fa1de96afd --- /dev/null +++ b/test/project/Extensions/ImplicitEnv/A/src/A.jl @@ -0,0 +1,5 @@ +module A + +greet() = print("Hello World!") + +end # module A diff --git a/test/project/Extensions/ImplicitEnv/B/Project.toml b/test/project/Extensions/ImplicitEnv/B/Project.toml new file mode 100644 index 00000000000000..d919c27be0467c --- /dev/null +++ b/test/project/Extensions/ImplicitEnv/B/Project.toml @@ -0,0 +1,3 @@ +name = "B" +uuid = "c2c18cb0-3543-497c-ac2a-523c527589e5" +version = "0.1.0" diff --git a/test/project/Extensions/ImplicitEnv/B/src/B.jl b/test/project/Extensions/ImplicitEnv/B/src/B.jl new file mode 100644 index 00000000000000..79b5a1204765ff --- /dev/null +++ b/test/project/Extensions/ImplicitEnv/B/src/B.jl @@ -0,0 +1,5 @@ +module B + +greet() = print("Hello World!") + +end # module B diff --git a/test/runtests.jl b/test/runtests.jl index 0f7a08446e014e..7c6fa8480d98e0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -429,9 +429,8 @@ cd(@__DIR__) do end if Base.get_bool_env("CI", false) - testresults = joinpath(@__DIR__, "results.json") - @info "Writing test result data to $testresults" - open(io -> write_testset_json(io, o_ts), testresults, "w") + @info "Writing test result data to $(@__DIR__)" + write_testset_json_files(@__DIR__, o_ts) end Test.TESTSET_PRINT_ENABLE[] = true diff --git a/test/sets.jl b/test/sets.jl index c490e66793cb44..4ab360c9fedd42 100644 --- a/test/sets.jl +++ b/test/sets.jl @@ -642,21 +642,38 @@ end @test !allunique((1,2,3,4,3)) @test allunique((0.0, -0.0)) @test !allunique((NaN, NaN)) + # Known length 1, need not evaluate: + @test allunique(error(x) for x in [1]) +end + +@testset "allunique(f, xs)" begin + @test allunique(sin, 1:3) + @test !allunique(sin, [1,2,3,1]) + @test allunique(sin, (1, 2, pi, im)) # eltype Any + @test allunique(abs2, 1:100) + @test !allunique(abs, -10:10) + @test allunique(abs2, Vector{Any}(1:100)) + # These cases don't call the function at all: + @test allunique(error, []) + @test allunique(error, [1]) end @testset "allequal" begin + # sets & dictionaries @test allequal(Set()) @test allequal(Set(1)) @test !allequal(Set([1, 2])) @test allequal(Dict()) @test allequal(Dict(:a => 1)) @test !allequal(Dict(:a => 1, :b => 2)) + # vectors @test allequal([]) @test allequal([1]) @test allequal([1, 1]) @test !allequal([1, 1, 2]) @test allequal([:a, :a]) @test !allequal([:a, :b]) + # ranges @test !allequal(1:2) @test allequal(1:1) @test !allequal(4.0:0.3:7.0) @@ -670,6 +687,26 @@ end @test allequal(LinRange(1, 1, 1)) @test allequal(LinRange(1, 1, 2)) @test !allequal(LinRange(1, 2, 2)) + # Known length 1, need not evaluate: + @test allequal(error(x) for x in [1]) + # Empty, but !haslength: + @test allequal(error(x) for x in 1:3 if false) +end + +@testset "allequal(f, xs)" begin + @test allequal(abs2, [3, -3]) + @test allequal(x -> 1, rand(3)) + @test !allequal(x -> rand(), [1,1,1]) + # tuples + @test allequal(abs2, (3, -3)) + @test allequal(x -> 1, Tuple(rand(3))) + @test !allequal(x -> rand(), (1,1,1)) + # These cases don't call the function at all: + @test allequal(error, []) + @test allequal(error, ()) + @test allequal(error, (x for x in 1:3 if false)) + @test allequal(error, [1]) + @test allequal(error, (1,)) end @testset "filter(f, ::$S)" for S = (Set, BitSet) diff --git a/test/show.jl b/test/show.jl index 3e67155d0acb7a..d6a691029d60a8 100644 --- a/test/show.jl +++ b/test/show.jl @@ -1250,12 +1250,12 @@ end @testset "PR 17117: print_array" begin s = IOBuffer(Vector{UInt8}(), read=true, write=true) Base.print_array(s, [1, 2, 3]) - @test String(resize!(s.data, s.size)) == " 1\n 2\n 3" + @test String(take!(s)) == " 1\n 2\n 3" close(s) s2 = IOBuffer(Vector{UInt8}(), read=true, write=true) z = zeros(0,0,0,0,0,0,0,0) Base.print_array(s2, z) - @test String(resize!(s2.data, s2.size)) == "" + @test String(take!(s2)) == "" close(s2) end diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 74898e09eb0088..9d812deea51f8a 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -91,7 +91,7 @@ end (6:11, :other => 0x02)]) str1 = Base.AnnotatedString("test", [(1:4, :label => 5)]) str2 = Base.AnnotatedString("case", [(2:3, :label => "oomph")]) - @test join([str1, str1], Base.AnnotatedString(" ")) == + @test join([str1, str1], ' ') == Base.AnnotatedString("test test", [(1:4, :label => 5), (6:9, :label => 5)]) @@ -122,7 +122,7 @@ end @test read(seek(aio, 1), Base.AnnotatedString) == Base.AnnotatedString("ello world", [(1:4, :tag => 1), (6:10, :tag => 2)]) @test read(seek(aio, 4), Base.AnnotatedString) == Base.AnnotatedString("o world", [(1:1, :tag => 1), (3:7, :tag => 2)]) @test read(seek(aio, 5), Base.AnnotatedString) == Base.AnnotatedString(" world", [(2:6, :tag => 2)]) - @test read(aio, Base.AnnotatedString) == Base.AnnotatedString("") + @test read(seekend(aio), Base.AnnotatedString) == Base.AnnotatedString("") @test read(seekstart(truncate(deepcopy(aio), 5)), Base.AnnotatedString) == Base.AnnotatedString("hello", [(1:5, :tag => 1)]) @test read(seekstart(truncate(deepcopy(aio), 6)), Base.AnnotatedString) == Base.AnnotatedString("hello ", [(1:5, :tag => 1)]) @test read(seekstart(truncate(deepcopy(aio), 7)), Base.AnnotatedString) == Base.AnnotatedString("hello w", [(1:5, :tag => 1), (7:7, :tag => 2)])