From 2e9b0bbf4e8301cd186d945117d78f1bbdbc458d Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 26 Feb 2024 21:42:36 -0500 Subject: [PATCH 01/33] `rm`: move open DLLs to temp dir to allow dir to be deleted (#53456) --- base/file.jl | 40 ++++++++++++++++++++++++++-------------- base/loading.jl | 45 +++++++++++++++++++++++++-------------------- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/base/file.jl b/base/file.jl index 6156bafacdc47..157951c6bc5bf 100644 --- a/base/file.jl +++ b/base/file.jl @@ -249,6 +249,10 @@ function mkpath(path::AbstractString; mode::Integer = 0o777) path end +# Files that were requested to be deleted but can't be by the current process +# i.e. loaded DLLs on Windows +delayed_delete_dir() = joinpath(tempdir(), "julia_delayed_deletes") + """ rm(path::AbstractString; force::Bool=false, recursive::Bool=false) @@ -270,20 +274,26 @@ Stacktrace: [...] ``` """ -function rm(path::AbstractString; force::Bool=false, recursive::Bool=false) +function rm(path::AbstractString; force::Bool=false, recursive::Bool=false, allow_delayed_delete::Bool=true) + # allow_delayed_delete is used by Pkg.gc() but is otherwise not part of the public API if islink(path) || !isdir(path) try - @static if Sys.iswindows() - # is writable on windows actually means "is deletable" - st = lstat(path) - if ispath(st) && (filemode(st) & 0o222) == 0 - chmod(path, 0o777) - end - end unlink(path) catch err - if force && isa(err, IOError) && err.code==Base.UV_ENOENT - return + if isa(err, IOError) + force && err.code==Base.UV_ENOENT && return + @static if Sys.iswindows() + if allow_delayed_delete && err.code==Base.UV_EACCES && endswith(path, ".dll") + # Loaded DLLs cannot be deleted on Windows, even with posix delete mode + # but they can be moved. So move out to allow the dir to be deleted + # TODO: Add a mechanism to delete these moved files after dlclose or process exit + dir = mkpath(delayed_delete_dir()) + temp_path = tempname(dir, cleanup = false, suffix = string("_", basename(path))) + @debug "Could not delete DLL most likely because it is loaded, moving to tempdir" path temp_path + mv(path, temp_path) + return + end + end end rethrow() end @@ -291,12 +301,14 @@ function rm(path::AbstractString; force::Bool=false, recursive::Bool=false) if recursive try for p in readdir(path) - rm(joinpath(path, p), force=force, recursive=true) + try + rm(joinpath(path, p), force=force, recursive=true) + catch err + (isa(err, IOError) && err.code==Base.UV_EACCES) || rethrow() + end end catch err - if !(isa(err, IOError) && err.code==Base.UV_EACCES) - rethrow(err) - end + (isa(err, IOError) && err.code==Base.UV_EACCES) || rethrow() end end req = Libc.malloc(_sizeof_uv_fs) diff --git a/base/loading.jl b/base/loading.jl index 1cff14203b291..e055eb3003c9e 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2902,26 +2902,10 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in end if cache_objects - try - rename(tmppath_so, ocachefile::String; force=true) - catch e - e isa IOError || rethrow() - isfile(ocachefile::String) || rethrow() - # Windows prevents renaming a file that is in use so if there is a Julia session started - # with a package image loaded, we cannot rename that file. - # The code belows append a `_i` to the name of the cache file where `i` is the smallest number such that - # that cache file does not exist. - ocachename, ocacheext = splitext(ocachefile::String) - old_cachefiles = Set(readdir(cachepath)) - num = 1 - while true - ocachefile = ocachename * "_$num" * ocacheext - in(basename(ocachefile), old_cachefiles) || break - num += 1 - end - # TODO: Risk for a race here if some other process grabs this name before us - cachefile = cachefile_from_ocachefile(ocachefile) - rename(tmppath_so, ocachefile::String; force=true) + ocachefile_new = rename_unique_ocachefile(tmppath_so, ocachefile) + if ocachefile_new != ocachefile + cachefile = cachefile_from_ocachefile(ocachefile_new) + ocachefile = ocachefile_new end @static if Sys.isapple() run(`$(Linking.dsymutil()) $ocachefile`, Base.DevNull(), Base.DevNull(), Base.DevNull()) @@ -2945,6 +2929,27 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in end end +function rename_unique_ocachefile(tmppath_so::String, ocachefile_orig::String, ocachefile::String = ocachefile_orig, num = 0) + try + rename(tmppath_so, ocachefile; force=true) + catch e + e isa IOError || rethrow() + # If `rm` was called on a dir containing a loaded DLL, we moved it to temp for cleanup + # on restart. However the old path cannot be used (UV_EACCES) while the DLL is loaded + if !isfile(ocachefile) && e.code != Base.UV_EACCES + rethrow() + end + # Windows prevents renaming a file that is in use so if there is a Julia session started + # with a package image loaded, we cannot rename that file. + # The code belows append a `_i` to the name of the cache file where `i` is the smallest number such that + # that cache file does not exist. + ocachename, ocacheext = splitext(ocachefile_orig) + ocachefile_unique = ocachename * "_$num" * ocacheext + ocachefile = rename_unique_ocachefile(tmppath_so, ocachefile_orig, ocachefile_unique, num + 1) + end + return ocachefile +end + function module_build_id(m::Module) hi, lo = ccall(:jl_module_build_id, NTuple{2,UInt64}, (Any,), m) return (UInt128(hi) << 64) | lo From 35cb8a556b1efd12b7052e48412590fe67bef3bf Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:12:46 +0900 Subject: [PATCH 02/33] minor fixes on test/precompile.jl (#53476) These changes are driven-by fixes I found during investigating into a more complex issue related to precompilation with external abs int. --- test/precompile.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/precompile.jl b/test/precompile.jl index ce7e9e77217b0..c06145cba8416 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1723,7 +1723,6 @@ let newinterp_path = abspath("compiler/newinterp.jl") import SimpleModule: basic_caller, basic_callee module Custom - const CC = Core.Compiler include("$($newinterp_path)") @newinterp PrecompileInterpreter end @@ -1826,7 +1825,7 @@ let newinterp_path = abspath("compiler/newinterp.jl") using CustomAbstractInterpreterCaching2 cache_owner = Core.Compiler.cache_owner( CustomAbstractInterpreterCaching2.Custom.PrecompileInterpreter()) - let m = only(methods(CustomAbstractInterpreterCaching.basic_callee)) + let m = only(methods(CustomAbstractInterpreterCaching2.basic_callee)) mi = only(Base.specializations(m)) ci = mi.cache @test isdefined(ci, :next) From b3b27360672f87af78e90f43737e32988d9b8be1 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 27 Feb 2024 08:57:49 -0500 Subject: [PATCH 03/33] add IR encoding for EnterNode (#53482) fixes #53248 --- src/ircode.c | 10 ++++++++++ src/serialize.h | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ircode.c b/src/ircode.c index 90dab5f63d494..a89d0140e0d88 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -322,6 +322,11 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) jl_encode_value(s, jl_get_nth_field(v, 0)); jl_encode_value(s, jl_get_nth_field(v, 1)); } + else if (jl_is_enternode(v)) { + write_uint8(s->s, TAG_ENTERNODE); + jl_encode_value(s, jl_get_nth_field(v, 0)); + jl_encode_value(s, jl_get_nth_field(v, 1)); + } else if (jl_is_argument(v)) { write_uint8(s->s, TAG_ARGUMENT); jl_encode_value(s, jl_get_nth_field(v, 0)); @@ -721,6 +726,11 @@ static jl_value_t *jl_decode_value(jl_ircode_state *s) JL_GC_DISABLED set_nth_field(jl_gotoifnot_type, v, 0, jl_decode_value(s), 0); set_nth_field(jl_gotoifnot_type, v, 1, jl_decode_value(s), 0); return v; + case TAG_ENTERNODE: + v = jl_new_struct_uninit(jl_enternode_type); + set_nth_field(jl_enternode_type, v, 0, jl_decode_value(s), 0); + set_nth_field(jl_enternode_type, v, 1, jl_decode_value(s), 0); + return v; case TAG_ARGUMENT: v = jl_new_struct_uninit(jl_argument_type); set_nth_field(jl_argument_type, v, 0, jl_decode_value(s), 0); diff --git a/src/serialize.h b/src/serialize.h index 1bd29e9cc5911..2a91189dce739 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -65,8 +65,9 @@ extern "C" { #define TAG_RELOC_METHODROOT 57 #define TAG_BINDING 58 #define TAG_MEMORYT 59 +#define TAG_ENTERNODE 60 -#define LAST_TAG 59 +#define LAST_TAG 60 #define write_uint8(s, n) ios_putc((n), (s)) #define read_uint8(s) ((uint8_t)ios_getc((s))) From c19c68e83ac372ebbe53d21c3c19d61f0dbc7e0a Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Tue, 27 Feb 2024 11:17:15 -0500 Subject: [PATCH 04/33] Add error hint for incorrect stacked indexing (#40934) A common entry level mistake is to try to index a matrix with two sets of brackets, e.g. `a = [1 2; 3 4]; a[1][2] = 5` This will lead to an error that `setindex!()` on the element type of `a` is missing. This PR adds an error hint for the case where a MethodError is raised when `setindex!` is called with a `Number` as the first argument. I considered going broader than numbers, but it seems more likely that this kind of mistake would happen when working with simple number arrays vs. something more advanced. Could also consider if it is possible to do the same for when `getindex()` is called on a `Number`, which emits a BoundsError. Co-authored-by: Michael Abbott <32575566+mcabbott@users.noreply.github.com> --- base/errorshow.jl | 27 +++++++++++++++++++++++++-- test/errorshow.jl | 18 ++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/base/errorshow.jl b/base/errorshow.jl index 9236e7da5baa6..1e6a6faaae5df 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -1022,6 +1022,31 @@ end Experimental.register_error_hint(noncallable_number_hint_handler, MethodError) +# handler for displaying a hint in case the user tries to call setindex! on +# something that doesn't support it: +# - a number (probably attempting to use wrong indexing) +# eg: a = [1 2; 3 4]; a[1][2] = 5 +# - a type (probably tried to initialize without parentheses) +# eg: d = Dict; d["key"] = 2 +function nonsetable_type_hint_handler(io, ex, arg_types, kwargs) + @nospecialize + if ex.f == setindex! + T = arg_types[1] + if T <: Number + print(io, "\nAre you trying to index into an array? For multi-dimensional arrays, separate the indices with commas: ") + printstyled(io, "a[1, 2]", color=:cyan) + print(io, " rather than a[1][2]") + else isType(T) + Tx = T.parameters[1] + print(io, "\nYou attempted to index the type $Tx, rather than an instance of the type. Make sure you create the type using its constructor: ") + printstyled(io, "d = $Tx([...])", color=:cyan) + print(io, " rather than d = $Tx") + end + end +end + +Experimental.register_error_hint(nonsetable_type_hint_handler, MethodError) + # Display a hint in case the user tries to use the + operator on strings # (probably attempting concatenation) function string_concatenation_hint_handler(io, ex, arg_types, kwargs) @@ -1035,7 +1060,6 @@ end Experimental.register_error_hint(string_concatenation_hint_handler, MethodError) - # Display a hint in case the user tries to use the min or max function on an iterable # or tries to use something like `collect` on an iterator without defining either IteratorSize or length function methods_on_iterable(io, ex, arg_types, kwargs) @@ -1061,7 +1085,6 @@ end Experimental.register_error_hint(methods_on_iterable, MethodError) - # ExceptionStack implementation size(s::ExceptionStack) = size(s.stack) getindex(s::ExceptionStack, i::Int) = s.stack[i] diff --git a/test/errorshow.jl b/test/errorshow.jl index 2b497ed3f5f1b..d03de54aab496 100644 --- a/test/errorshow.jl +++ b/test/errorshow.jl @@ -9,6 +9,8 @@ include("testenv.jl") Base.Experimental.register_error_hint(Base.noncallable_number_hint_handler, MethodError) Base.Experimental.register_error_hint(Base.string_concatenation_hint_handler, MethodError) Base.Experimental.register_error_hint(Base.methods_on_iterable, MethodError) +Base.Experimental.register_error_hint(Base.nonsetable_type_hint_handler, MethodError) + @testset "SystemError" begin err = try; systemerror("reason", Cint(0)); false; catch ex; ex; end::SystemError @@ -745,6 +747,22 @@ let err_str @test count(==("Maybe you forgot to use an operator such as *, ^, %, / etc. ?"), split(err_str, '\n')) == 1 end +let err_str + a = [1 2; 3 4]; + err_str = @except_str (a[1][2] = 5) MethodError + @test occursin("\nAre you trying to index into an array? For multi-dimensional arrays, separate the indices with commas: ", err_str) + @test occursin("a[1, 2]", err_str) + @test occursin("rather than a[1][2]", err_str) +end + +let err_str + d = Dict + err_str = @except_str (d[1] = 5) MethodError + @test occursin("\nYou attempted to index the type Dict, rather than an instance of the type. Make sure you create the type using its constructor: ", err_str) + @test occursin("d = Dict([...])", err_str) + @test occursin(" rather than d = Dict", err_str) +end + # Execute backtrace once before checking formatting, see #38858 backtrace() From 2b79326037ac4092733b8eb091d94f0c41bc92a2 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 27 Feb 2024 12:22:16 -0500 Subject: [PATCH 05/33] Move the list of Base's `public` names from `base/exports.jl` to `base/public.jl` (#53487) --- base/Base.jl | 1 + base/exports.jl | 100 ----------------------------------------------- base/public.jl | 101 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 100 deletions(-) create mode 100644 base/public.jl diff --git a/base/Base.jl b/base/Base.jl index e02b132aae6e4..30bc0b91411b8 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -137,6 +137,7 @@ if isdefined(Core, :Compiler) end include("exports.jl") +include("public.jl") if false # simple print definitions for debugging. enable these if something diff --git a/base/exports.jl b/base/exports.jl index a31ea1b0fa842..7f3ad451bcc83 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -1078,103 +1078,3 @@ export @static, @main - -public -# Modules - Checked, - Filesystem, - Order, - Sort, - -# Types - AbstractLock, - AsyncCondition, - CodeUnits, - Event, - Fix1, - Fix2, - Generator, - ImmutableDict, - OneTo, - LogRange, - AnnotatedString, - AnnotatedChar, - UUID, - -# Annotated strings - annotatedstring, - annotate!, - annotations, - -# Semaphores - Semaphore, - acquire, - release, - -# collections - IteratorEltype, - IteratorSize, - to_index, - vect, - isdone, - front, - rest, - split_rest, - tail, - checked_length, - -# Loading - DL_LOAD_PATH, - load_path, - active_project, - -# Reflection and introspection - isambiguous, - isexpr, - isidentifier, - issingletontype, - identify_package, - locate_package, - moduleroot, - jit_total_bytes, - summarysize, - isexported, - ispublic, - remove_linenums!, - -# Opperators - operator_associativity, - operator_precedence, - isbinaryoperator, - isoperator, - isunaryoperator, - -# C interface - cconvert, - unsafe_convert, - -# Error handling - exit_on_sigint, - windowserror, - -# Macros - @assume_effects, - @constprop, - @locals, - @propagate_inbounds, - -# IO - # types - BufferStream, - IOServer, - OS_HANDLE, - PipeEndpoint, - TTY, - # functions - reseteof, - link_pipe!, - -# misc - notnothing, - runtests, - text_colors diff --git a/base/public.jl b/base/public.jl new file mode 100644 index 0000000000000..912953795c801 --- /dev/null +++ b/base/public.jl @@ -0,0 +1,101 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +public +# Modules + Checked, + Filesystem, + Order, + Sort, + +# Types + AbstractLock, + AsyncCondition, + CodeUnits, + Event, + Fix1, + Fix2, + Generator, + ImmutableDict, + OneTo, + LogRange, + AnnotatedString, + AnnotatedChar, + UUID, + +# Annotated strings + annotatedstring, + annotate!, + annotations, + +# Semaphores + Semaphore, + acquire, + release, + +# collections + IteratorEltype, + IteratorSize, + to_index, + vect, + isdone, + front, + rest, + split_rest, + tail, + checked_length, + +# Loading + DL_LOAD_PATH, + load_path, + active_project, + +# Reflection and introspection + isambiguous, + isexpr, + isidentifier, + issingletontype, + identify_package, + locate_package, + moduleroot, + jit_total_bytes, + summarysize, + isexported, + ispublic, + remove_linenums!, + +# Opperators + operator_associativity, + operator_precedence, + isbinaryoperator, + isoperator, + isunaryoperator, + +# C interface + cconvert, + unsafe_convert, + +# Error handling + exit_on_sigint, + windowserror, + +# Macros + @assume_effects, + @constprop, + @locals, + @propagate_inbounds, + +# IO + # types + BufferStream, + IOServer, + OS_HANDLE, + PipeEndpoint, + TTY, + # functions + reseteof, + link_pipe!, + +# misc + notnothing, + runtests, + text_colors From b18a62d624d4be2f0b30f6d7245a01bdc7c13d09 Mon Sep 17 00:00:00 2001 From: pyfagorass Date: Tue, 27 Feb 2024 21:54:46 +0200 Subject: [PATCH 06/33] Fix formatting & typo in methods.md (#53486) * Fixed what I suspect is a typo: commonly -> common * Added code quotes to eltype: eltype -> `eltype` Sentence now reads: Instead, common code will dispatch first on the container type, then recurse down to a more specific method based on `eltype`. --- doc/src/manual/methods.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/manual/methods.md b/doc/src/manual/methods.md index 6b90990a6dd7c..810f81f3e9c8f 100644 --- a/doc/src/manual/methods.md +++ b/doc/src/manual/methods.md @@ -745,8 +745,8 @@ often it is best to separate each level of dispatch into distinct functions. This may sound similar in approach to single-dispatch, but as we shall see below, it is still more flexible. For example, trying to dispatch on the element-type of an array will often run into ambiguous situations. -Instead, commonly code will dispatch first on the container type, -then recurse down to a more specific method based on eltype. +Instead, common code will dispatch first on the container type, +then recurse down to a more specific method based on `eltype`. In most cases, the algorithms lend themselves conveniently to this hierarchical approach, while in other cases, this rigor must be resolved manually. This dispatching branching can be observed, for example, in the logic to sum two matrices: From c379db77135e55b71707704f58d901e78924804d Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:33:57 -0500 Subject: [PATCH 07/33] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20Pk?= =?UTF-8?q?g=20stdlib=20from=2076070d295=20to=201f16df404=20(#53495)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- base/file.jl | 4 ++-- .../Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/md5 | 1 + .../sha512 | 1 + .../Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 | 1 - .../sha512 | 1 - stdlib/Pkg.version | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/md5 create mode 100644 deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 diff --git a/base/file.jl b/base/file.jl index 157951c6bc5bf..6c4f7bc5fb8f3 100644 --- a/base/file.jl +++ b/base/file.jl @@ -285,8 +285,8 @@ function rm(path::AbstractString; force::Bool=false, recursive::Bool=false, allo @static if Sys.iswindows() if allow_delayed_delete && err.code==Base.UV_EACCES && endswith(path, ".dll") # Loaded DLLs cannot be deleted on Windows, even with posix delete mode - # but they can be moved. So move out to allow the dir to be deleted - # TODO: Add a mechanism to delete these moved files after dlclose or process exit + # but they can be moved. So move out to allow the dir to be deleted. + # Pkg.gc() cleans up this dir when possible dir = mkpath(delayed_delete_dir()) temp_path = tempname(dir, cleanup = false, suffix = string("_", basename(path))) @debug "Could not delete DLL most likely because it is loaded, moving to tempdir" path temp_path diff --git a/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/md5 b/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/md5 new file mode 100644 index 0000000000000..6313b4844bd73 --- /dev/null +++ b/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/md5 @@ -0,0 +1 @@ +c8077b74f0e37e3939f74367d9ab55cd diff --git a/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/sha512 b/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/sha512 new file mode 100644 index 0000000000000..42b796b4264db --- /dev/null +++ b/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/sha512 @@ -0,0 +1 @@ +d52830b86bb89d5831768214a4b1b0cc272a810d82f8fdcbf0ccad85085c999c626e75483f443f3d3db6cc29f6fe1bba17c33ef3fffceb5fbcc1e2909b0517cc diff --git a/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 b/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 deleted file mode 100644 index 279472ca7aa3d..0000000000000 --- a/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -777fe3e3dd5b6c7c1b436d3077d91c13 diff --git a/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 b/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 deleted file mode 100644 index 5e26fbd8577fa..0000000000000 --- a/deps/checksums/Pkg-76070d295fc4a1f27f852e05400bbc956962e084.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -b832093b4e387460a1d96526169e12c6ee2ff5a8bd1961c362dbcf4f4839790f3a0bdcf1d0e1524b77eea662cc7590483fc7e4e674f94f5e7f291970778ab128 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 048b652a0a272..532edda44cd99 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 76070d295fc4a1f27f852e05400bbc956962e084 +PKG_SHA1 = 1f16df404a2fbe8642ea3eecc9f4d7064c400a73 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 71f68b4ce9189e64f320631f3f74ffb3dd10e875 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Wed, 28 Feb 2024 06:46:20 +0100 Subject: [PATCH 08/33] Avoid compiler warning about redefining jl_globalref_t (#53499) --- src/julia.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/julia.h b/src/julia.h index a7e0c1b2db17b..5c3a983367470 100644 --- a/src/julia.h +++ b/src/julia.h @@ -645,11 +645,11 @@ typedef struct _jl_module_t { intptr_t hash; } jl_module_t; -typedef struct _jl_globalref_t { +struct _jl_globalref_t { jl_module_t *mod; jl_sym_t *name; jl_binding_t *binding; -} jl_globalref_t; +}; // one Type-to-Value entry typedef struct _jl_typemap_entry_t { From c80a9647166d59782eff6be8c75a5052e17483d7 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Thu, 29 Feb 2024 07:34:54 +0900 Subject: [PATCH 09/33] update staled `Core.Compiler.Effects` documentation (#53507) --- base/compiler/effects.jl | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/base/compiler/effects.jl b/base/compiler/effects.jl index a864925b23eb1..a3d30baef9efa 100644 --- a/base/compiler/effects.jl +++ b/base/compiler/effects.jl @@ -43,15 +43,16 @@ following meanings: except that it may access or modify mutable memory pointed to by its call arguments. This may later be refined to `ALWAYS_TRUE` in a case when call arguments are known to be immutable. This state corresponds to LLVM's `inaccessiblemem_or_argmemonly` function attribute. -- `noub::Bool`: indicates that the method will not execute any undefined behavior (for any input). +- `noub::UInt8`: indicates that the method will not execute any undefined behavior (for any input). Note that undefined behavior may technically cause the method to violate any other effect assertions (such as `:consistent` or `:effect_free`) as well, but we do not model this, and they assume the absence of undefined behavior. + * `ALWAYS_TRUE`: this method is guaranteed to not execute any undefined behavior. + * `ALWAYS_FALSE`: this method may execute undefined behavior. + * `NOUB_IF_NOINBOUNDS`: this method is guaranteed to not execute any undefined behavior + if the caller does not set nor propagate the `@inbounds` context. - `nonoverlayed::Bool`: indicates that any methods that may be called within this method are not defined in an [overlayed method table](@ref OverlayMethodTable). -- `noinbounds::Bool`: If set, indicates that this method does not read the parent's `:inbounds` - state. In particular, it does not have any reached `:boundscheck` exprs, not propagates inbounds - to any children that do. Note that the representations above are just internal implementation details and thus likely to change in the future. See [`Base.@assume_effects`](@ref) for more detailed explanation @@ -92,9 +93,7 @@ The output represents the state of different effect properties in the following 7. `noub` (`u`): - `+u` (green): `true` - `-u` (red): `false` -8. `noinbounds` (`i`): - - `+i` (green): `true` - - `-i` (red): `false` + - `?u` (yellow): `NOUB_IF_NOINBOUNDS` Additionally, if the `nonoverlayed` property is false, a red prime symbol (′) is displayed after the tuple. """ From 77c06727559c3dbb87e5fde46cec87ce59ba61c8 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Wed, 28 Feb 2024 18:06:12 -0500 Subject: [PATCH 10/33] task splitting: change additive accumulation to multiplicative (#53408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue raised [here](https://discourse.julialang.org/t/linear-relationship-between-xoshiro-tasks/110454). Given this definition, look at output values: ```jl using .Threads macro fetch(ex) :(fetch(@spawn($(esc(ex))))) end function taskCorrelatedXoshiros(+ = +) r11, r10 = @fetch (@fetch(rand(UInt64)), rand(UInt64)) r01, r00 = (@fetch(rand(UInt64)), rand(UInt64)) r01 + r10 - (r00 + r11) end ``` Before: ```jl julia> sort!(unique(taskCorrelatedXoshiros() for _ = 1:1000)) 9-element Vector{UInt64}: 0x0000000000000000 0x0000000000000001 0x00000000007fffff 0x0000000000800000 0x0000000000800001 0xffffffffff7fffff 0xffffffffff800000 0xffffffffff800001 0xffffffffffffffff ``` After: ```jl julia> sort!(unique(taskCorrelatedXoshiros() for _ = 1:1000)) 1000-element Vector{UInt64}: 0x000420647a085960 0x0038c5b889b585c6 0x007b29fae8107ff7 0x00e73b9e883ac1c8 ⋮ 0xfe68be9c0dde1e88 0xfeca042354218c35 0xfeeb8203e470c96b 0xfff5dbb8771307b9 julia> sort!(unique(taskCorrelatedXoshiros(*) for _ = 1:1000)) 1000-element Vector{UInt64}: 0x00126f951c1e56dc 0x0025a82477ffac08 0x002dd82c9986457a 0x00a713c4d56a3dbc ⋮ 0xfe2e40a5b345095e 0xfe77b90881967436 0xfea2559be63f1701 0xff88b5b28cefac5f ``` --- src/task.c | 341 ++++++++++++++++++++++------------------------------- 1 file changed, 143 insertions(+), 198 deletions(-) diff --git a/src/task.c b/src/task.c index a1e8dabb89f9d..7c8770530c326 100644 --- a/src/task.c +++ b/src/task.c @@ -856,202 +856,143 @@ guaranteed to avoid collisions between the RNG streams of all tasks. The main RNG is the xoshiro256++ RNG whose state is stored in rngState[0..3]. There is also a small internal RNG used for task forking stored in rngState[4]. This state is used to iterate a linear congruential generator (LCG), which is then -put through four different variations of the strongest PCG output function, -referred to as PCG-RXS-M-XS-64 [1]. This output function is invertible: it maps -a 64-bit state to 64-bit output. This is one of the reasons it's not recommended -for general purpose RNGs unless space is at an absolute premium, but in our -usage invertibility is actually a benefit (as is explained below) and adding as -little additional memory overhead to each task object as possible is preferred. +combined with xoshiro256's state and put through four different variations of +the strongest PCG output function, referred to as PCG-RXS-M-XS-64 [1]. The goal of jl_rng_split is to perturb the state of each child task's RNG in -such a way each that for an entire tree of tasks spawned starting with a given -state in a root task, no two tasks have the same RNG state. Moreover, we want to -do this in a way that is deterministic and repeatable based on (1) the root -task's seed, (2) how many random numbers are generated, and (3) the task tree -structure. The RNG state of a parent task is allowed to affect the initial RNG -state of a child task, but the mere fact that a child was spawned should not -alter the RNG output of the parent. This second requirement rules out using the -main RNG to seed children: if we use the main RNG, we either advance it, which -affects the parent's RNG stream or, if we don't advance it, then every child -would have an identical RNG stream. Therefore some separate state must be -maintained and changed upon forking a child task while leaving the main RNG -state unchanged. - -The basic approach is that used by the DotMix [2] and SplitMix [3] RNG systems: -each task is uniquely identified by a sequence of "pedigree" numbers, indicating -where in the task tree it was spawned. This vector of pedigree coordinates is -then reduced to a single value by computing a dot product with a shared vector -of random weights. The weights are common but each pedigree of each task is -distinct, so the dot product of each task is unlikely to be the same. The DotMix -paper provides a proof that this dot product hash value (referred to as a -"compression function") is collision resistant in the sense that the pairwise -collision probability of two distinct tasks is 1/N where N is the number of -possible weight values. Both DotMix and SplitMix use a prime value of N because -the proof requires that the difference between two distinct pedigree coordinates -have a multiplicative inverse, which is guaranteed by N being prime since all -values are invertible then. We take a somewhat different approach: instead of -assigning n-ary pedigree coordinates, we assign binary tree coordinates to -tasks, which means that our pedigree vectors have only 0/1 and differences -between them can only be -1, 0 or 1. Since the only possible non-zero coordinate -differences are ±1 which are invertible regardless of the modulus, we can use a -modulus of 2^64, which is far easier and more efficient then using a prime -modulus. It also means that when accumulating the dot product incrementally, as -described in SplitMix, we don't need to multiply weights by anything, we simply -add the random weight for the current task tree depth to the parent's dot -product to derive the child's dot product. - -we instead limit pedigree coordinates to being binary, guaranteeing -invertibility regardless of modulus. When a task spawns a child, the parent and -child share the parent's previous pedigree prefix and the parent appends a zero -to its coordinates, which doesn't affect the task's dot product value, while the -child appends a one, which does produce a new dot product. In this manner a -binary pedigree vector uniquely identifies each task and since the coordinates -are binary, the difference between coordinates is always invertible: 1 and -1 -are their own multiplicative inverses regardless of the modulus. - -How does our assignment of pedigree coordinates to tasks differ from DotMix and -SplitMix? In DotMix and SplitMix, each task has a fixed pedigree vector that -never changes. The root tasks's pedigree is `()`, its first child's pedigree is -`(0,)`, its second child's pedigree is `(2,)` and so on. The length of a task's -pedigree tuple corresponds to how many ancestors tasks it has. Our approach -instead appends 0 to the parent's pedigree when it forks a child and appends 1 -to the child's pedigree at the same time. The root task starts with a pedigree -of `()` as before, but when it spawns a child, we update its pedigree to `(0,)` -and give its child a pedigree of `(1,)`. When the root task then spawns a second -child, we update its pedigree to `(0,0)` and give it's second child a pedigree -of `(0,1)`. If the first child spawns a grandchild, the child's pedigree is -changed from `(1,)` to `(1,0)` and the grandchild is assigned a pedigree of -`(1,1)`. In other words, DotMix and SplitMix build an n-ary tree where every -node is a task: parent nodes are higher up the tree and child tasks are children -in the pedigree tree. Our approach is to build a binary tree where only leaves -are tasks and each task spawn replaces a leaf in the tree with two leaves: the -parent moves to the left/zero leaf while the child is the right/one leaf. Since -the tree is binary, the pedigree coordinates are binary. - -It may seem odd for a task's pedigree coordinates to change, but note that we -only ever append zeros to a task's pedigree, which does not change its dot -product. So while the pedigree changes, the dot product is fixed. What purpose -does appending zeros like this serve if the task's dot product doesn't change? -Changing the pedigree length (which is also the binary tree depth) ensures that -the next child spawned by that task will have new and different dot product from -the previous child since it will have a different pseudo-random weight added to -the parent's dot product value. Whereas the pedigree length in DotMix and -SplitMix is unchanging and corresponds to how many ancestors a task has, in our -scheme the pedigree length corresponds to the number of ancestors *plus* -children a task has, which increases every time it spawns another child. - -We use the LCG in rngState[4] to generate pseudorandom weights for the dot -product. Each time a child is forked, we update the LCG in both parent and child -tasks. In the parent, that's all we have to do -- the main RNG state remains -unchanged. (Recall that spawning a child should *not* affect subsequent RNG -draws in the parent). The next time the parent forks a child, the dot product -weight used will be different, corresponding to being a level deeper in the -pedigree tree. In the child, we use the LCG state to generate four pseudorandom -64-bit weights (more below) and add each weight to one of the xoshiro256 state -registers, rngState[0..3]. If we assume the main RNG remains unused in all -tasks, then each register rngState[0..3] accumulates a different dot product -hash as additional child tasks are spawned. Each one is collision resistant with -a pairwise collision chance of only 1/2^64. Assuming that the four pseudorandom -64-bit weight streams are sufficiently independent, the pairwise collision -probability for distinct tasks is 1/2^256. If we somehow managed to spawn a -trillion tasks, the probability of a collision would be on the order of 1/10^54. -In other words, practically impossible. Put another way, this is the same as the -probability of two SHA256 hash values accidentally colliding, which we generally -consider so unlikely as not to be worth worrying about. - -What about the random "junk" that's in the xoshiro256 state registers from -normal use of the RNG? For a tree of tasks spawned with no intervening samples -taken from the main RNG, all tasks start with the same junk which doesn't affect -the chance of collision. The Dot/SplitMix papers even suggest adding a random -base value to the dot product, so we can consider whatever happens to be in the -xoshiro256 registers to be that. What if the main RNG gets used between task -forks? In that case, the initial state registers will be different. The DotMix -collision resistance proof doesn't apply without modification, but we can -generalize the setup by adding a different base constant to each compression -function and observe that we still have a 1/N chance of the weight value -matching that exact difference. This proves collision resistance even between -tasks whose dot product hashes are computed with arbitrary offsets. We can -conclude that this scheme provides collision resistance even in the face of -different starting states of the main RNG. Does this seem too good to be true? -Perhaps another way of thinking about it will help. Suppose we seeded each task -completely randomly. Then there would also be a 1/2^256 chance of collision, -just as the DotMix proof gives. Essentially what the proof is telling us is that -if the weights are chosen uniformly and uncorrelated with the rest of the -compression function, then the dot product construction is a good enough way to -pseudorandomly seed each task based on its parent's RNG state and where in the -task tree it lives. From that perspective, all we need to believe is that the -dot product construction is random enough (assuming the weights are), and it -becomes easier to believe that adding an arbitrary constant to each dot product -value doesn't make its randomness any worse. - -This leaves us with the question of how to generate four pseudorandom weights to -add to the rngState[0..3] registers at each depth of the task tree. The scheme -used here is that a single 64-bit LCG state is iterated in both parent and child -at each task fork, and four different variations of the PCG-RXS-M-XS-64 output -function are applied to that state to generate four different pseudorandom -weights. Another obvious way to generate four weights would be to iterate the -LCG four times per task split. There are two main reasons we've chosen to use -four output variants instead: - -1. Advancing four times per fork reduces the set of possible weights that each - register can be perturbed by from 2^64 to 2^60. Since collision resistance is - proportional to the number of possible weight values, that would reduce - collision resistance. While it would still be strong engough, why reduce it? - -2. It's easier to compute four PCG output variants in parallel. Iterating the - LCG is inherently sequential. PCG variants can be computed independently. All - four can even be computed at once with SIMD vector instructions. The C - compiler doesn't currently choose to do that transformation, but it could. - -A key question is whether the approach of using four variations of PCG-RXS-M-XS -is sufficiently random both within and between streams to provide the collision -resistance we expect. We obviously can't test that with 256 bits, but we have -tested it with a reduced state analogue using four PCG-RXS-M-XS-8 output -variations applied to a common 8-bit LCG. Test results do indicate sufficient -independence: a single register has collisions at 2^5 while four registers only -start having collisions at 2^20. This is actually better scaling of collision -resistance than we theoretically expect. In theory, with one byte of resistance -we have a 50% chance of some collision at 20 tasks, which matches what we see, -but four bytes should give a 50% chance of collision at 2^17 tasks and our -reduced size analogue construction remains collision free at 2^19 tasks. This -may be due to the next observation, which is that the way we generate -pseudorandom weights actually guarantees collision avoidance in many common -situations rather than merely providing collision resistance and thus is better -than true randomness. - -In the specific case where a parent task spawns a sequence of child tasks with -no intervening usage of its main RNG, the parent and child tasks are actually -_guaranteed_ to have different RNG states. This is true because the four PCG -streams each produce every possible 2^64 bit output exactly once in the full -2^64 period of the LCG generator. This is considered a weakness of PCG-RXS-M-XS -when used as a general purpose RNG, but is quite beneficial in this application. -Since each of up to 2^64 children will be perturbed by different weights, they -cannot have hash collisions. What about parent colliding with child? That can -only happen if all four main RNG registers are perturbed by exactly zero. This -seems unlikely, but could it occur? Consider the core of the output function: - - p ^= p >> ((p >> 59) + 5); - p *= m[i]; - p ^= p >> 43 - -It's easy to check that this maps zero to zero. An unchanged parent RNG can only -happen if all four `p` values are zero at the end of this, which implies that -they were all zero at the beginning. However, that is impossible since the four -`p` values differ from `x` by different additive constants, so they cannot all -be zero. Stated more generally, this non-collision property: assuming the main -RNG isn't used between task forks, sibling and parent tasks cannot have RNG -collisions. If the task tree structure is more deeply nested or if there are -intervening uses of the main RNG, we're back to relying on "merely" 256 bits of -collision resistance, but it's nice to know that in what is likely the most -common case, RNG collisions are actually impossible. This fact may also explain -better-than-theoretical collision resistance observed in our experiment with a -reduced size analogue of our hashing system. +such a way that for an entire tree of tasks spawned starting with a given root +task state, no two tasks have the same RNG state. Moreover, we want to do this +in a way that is deterministic and repeatable based on (1) the root task's seed, +(2) how many random numbers are generated, and (3) the task tree structure. The +RNG state of a parent task is allowed to affect the initial RNG state of a child +task, but the mere fact that a child was spawned should not alter the RNG output +of the parent. This second requirement rules out using the main RNG to seed +children: if we use the main RNG, we either advance it, which affects the +parent's RNG stream or, if we don't advance it, then every child would have an +identical RNG stream. Therefore some separate state must be maintained and +changed upon forking a child task while leaving the main RNG state unchanged. + +The basic approach is a generalization and simplification of that used in the +DotMix [2] and SplitMix [3] RNG systems: each task is uniquely identified by a +sequence of "pedigree" numbers, indicating where in the task tree it was +spawned. This vector of pedigree coordinates is then reduced to a single value +by computing a "dot product" with a shared vector of random weights. I write +"dot product" in quotes because what we use is not an actual dot product. The +linear dot product construction used in both DotMix and SplitMix was found by +@foobar_iv2 [4] to allow easy construction of linear relationships between the +main RNG states of tasks, which was in turn reflected in observable linear +relationships between the outputs of their RNGs. This relationship was between a +minimum of four tasks, so doesn't constitute a collision, per se, but is clearly +undesirable and highlights a hazard of the plain dot product construction. + +As in DotMix and SplitMix, each task is assigned unique task "pedigree" +coordinates. Our pedigree construction is a bit different and uses only binary +coordinates rather than arbitrary integers. Each pedigree is an infinite +sequence of ones and zeros with only finitely many ones. Each task has a "fork +index": the root task has index 0; the fork index of a task the jth child task +of a parent task with fork index i is i+j. The root task's coordinates are all +zeros; each child task's coordinates are the same as its parents except at its +fork index, where the parent has a zero while the child has a one. A task's +coordinates after its fork index are all zeros. The coordinates of a tasks +ancestors are all prefixes of its own coordinates, padded with zeros. + +Also as in DotMix and SplitMix, we generate a sequence of pseudorandom weights +to combine with the coordinates of each task. This sequence is common across all +tasks, and different mix values for each task derive from their coordinates +being different. In DotMix and SplitMix, this is a literal dot product: the +pseudorandom weights are multiplied by corresponding task coordinate and added +up. While this does provably make collisions as unlikely as randomly assigned +task seeds, this linear construction can be used to create linearly correlated +states between tasks. However, it turns out that the compression construction +need not be linear, commutative, associative, etc. which allows us to avoid any +linear or other obvious correlations between related sets of tasks. + +To generalize SplitMix's optimized dot product construction, we similarly +compute each task's compression function value incrementally by combining the +parent's compression value with pseudorandom weight corresponding with the +child's fork index. Formally, if the parent's compression value is c then we can +compute the child's compression value as c′ = f(c, wᵢ) where w is the vector of +pseudorandom weights. What is f? It can be any function that is bijective in +each argument for all values of the other argument: + + * For all c: w ↦ f(c, w) is bijective + * For all w: c ↦ f(c, w) is bijective + +The proof that these requirements are sufficient to ensure collision resistance +is in the linked discussion [4]. DotMix/SplitMix are a special case where f is +just addition. Instead we use a much less simple mixing function: + + 1. We use (2c+1)(2w+1)÷2 % 2^64 to mix the bits of c and w + 2. We then apply the PCG-RXS-M-XS-64 output function + +The first step thoroughly mixes the bits of the previous compression value and +the pseudorandom weight value using multiplication, which is non-commutative +with xoshiro's operations (xor, shift, rotate). This mixing function is a +bijection on each argument witnessed by these inverses: + + * c′ ↦ (2c′+1)(2w+1)⁻¹÷2 % 2^64 + * w′ ↦ (2c+1)⁻¹(2w′+1)÷2 % 2^64 + +The second PCG output step is a bijection and designed to be significantly +non-linear -- non-linear enough to mask the linearity of the LCG that drives the +PCG-RXS-M-XS-64 RNG and allows it to pass statistical RNG test suites despite +having the same size state and output. In particular, since this mixing function +is highly non-associative and non-linear, we (hopefully) don't have any +discernible relationship between these values: + + * c₀₀ = c + * c₁₀ = f(c, wᵢ) + * c₀₁ = f(c, wⱼ) + * c₁₁ = f(f(c, wᵢ), wⱼ) + +When f is simply `+` then these have a very obvious relationship: + + c₀₀ + c₁₁ == c₁₀ + c₀₁ + +This relationship holds regardless of what wᵢ and wⱼ are and is precisely what +allows easy creation of correlated tasks with the DotMix/SplitMix construction +that we previously used. Expressing any relationship between these values with +our mixing function would require inverting the PCG output function (doable but +non-trivial), knowing the weights wᵢ and wⱼ, and then applying the inversion +functions for those weights appropriately. Since the weights are pseudo-randomly +generated and not directly observable, this is infeasible. + +We maintain an LCG in rngState[4] to generate pseudorandom weights. An LCG by +itself is a very bad RNG, but we combine this one with xoshiro256 state +registers in a non-trivial way and then apply the PCG-RXS-M-XS-64 output +function to that. Even if the xoshiro256 states are all zeros, which they should +never be, the output would be the same as PCG-RXS-M-XS-64, which is a solid +statistical RNG. + +Each time a child is forked, we update the LCG in both parent and child tasks, +corresponding to increasing the fork index. In the parent, that's all we have to +do -- the main RNG state remains unchanged. Recall that spawning a child should +*not* affect subsequent RNG draws in the parent. The next time the parent forks +a child, the mixing weight used will be different. In the child, we use the LCG +state to perturb the child's main RNG state registers, rngState[0..3]. + +Since we want these registers to behave independently, we use four different +variations on f to mix the LCG state with each of the four main RNG registers. +Each variation first xors the LCG state with a different random constant before +combining that value above with the old register state via multiplication; the +PCG-RXS-M-XS-64 output function is then applied to that mixed state, with a +different multiplier constant for each variation / register index. Xor is used +in the first step since we multiply the result with the state immediately after +and multiplication distributes over `+` and commutes with `*`, which makes both +options suspect; multiplication doesn't distribute over or commute with xor. We +also use a different odd multiplier in PCG-RXS-M-XS-64 for each RNG register. +These three sources of variation (different xor constants, different xoshiro256 +state, different PCG multipliers) are sufficient for each of the four outputs to +behave statistically independently. [1]: https://www.pcg-random.org/pdf/hmc-cs-2014-0905.pdf [2]: http://supertech.csail.mit.edu/papers/dprng.pdf [3]: https://gee.cs.oswego.edu/dl/papers/oopsla14.pdf + +[4]: +https://discourse.julialang.org/t/linear-relationship-between-xoshiro-tasks/110454 */ void jl_rng_split(uint64_t dst[JL_RNG_SIZE], uint64_t src[JL_RNG_SIZE]) JL_NOTSAFEPOINT { @@ -1060,26 +1001,30 @@ void jl_rng_split(uint64_t dst[JL_RNG_SIZE], uint64_t src[JL_RNG_SIZE]) JL_NOTSA src[4] = dst[4] = x * 0xd1342543de82ef95 + 1; // high spectrum multiplier from https://arxiv.org/abs/2001.05304 + // random xor constants static const uint64_t a[4] = { - 0xe5f8fa077b92a8a8, // random additive offsets... - 0x7a0cd918958c124d, - 0x86222f7d388588d4, - 0xd30cbd35f2b64f52 + 0x214c146c88e47cb7, + 0xa66d8cc21285aafa, + 0x68c7ef2d7b1a54d4, + 0xb053a7d7aa238c61 }; + // random odd multipliers static const uint64_t m[4] = { 0xaef17502108ef2d9, // standard PCG multiplier - 0xf34026eeb86766af, // random odd multipliers... + 0xf34026eeb86766af, 0x38fd70ad58dd9fbb, 0x6677f9b93ab0c04d }; // PCG-RXS-M-XS-64 output with four variants for (int i = 0; i < 4; i++) { - uint64_t p = x + a[i]; - p ^= p >> ((p >> 59) + 5); - p *= m[i]; - p ^= p >> 43; - dst[i] = src[i] + p; // SplitMix dot product + uint64_t c = src[i]; + uint64_t w = x ^ a[i]; + c += w*(2*c + 1); // c = (2c+1)(2w+1)÷2 % 2^64 (double bijection) + c ^= c >> ((c >> 59) + 5); + c *= m[i]; + c ^= c >> 43; + dst[i] = c; } } From 84ae35190ed265899c7a46df7f3b6305cd614405 Mon Sep 17 00:00:00 2001 From: DilumAluthgeBot <43731525+DilumAluthgeBot@users.noreply.github.com> Date: Thu, 29 Feb 2024 05:15:12 -0500 Subject: [PATCH 11/33] =?UTF-8?q?=F0=9F=A4=96=20[master]=20Bump=20the=20Pk?= =?UTF-8?q?g=20stdlib=20from=201f16df404=20to=2048eea8dbd=20(#53517)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stdlib: Pkg URL: https://github.com/JuliaLang/Pkg.jl.git Stdlib branch: master Julia branch: master Old commit: 1f16df404 New commit: 48eea8dbd Julia version: 1.12.0-DEV Pkg version: 1.11.0(Does not match) Bump invoked by: @KristofferC Powered by: [BumpStdlibs.jl](https://github.com/JuliaLang/BumpStdlibs.jl) Diff: https://github.com/JuliaLang/Pkg.jl/compare/1f16df404a2fbe8642ea3eecc9f4d7064c400a73...48eea8dbd7b651cdc932b909c1b718bb9c3f94f4 ``` $ git log --oneline 1f16df404..48eea8dbd 48eea8dbd setenv -> addenv (#3819) ``` Co-authored-by: Dilum Aluthge --- .../Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/md5 | 1 - .../Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/sha512 | 1 - .../Pkg-48eea8dbd7b651cdc932b909c1b718bb9c3f94f4.tar.gz/md5 | 1 + .../Pkg-48eea8dbd7b651cdc932b909c1b718bb9c3f94f4.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-48eea8dbd7b651cdc932b909c1b718bb9c3f94f4.tar.gz/md5 create mode 100644 deps/checksums/Pkg-48eea8dbd7b651cdc932b909c1b718bb9c3f94f4.tar.gz/sha512 diff --git a/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/md5 b/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/md5 deleted file mode 100644 index 6313b4844bd73..0000000000000 --- a/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -c8077b74f0e37e3939f74367d9ab55cd diff --git a/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/sha512 b/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/sha512 deleted file mode 100644 index 42b796b4264db..0000000000000 --- a/deps/checksums/Pkg-1f16df404a2fbe8642ea3eecc9f4d7064c400a73.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -d52830b86bb89d5831768214a4b1b0cc272a810d82f8fdcbf0ccad85085c999c626e75483f443f3d3db6cc29f6fe1bba17c33ef3fffceb5fbcc1e2909b0517cc diff --git a/deps/checksums/Pkg-48eea8dbd7b651cdc932b909c1b718bb9c3f94f4.tar.gz/md5 b/deps/checksums/Pkg-48eea8dbd7b651cdc932b909c1b718bb9c3f94f4.tar.gz/md5 new file mode 100644 index 0000000000000..341ea80c32a7e --- /dev/null +++ b/deps/checksums/Pkg-48eea8dbd7b651cdc932b909c1b718bb9c3f94f4.tar.gz/md5 @@ -0,0 +1 @@ +fea1786e8202d07744c17457a114e911 diff --git a/deps/checksums/Pkg-48eea8dbd7b651cdc932b909c1b718bb9c3f94f4.tar.gz/sha512 b/deps/checksums/Pkg-48eea8dbd7b651cdc932b909c1b718bb9c3f94f4.tar.gz/sha512 new file mode 100644 index 0000000000000..8f6d09bbb6001 --- /dev/null +++ b/deps/checksums/Pkg-48eea8dbd7b651cdc932b909c1b718bb9c3f94f4.tar.gz/sha512 @@ -0,0 +1 @@ +e79913b68643ec4c448a861f446d0358484b01d16794c9dfe700b1f539d4c01ea259014d37df263b4d7dd3f3c2e1df3966c584b428041dbdf03d4e4794b52a64 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 532edda44cd99..ca45bee7ee736 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 1f16df404a2fbe8642ea3eecc9f4d7064c400a73 +PKG_SHA1 = 48eea8dbd7b651cdc932b909c1b718bb9c3f94f4 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 24aaf00819e49ba5c377b9e9deed287e54333129 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 29 Feb 2024 15:22:15 +0100 Subject: [PATCH 12/33] add back an alias for `check_top_bit` (#53523) Used in some packages (e.g. rfourquet/BitIntegers.jl) Renamed in https://github.com/JuliaLang/julia/pull/53166 --- base/boot.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/boot.jl b/base/boot.jl index 8ca6d392ead67..f8ed86b0fb6b7 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -969,6 +969,7 @@ arrayset(inbounds::Bool, A::Array{T}, x::Any, i::Int...) where {T} = Main.Base.s arraysize(a::Array) = a.size arraysize(a::Array, i::Int) = sle_int(i, nfields(a.size)) ? getfield(a.size, i) : 1 export arrayref, arrayset, arraysize, const_arrayref +const check_top_bit = check_sign_bit # For convenience EnterNode(old::EnterNode, new_dest::Int) = isdefined(old, :scope) ? From 715c50d82af8d2856281e99bbed68014c3ddefd9 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 29 Feb 2024 09:32:17 -0500 Subject: [PATCH 13/33] codegen: make gcroots for argument-derived values (#53501) This function might previously refuse to make gc roots for any of the function's arguments (i.e. it potentially assumed everything was in an alloca), which was only valid if we could guarantee that no IPO passes run. This commit aims to make that safe, but deferring such decisions about their source until llvm-late-gc-lowering can make them valid. This should make inlining-safe gc annotations for code_llvm(raw=true, optimize=false, (Some{Int},)) do x; GC.@preserve x GC.safepoint(); end and also better annotations even without inlining for code_llvm(raw=true, optimize=false, (Some{Any},)) do x; GC.@preserve x GC.safepoint(); end --- src/ccall.cpp | 10 ++-- src/cgutils.cpp | 104 +++++++++++++++++----------------- src/codegen.cpp | 17 ++---- src/llvm-late-gc-lowering.cpp | 20 ++++--- 4 files changed, 72 insertions(+), 79 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 5dd23d083186a..0fcc838905004 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1065,7 +1065,7 @@ class function_sig_t { jl_codectx_t &ctx, const native_sym_arg_t &symarg, jl_cgval_t *argv, - SmallVector &gc_uses, + SmallVectorImpl &gc_uses, bool static_rt) const; private: @@ -1389,16 +1389,14 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) } // emit roots - SmallVector gc_uses; + SmallVector gc_uses; for (size_t i = nccallargs + fc_args_start; i <= nargs; i++) { // Julia (expression) value of current parameter gcroot jl_value_t *argi_root = args[i]; if (jl_is_long(argi_root)) continue; jl_cgval_t arg_root = emit_expr(ctx, argi_root); - Value *gc_root = get_gc_root_for(ctx, arg_root); - if (gc_root) - gc_uses.push_back(gc_root); + gc_uses.append(get_gc_roots_for(ctx, arg_root)); } jl_unionall_t *unionall = (jl_is_method(ctx.linfo->def.method) && jl_is_unionall(ctx.linfo->def.method->sig)) @@ -1855,7 +1853,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( jl_codectx_t &ctx, const native_sym_arg_t &symarg, jl_cgval_t *argv, - SmallVector &gc_uses, + SmallVectorImpl &gc_uses, bool static_rt) const { ++EmittedCCalls; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 695090f767546..7bbf7a84a0385 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -312,32 +312,64 @@ static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V) static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_value_t *jt); static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value* dest, MDNode *tbaa_dest, unsigned alignment, bool isVolatile=false); -static Value *get_gc_root_for(jl_codectx_t &ctx, const jl_cgval_t &x) +static bool type_is_permalloc(jl_value_t *typ) +{ + // Singleton should almost always be handled by the later optimization passes. + // Also do it here since it is cheap and save some effort in LLVM passes. + if (jl_is_datatype(typ) && jl_is_datatype_singleton((jl_datatype_t*)typ)) + return true; + return typ == (jl_value_t*)jl_symbol_type || + typ == (jl_value_t*)jl_int8_type || + typ == (jl_value_t*)jl_uint8_type; +} + + +static void find_perm_offsets(jl_datatype_t *typ, SmallVectorImpl &res, unsigned offset) +{ + // This is a inlined field at `offset`. + if (!typ->layout || typ->layout->npointers == 0) + return; + jl_svec_t *types = jl_get_fieldtypes(typ); + size_t nf = jl_svec_len(types); + for (size_t i = 0; i < nf; i++) { + jl_value_t *_fld = jl_svecref(types, i); + if (!jl_is_datatype(_fld)) + continue; + jl_datatype_t *fld = (jl_datatype_t*)_fld; + if (jl_field_isptr(typ, i)) { + // pointer field, check if field is perm-alloc + if (type_is_permalloc((jl_value_t*)fld)) + res.push_back(offset + jl_field_offset(typ, i)); + continue; + } + // inline field + find_perm_offsets(fld, res, offset + jl_field_offset(typ, i)); + } +} + +static llvm::SmallVector get_gc_roots_for(jl_codectx_t &ctx, const jl_cgval_t &x) { if (x.constant || x.typ == jl_bottom_type) - return nullptr; + return {}; if (x.Vboxed) // superset of x.isboxed - return x.Vboxed; + return {x.Vboxed}; assert(!x.isboxed); -#ifndef NDEBUG if (x.ispointer()) { assert(x.V); - if (PointerType *T = dyn_cast(x.V->getType())) { - assert(T->getAddressSpace() != AddressSpace::Tracked); - if (T->getAddressSpace() == AddressSpace::Derived) { - // n.b. this IR would not be valid after LLVM-level inlining, - // since codegen does not have a way to determine the whether - // this argument value needs to be re-rooted - } - } - } -#endif - if (jl_is_concrete_immutable(x.typ) && !jl_is_pointerfree(x.typ)) { - Type *T = julia_type_to_llvm(ctx, x.typ); - return emit_unbox(ctx, T, x, x.typ); + assert(x.V->getType()->getPointerAddressSpace() != AddressSpace::Tracked); + return {x.V}; + } + else if (jl_is_concrete_immutable(x.typ) && !jl_is_pointerfree(x.typ)) { + jl_value_t *jltype = x.typ; + Type *T = julia_type_to_llvm(ctx, jltype); + Value *agg = emit_unbox(ctx, T, x, jltype); + SmallVector perm_offsets; + if (jltype && jl_is_datatype(jltype) && ((jl_datatype_t*)jltype)->layout) + find_perm_offsets((jl_datatype_t*)jltype, perm_offsets, 0); + return ExtractTrackedValues(agg, agg->getType(), false, ctx.builder, perm_offsets); } // nothing here to root, move along - return nullptr; + return {}; } // --- emitting pointers directly into code --- @@ -590,17 +622,6 @@ static Value *julia_binding_gv(jl_codectx_t &ctx, jl_binding_t *b) // --- mapping between julia and llvm types --- -static bool type_is_permalloc(jl_value_t *typ) -{ - // Singleton should almost always be handled by the later optimization passes. - // Also do it here since it is cheap and save some effort in LLVM passes. - if (jl_is_datatype(typ) && jl_is_datatype_singleton((jl_datatype_t*)typ)) - return true; - return typ == (jl_value_t*)jl_symbol_type || - typ == (jl_value_t*)jl_int8_type || - typ == (jl_value_t*)jl_uint8_type; -} - static unsigned convert_struct_offset(const llvm::DataLayout &DL, Type *lty, unsigned byte_offset) { const StructLayout *SL = DL.getStructLayout(cast(lty)); @@ -1905,7 +1926,7 @@ Value *extract_first_ptr(jl_codectx_t &ctx, Value *V) if (path.empty()) return NULL; std::reverse(std::begin(path), std::end(path)); - return ctx.builder.CreateExtractValue(V, path); + return CreateSimplifiedExtractValue(ctx, V, path); } @@ -3590,29 +3611,6 @@ static void emit_write_barrier(jl_codectx_t &ctx, Value *parent, ArrayRef &res, unsigned offset) -{ - // This is a inlined field at `offset`. - if (!typ->layout || typ->layout->npointers == 0) - return; - jl_svec_t *types = jl_get_fieldtypes(typ); - size_t nf = jl_svec_len(types); - for (size_t i = 0; i < nf; i++) { - jl_value_t *_fld = jl_svecref(types, i); - if (!jl_is_datatype(_fld)) - continue; - jl_datatype_t *fld = (jl_datatype_t*)_fld; - if (jl_field_isptr(typ, i)) { - // pointer field, check if field is perm-alloc - if (type_is_permalloc((jl_value_t*)fld)) - res.push_back(offset + jl_field_offset(typ, i)); - continue; - } - // inline field - find_perm_offsets(fld, res, offset + jl_field_offset(typ, i)); - } -} - static void emit_write_multibarrier(jl_codectx_t &ctx, Value *parent, Value *agg, jl_value_t *jltype) { diff --git a/src/codegen.cpp b/src/codegen.cpp index bb9622c8e75c8..59c7763350fff 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3365,21 +3365,18 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t a value_to_pointer(ctx, arg2).V; varg1 = emit_pointer_from_objref(ctx, varg1); varg2 = emit_pointer_from_objref(ctx, varg2); - Value *gc_uses[2]; - int nroots = 0; + SmallVector gc_uses; // these roots may seem a bit overkill, but we want to make sure // that a!=b implies (a,)!=(b,) even if a and b are unused and // therefore could be freed and then the memory for a reused for b - if ((gc_uses[nroots] = get_gc_root_for(ctx, arg1))) - nroots++; - if ((gc_uses[nroots] = get_gc_root_for(ctx, arg2))) - nroots++; - OperandBundleDef OpBundle("jl_roots", ArrayRef(gc_uses, nroots)); + gc_uses.append(get_gc_roots_for(ctx, arg1)); + gc_uses.append(get_gc_roots_for(ctx, arg2)); + OperandBundleDef OpBundle("jl_roots", gc_uses); auto answer = ctx.builder.CreateCall(prepare_call(memcmp_func), { ctx.builder.CreateBitCast(varg1, getInt8PtrTy(ctx.builder.getContext())), ctx.builder.CreateBitCast(varg2, getInt8PtrTy(ctx.builder.getContext())), ConstantInt::get(ctx.types().T_size, sz) }, - ArrayRef(&OpBundle, nroots ? 1 : 0)); + ArrayRef(&OpBundle, gc_uses.empty() ? 0 : 1)); if (arg1.tbaa || arg2.tbaa) { jl_aliasinfo_t ai; @@ -6432,9 +6429,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ } SmallVector vals; for (size_t i = 0; i < nargs; ++i) { - Value *gc_root = get_gc_root_for(ctx, argv[i]); - if (gc_root) - vals.push_back(gc_root); + vals.append(get_gc_roots_for(ctx, argv[i])); } Value *token = vals.empty() ? (Value*)ConstantTokenNone::get(ctx.builder.getContext()) diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index e081c6f067245..b35f64be813d7 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -12,7 +12,8 @@ #include #include #include -#include "llvm/Analysis/CFG.h" +#include +#include #include #include #include @@ -455,7 +456,6 @@ SmallVector, 0> TrackCompositeType(Type *T) { } - // Walk through simple expressions to until we hit something that requires root numbering // If the input value is a scalar (pointer), we may return a composite value as base // in which case the second member of the pair is the index of the value in the vector. @@ -616,12 +616,12 @@ Value *LateLowerGCFrame::MaybeExtractScalar(State &S, std::pair ValE V = ConstantPointerNull::get(cast(T_prjlvalue)); return V; } + IRBuilder foldbuilder(InsertBefore->getContext(), InstSimplifyFolder(InsertBefore->getModule()->getDataLayout())); + foldbuilder.SetInsertPoint(InsertBefore); if (Idxs.size() > IsVector) - V = ExtractValueInst::Create(V, IsVector ? IdxsNotVec : Idxs, "", InsertBefore); + V = foldbuilder.CreateExtractValue(V, IsVector ? IdxsNotVec : Idxs); if (IsVector) - V = ExtractElementInst::Create(V, - ConstantInt::get(Type::getInt32Ty(V->getContext()), Idxs.back()), - "", InsertBefore); + V = foldbuilder.CreateExtractElement(V, ConstantInt::get(Type::getInt32Ty(V->getContext()), Idxs.back())); } return V; } @@ -1817,11 +1817,13 @@ static Value *ExtractScalar(Value *V, Type *VTy, bool isptr, ArrayRef auto IdxsNotVec = Idxs.slice(0, Idxs.size() - 1); Type *FinalT = ExtractValueInst::getIndexedType(V->getType(), IdxsNotVec); bool IsVector = isa(FinalT); + IRBuilder foldbuilder(irbuilder.getContext(), InstSimplifyFolder(irbuilder.GetInsertBlock()->getModule()->getDataLayout())); + foldbuilder.restoreIP(irbuilder.saveIP()); + foldbuilder.SetCurrentDebugLocation(irbuilder.getCurrentDebugLocation()); if (Idxs.size() > IsVector) - V = irbuilder.Insert(ExtractValueInst::Create(V, IsVector ? IdxsNotVec : Idxs)); + V = foldbuilder.CreateExtractValue(V, IsVector ? IdxsNotVec : Idxs); if (IsVector) - V = irbuilder.Insert(ExtractElementInst::Create(V, - ConstantInt::get(Type::getInt32Ty(V->getContext()), Idxs.back()))); + V = foldbuilder.CreateExtractElement(V, ConstantInt::get(Type::getInt32Ty(V->getContext()), Idxs.back())); } return V; } From e50ca99ae75752c6314e394c1c1e923a7d8844a0 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 29 Feb 2024 10:40:01 -0500 Subject: [PATCH 14/33] fix potential double-free in flisp table implementation (#53519) In this case we would add two finalizers, leading to a double free. Introduced by 5fc4ba91931526a08fc1bf8d3937aac731ee2cc6. However, I believe we never hit this in the julia front end, since it requires making a large pre-initialized table (i.e. with many arguments to `table`) which is later freed. That tends not to happen since initialized tables tend to be global constants. --- src/flisp/table.c | 2 +- src/flisp/unittest.lsp | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/flisp/table.c b/src/flisp/table.c index 1d8aed358e88d..8836c93f81513 100644 --- a/src/flisp/table.c +++ b/src/flisp/table.c @@ -102,7 +102,7 @@ value_t fl_table(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) else k = arg; } - if (h->table != &h->_space[0]) { + if (cnt <= HT_N_INLINE && h->table != &h->_space[0]) { // We expected to use the inline table, but we ended up outgrowing it. // Make sure to register the finalizer. add_finalizer(fl_ctx, (cvalue_t*)ptr(nt)); diff --git a/src/flisp/unittest.lsp b/src/flisp/unittest.lsp index 584d5c81225e8..16774a97e3233 100644 --- a/src/flisp/unittest.lsp +++ b/src/flisp/unittest.lsp @@ -267,4 +267,23 @@ (assert (equal? `(a `(b c)) '(a (quasiquote (b c))))) (assert (equal? ````x '```x)) +;; make many initialized tables large enough not to be stored in-line +(for 1 100 + (lambda (i) + (table eq? 2 eqv? 2 + equal? 2 atom? 1 + not 1 null? 1 + boolean? 1 symbol? 1 + number? 1 bound? 1 + pair? 1 builtin? 1 + vector? 1 fixnum? 1 + cons 2 car 1 + cdr 1 set-car! 2 + set-cdr! 2 = 2 + < 2 compare 2 + aref 2 aset! 3 + div0 2))) +;; now allocate enough to trigger GC +(for 1 8000000 (lambda (i) (cons 1 2))) + #t From 962bbf75e8e4a5cc5181fbccf895b0a5b20b6e34 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 29 Feb 2024 19:01:22 +0100 Subject: [PATCH 15/33] Introduce -m/--module flag to execute a `main` function in a package (#52103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This aims to bring similar functionality to Julia as the `-m` flag for Python which exists to directly run some function in a package and being able to pass arguments to that function. While in Python, `python -m package args` runs the file `.__main__.py`, the equivalent Julia command (`julia -m Package args`) instead runs `.main(args)`. The package is assumed to be installed in the environment `julia` is run in. An example usage could be: Add the package: ```julia (@v1.11) pkg> add https://github.com/KristofferC/Rot13.jl Cloning git-repo `https://github.com/KristofferC/Rot13.jl` Updating git-repo `https://github.com/KristofferC/Rot13.jl` Resolving package versions... Updating `~/.julia/environments/v1.11/Project.toml` [43ef800a] + Rot13 v0.1.0 `https://github.com/KristofferC/Rot13.jl#master` Updating `~/.julia/environments/v1.11/Manifest.toml` [43ef800a] + Rot13 v0.1.0 `https://github.com/KristofferC/Rot13.jl#master` ``` And then it can be run (since it has a `main` function) via: ``` ❯ ./julia/julia -m Rot13 "encrypt this for me" "and this as well" rapelcg guvf sbe zr naq guvf nf jryy ``` I'm not sure if `-m/--module` is the best choice but perhaps the association to Python makes it worth it. --- NEWS.md | 3 +++ base/client.jl | 9 ++++++++- doc/man/julia.1 | 5 +++++ src/jloptions.c | 11 ++++++++++- test/loading.jl | 5 +++++ test/project/Rot13/Project.toml | 3 +++ test/project/Rot13/src/Rot13.jl | 15 +++++++++++++++ 7 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 test/project/Rot13/Project.toml create mode 100644 test/project/Rot13/src/Rot13.jl diff --git a/NEWS.md b/NEWS.md index 56a2ffb8d539d..432211ba003f4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -23,6 +23,9 @@ Compiler/Runtime improvements Command-line option changes --------------------------- +* The `-m/--module` flag can be passed to run the `main` function inside a package with a set of arguments. + This `main` function should be declared using `@main` to indicate that it is an entry point. + Multi-threading changes ----------------------- diff --git a/base/client.jl b/base/client.jl index 31a3016489dcd..087efe3d3b99c 100644 --- a/base/client.jl +++ b/base/client.jl @@ -240,7 +240,7 @@ function exec_options(opts) if cmd_suppresses_program(cmd) arg_is_program = false repl = false - elseif cmd == 'L' + elseif cmd == 'L' || cmd == 'm' # nothing elseif cmd == 'B' # --bug-report # If we're doing a bug report, don't load anything else. We will @@ -292,6 +292,13 @@ function exec_options(opts) elseif cmd == 'E' invokelatest(show, Core.eval(Main, parse_input_line(arg))) println() + elseif cmd == 'm' + @eval Main import $(Symbol(arg)).main + if !should_use_main_entrypoint() + error("`main` in `$arg` not declared as entry point (use `@main` to do so)") + end + return false + elseif cmd == 'L' # load file immediately on all processors if !distributed_mode diff --git a/doc/man/julia.1 b/doc/man/julia.1 index afde0326e4952..a7d16abe3f85c 100644 --- a/doc/man/julia.1 +++ b/doc/man/julia.1 @@ -106,6 +106,11 @@ Enable or disable usage of native code caching in the form of pkgimages -e, --eval Evaluate +.TP +-m, --module [args] +Run entry point of `Package` (`@main` function) with `args'. + + .TP -E, --print Evaluate and display the result diff --git a/src/jloptions.c b/src/jloptions.c index 570d021351104..2ba5174e5c730 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -128,6 +128,8 @@ static const char opts[] = // actions " -e, --eval Evaluate \n" " -E, --print Evaluate and display the result\n" + " -m, --module [args]\n" + " Run entry point of `Package` (`@main` function) with `args'.\n" " -L, --load Load immediately on all processors\n\n" // parallel options @@ -271,7 +273,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_gc_threads, opt_permalloc_pkgimg }; - static const char* const shortopts = "+vhqH:e:E:L:J:C:it:p:O:g:"; + static const char* const shortopts = "+vhqH:e:E:L:J:C:it:p:O:g:m:"; static const struct option longopts[] = { // exposed command line options // NOTE: This set of required arguments need to be kept in sync @@ -284,6 +286,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "banner", required_argument, 0, opt_banner }, { "home", required_argument, 0, 'H' }, { "eval", required_argument, 0, 'e' }, + { "module", required_argument, 0, 'm' }, { "print", required_argument, 0, 'E' }, { "load", required_argument, 0, 'L' }, { "bug-report", required_argument, 0, opt_bug_report }, @@ -421,6 +424,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) case 'e': // eval case 'E': // print case 'L': // load + case 'm': // module case opt_bug_report: // bug { size_t sz = strlen(optarg) + 1; @@ -434,6 +438,10 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) ncmds++; cmds[ncmds] = 0; jl_options.cmds = cmds; + if (c == 'm') { + optind -= 1; + goto parsing_args_done; + } break; } case 'J': // sysimage @@ -886,6 +894,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) "This is a bug, please report it.", c); } } + parsing_args_done: jl_options.code_coverage = codecov; jl_options.malloc_log = malloclog; int proc_args = *argcp < optind ? *argcp : optind; diff --git a/test/loading.jl b/test/loading.jl index 4a72a0f8060ad..8ba2cf3026120 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1545,3 +1545,8 @@ end @test_throws SystemError("opening file $(repr(file))") include(file) end end + +@testset "-m" begin + rot13proj = joinpath(@__DIR__, "project", "Rot13") + @test readchomp(`$(Base.julia_cmd()) --startup-file=no --project=$rot13proj -m Rot13 --project nowhere ABJURER`) == "--cebwrpg abjurer NOWHERE " +end diff --git a/test/project/Rot13/Project.toml b/test/project/Rot13/Project.toml new file mode 100644 index 0000000000000..eb03cb84d588e --- /dev/null +++ b/test/project/Rot13/Project.toml @@ -0,0 +1,3 @@ +name = "Rot13" +uuid = "43ef800a-eac4-47f4-949b-25107b932e8f" +version = "0.1.0" diff --git a/test/project/Rot13/src/Rot13.jl b/test/project/Rot13/src/Rot13.jl new file mode 100644 index 0000000000000..0672799d61f24 --- /dev/null +++ b/test/project/Rot13/src/Rot13.jl @@ -0,0 +1,15 @@ +module Rot13 + +function rot13(c::Char) + shft = islowercase(c) ? 'a' : 'A' + isletter(c) ? c = shft + (c - shft + 13) % 26 : c +end + +rot13(str::AbstractString) = map(rot13, str) + +function (@main)(ARGS) + foreach(arg -> print(rot13(arg), " "), ARGS) + return 0 +end + +end # module Rot13 From c06662a4f8d3fb277a30df82f2a012739f3c84d6 Mon Sep 17 00:00:00 2001 From: KronosTheLate <61620837+KronosTheLate@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:01:47 +0100 Subject: [PATCH 16/33] Implement and export `isfull` (#53159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR implements `isfull(c::Channel)`. It calls `n_avail(c) ≥ c.sz_max` in all cases. The original implementation was inspired by [this comment](https://discourse.julialang.org/t/function-to-check-if-channel-is-full/44795/3?u=thelatekronos), and therefore had a special case for unbuffered channels, which fell back to `isready`. I opted against this behaviour, because it fails to respect that an unbuffered channel is always full, in two important senses: 1) The number of elements available is greater than or equal the capacity 2) A call to `put!` will block With the current implementation, the behaviour is simply understood and summarized in all cases by the start of the docstring: > Determines whether a `Channel` is full, in the sense that calling `put!(c, some_value)` will block. Shoutout to @SamuraiAku for their work in https://github.com/JuliaLang/julia/pull/40720, which helped me a lot on thinking this through, and remembering to change all relevant files. In particular, the detail around how `c.cond_take.waitq` may result in immediate unblocking, which is a really important caveat on a function that may be used to check if `put!`ing will block. However, for buffered channels, `isfull` is extremely close to `putwillblock` from #40720 (just a little better, with >= instead of ==), and for unbuffered channels it does not make much sense to see if `put!`ing will block. This PR is created based on [this](https://github.com/JuliaLang/julia/issues/22863#issuecomment-1921498544) "call to action". Checklist: - [x] Entry in news - [x] Docstring with example - [x] Export function - [x] Mention in manual - [x] Entry in [docs-reference](https://docs.julialang.org/en/v1/base/parallel/) --------- Co-authored-by: Jameson Nash --- NEWS.md | 1 + base/channels.jl | 43 +++++++++++++++++++++- base/exports.jl | 1 + doc/src/base/parallel.md | 1 + doc/src/manual/asynchronous-programming.md | 5 ++- test/channels.jl | 7 ++++ 6 files changed, 55 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 432211ba003f4..4d312c28ce34e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -36,6 +36,7 @@ New library functions --------------------- * `logrange(start, stop; length)` makes a range of constant ratio, instead of constant step ([#39071]) +* The new `isfull(c::Channel)` function can be used to check if `put!(c, some_value)` will block. ([#53159]) New library features -------------------- diff --git a/base/channels.jl b/base/channels.jl index 3ddf9cff7634c..3acbf37246a58 100644 --- a/base/channels.jl +++ b/base/channels.jl @@ -519,7 +519,7 @@ end Determines whether a [`Channel`](@ref) has a value stored in it. Returns immediately, does not block. -For unbuffered channels returns `true` if there are tasks waiting on a [`put!`](@ref). +For unbuffered channels, return `true` if there are tasks waiting on a [`put!`](@ref). # Examples @@ -559,6 +559,47 @@ function n_avail(c::Channel) @atomic :monotonic c.n_avail_items end +""" + isfull(c::Channel) + +Determines if a [`Channel`](@ref) is full, in the sense +that calling `put!(c, some_value)` would have blocked. +Returns immediately, does not block. + +Note that it may frequently be the case that `put!` will +not block after this returns `true`. Users must take +precautions not to accidentally create live-lock bugs +in their code by calling this method, as these are +generally harder to debug than deadlocks. It is also +possible that `put!` will block after this call +returns `false`, if there are multiple producer +tasks calling `put!` in parallel. + +# Examples + +Buffered channel: +```jldoctest +julia> c = Channel(1); # capacity = 1 + +julia> isfull(c) +false + +julia> put!(c, 1); + +julia> isfull(c) +true +``` + +Unbuffered channel: +```jldoctest +julia> c = Channel(); # capacity = 0 + +julia> isfull(c) # unbuffered channel is always full +true +``` +""" +isfull(c::Channel) = n_avail(c) ≥ c.sz_max + lock(c::Channel) = lock(c.cond_take) lock(f, c::Channel) = lock(f, c.cond_take) unlock(c::Channel) = unlock(c.cond_take) diff --git a/base/exports.jl b/base/exports.jl index 7f3ad451bcc83..cb4acce27c66d 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -719,6 +719,7 @@ export # channels take!, put!, + isfull, isready, fetch, bind, diff --git a/doc/src/base/parallel.md b/doc/src/base/parallel.md index 5798da54a5cde..7605d4e6c9e7e 100644 --- a/doc/src/base/parallel.md +++ b/doc/src/base/parallel.md @@ -62,6 +62,7 @@ Base.Channel Base.Channel(::Function) Base.put!(::Channel, ::Any) Base.take!(::Channel) +Base.isfull(::Channel) Base.isready(::Channel) Base.fetch(::Channel) Base.close(::Channel) diff --git a/doc/src/manual/asynchronous-programming.md b/doc/src/manual/asynchronous-programming.md index 5b43ba971ee1c..15db6eda5f807 100644 --- a/doc/src/manual/asynchronous-programming.md +++ b/doc/src/manual/asynchronous-programming.md @@ -194,10 +194,11 @@ A channel can be visualized as a pipe, i.e., it has a write end and a read end : to the maximum number of elements that can be held in the channel at any time. For example, `Channel(32)` creates a channel that can hold a maximum of 32 objects of any type. A `Channel{MyType}(64)` can hold up to 64 objects of `MyType` at any time. - * If a [`Channel`](@ref) is empty, readers (on a [`take!`](@ref) call) will block until data is available. - * If a [`Channel`](@ref) is full, writers (on a [`put!`](@ref) call) will block until space becomes available. + * If a [`Channel`](@ref) is empty, readers (on a [`take!`](@ref) call) will block until data is available (see [`isempty`](@ref)). + * If a [`Channel`](@ref) is full, writers (on a [`put!`](@ref) call) will block until space becomes available (see [`isfull`](@ref)). * [`isready`](@ref) tests for the presence of any object in the channel, while [`wait`](@ref) waits for an object to become available. + * Note that if another task is currently waiting to `put!` an object into a channel, a channel can have more items available than its capacity. * A [`Channel`](@ref) is in an open state initially. This means that it can be read from and written to freely via [`take!`](@ref) and [`put!`](@ref) calls. [`close`](@ref) closes a [`Channel`](@ref). On a closed [`Channel`](@ref), [`put!`](@ref) will fail. For example: diff --git a/test/channels.jl b/test/channels.jl index 5633d9480d0b8..25b5185c1a336 100644 --- a/test/channels.jl +++ b/test/channels.jl @@ -40,6 +40,8 @@ end c = Channel() @test eltype(c) == Any @test c.sz_max == 0 + @test isempty(c) == true # Nothing in it + @test isfull(c) == true # But no more room c = Channel(1) @test eltype(c) == Any @@ -49,6 +51,11 @@ end @test isready(c) == false @test eltype(Channel(1.0)) == Any + c = Channel(1) + @test isfull(c) == false + put!(c, 1) + @test isfull(c) == true + c = Channel{Int}(1) @test eltype(c) == Int @test_throws MethodError put!(c, "Hello") From fbcc99a60a3130ad1e28adc841e793d66c7e1367 Mon Sep 17 00:00:00 2001 From: Sukera <11753998+Seelengrab@users.noreply.github.com> Date: Fri, 1 Mar 2024 00:04:08 +0100 Subject: [PATCH 17/33] Add documentation about failure cases of trig functions & return types (#50855) --- base/math.jl | 59 +++++++++++++++++++++---------- base/special/trig.jl | 82 +++++++++++++++++++++++++++++++------------- 2 files changed, 99 insertions(+), 42 deletions(-) diff --git a/base/math.jl b/base/math.jl index 6ed69188371dd..71a4ef9f34882 100644 --- a/base/math.jl +++ b/base/math.jl @@ -359,7 +359,7 @@ log(b::T, x::T) where {T<:Number} = log(x)/log(b) """ log(b,x) -Compute the base `b` logarithm of `x`. Throws [`DomainError`](@ref) for negative +Compute the base `b` logarithm of `x`. Throw a [`DomainError`](@ref) for negative [`Real`](@ref) arguments. # Examples @@ -408,6 +408,8 @@ const libm = Base.libm_name sinh(x) Compute hyperbolic sine of `x`. + +See also [`sin`](@ref). """ sinh(x::Number) @@ -415,6 +417,8 @@ sinh(x::Number) cosh(x) Compute hyperbolic cosine of `x`. + +See also [`cos`](@ref). """ cosh(x::Number) @@ -493,10 +497,12 @@ asinh(x::Number) # functions that return NaN on non-NaN argument for domain error """ - sin(x) + sin(x::T) where {T <: Number} -> float(T) Compute sine of `x`, where `x` is in radians. +Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. + See also [`sind`](@ref), [`sinpi`](@ref), [`sincos`](@ref), [`cis`](@ref), [`asin`](@ref). # Examples @@ -524,26 +530,34 @@ julia> round(exp(im*pi/6), digits=3) sin(x::Number) """ - cos(x) + cos(x::T) where {T <: Number} -> float(T) Compute cosine of `x`, where `x` is in radians. +Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. + See also [`cosd`](@ref), [`cospi`](@ref), [`sincos`](@ref), [`cis`](@ref). """ cos(x::Number) """ - tan(x) + tan(x::T) where {T <: Number} -> float(T) Compute tangent of `x`, where `x` is in radians. + +Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. + +See also [`tanh`](@ref). """ tan(x::Number) """ - asin(x) + asin(x::T) where {T <: Number} -> float(T) Compute the inverse sine of `x`, where the output is in radians. +Return a `T(NaN)` if `isnan(x)`. + See also [`asind`](@ref) for output in degrees. # Examples @@ -558,9 +572,11 @@ julia> asind.((0, 1/2, 1)) asin(x::Number) """ - acos(x) + acos(x::T) where {T <: Number} -> float(T) Compute the inverse cosine of `x`, where the output is in radians + +Return a `T(NaN)` if `isnan(x)`. """ acos(x::Number) @@ -583,9 +599,12 @@ atanh(x::Number) Compute the natural logarithm of `x`. -Throws [`DomainError`](@ref) for negative [`Real`](@ref) arguments. -Use complex arguments to obtain complex results. -Has a branch cut along the negative real axis, for which `-0.0im` is taken to be below the axis. +Throw a [`DomainError`](@ref) for negative [`Real`](@ref) arguments. +Use [`Complex`](@ref) arguments to obtain [`Complex`](@ref) results. + +!!! note "Branch cut" + `log` has a branch cut along the negative real axis; `-0.0im` is taken + to be below the axis. See also [`ℯ`](@ref), [`log1p`](@ref), [`log2`](@ref), [`log10`](@ref). @@ -619,7 +638,7 @@ log(x::Number) """ log2(x) -Compute the logarithm of `x` to base 2. Throws [`DomainError`](@ref) for negative +Compute the logarithm of `x` to base 2. Throw a [`DomainError`](@ref) for negative [`Real`](@ref) arguments. See also: [`exp2`](@ref), [`ldexp`](@ref), [`ispow2`](@ref). @@ -652,7 +671,7 @@ log2(x) log10(x) Compute the logarithm of `x` to base 10. -Throws [`DomainError`](@ref) for negative [`Real`](@ref) arguments. +Throw a [`DomainError`](@ref) for negative [`Real`](@ref) arguments. # Examples ```jldoctest; filter = r"Stacktrace:(\\n \\[[0-9]+\\].*)*" @@ -675,7 +694,7 @@ log10(x) """ log1p(x) -Accurate natural logarithm of `1+x`. Throws [`DomainError`](@ref) for [`Real`](@ref) +Accurate natural logarithm of `1+x`. Throw a [`DomainError`](@ref) for [`Real`](@ref) arguments less than -1. # Examples @@ -706,12 +725,15 @@ end Return ``\\sqrt{x}``. -Throws [`DomainError`](@ref) for negative [`Real`](@ref) arguments. -Use complex negative arguments instead. Note that `sqrt` has a branch cut -along the negative real axis. +Throw a [`DomainError`](@ref) for negative [`Real`](@ref) arguments. +Use [`Complex`](@ref) negative arguments instead to obtain a [`Complex`](@ref) result. The prefix operator `√` is equivalent to `sqrt`. +!!! note "Branch cut" + `sqrt` has a branch cut along the negative real axis; `-0.0im` is taken + to be below the axis. + See also: [`hypot`](@ref). # Examples @@ -1005,7 +1027,8 @@ ldexp(x::Float16, q::Integer) = Float16(ldexp(Float32(x), q)) """ exponent(x::Real) -> Int -Returns the largest integer `y` such that `2^y ≤ abs(x)`. +Return the largest integer `y` such that `2^y ≤ abs(x)`. +For a normalized floating-point number `x`, this corresponds to the exponent of `x`. Throws a `DomainError` when `x` is zero, infinite, or [`NaN`](@ref). For any other non-subnormal floating-point number `x`, this corresponds to the exponent bits of `x`. @@ -1330,8 +1353,8 @@ end function add22condh(xh::Float64, xl::Float64, yh::Float64, yl::Float64) # This algorithm, due to Dekker, computes the sum of two - # double-double numbers and returns the high double. References: - # [1] https://www.digizeitschriften.de/en/dms/img/?PID=GDZPPN001170007 + # double-double numbers and return the high double. References: + # [1] http://www.digizeitschriften.de/en/dms/img/?PID=GDZPPN001170007 # [2] https://doi.org/10.1007/BF01397083 r = xh+yh s = (abs(xh) > abs(yh)) ? (xh-r+yh+yl+xl) : (yh-r+xh+xl+yl) diff --git a/base/special/trig.jl b/base/special/trig.jl index 99b4ca91608f2..15ecb0ca0492f 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -165,11 +165,13 @@ end @noinline sincos_domain_error(x) = throw(DomainError(x, "sincos(x) is only defined for finite x.")) """ - sincos(x) + sincos(x::T) where T -> float(T) Simultaneously compute the sine and cosine of `x`, where `x` is in radians, returning a tuple `(sine, cosine)`. +Throw a [`DomainError`](@ref) if `isinf(x)`, return a `(T(NaN), T(NaN))` if `isnan(x)`. + See also [`cis`](@ref), [`sincospi`](@ref), [`sincosd`](@ref). """ function sincos(x::T) where T<:Union{Float32, Float64} @@ -783,17 +785,19 @@ end end """ - sinpi(x) + sinpi(x::T) where T -> float(T) Compute ``\\sin(\\pi x)`` more accurately than `sin(pi*x)`, especially for large `x`. +Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. + See also [`sind`](@ref), [`cospi`](@ref), [`sincospi`](@ref). """ function sinpi(_x::T) where T<:IEEEFloat x = abs(_x) if !isfinite(x) isnan(x) && return x - throw(DomainError(x, "`x` cannot be infinite.")) + throw(DomainError(x, "`sinpi(x)` is only defined for finite `x`.")) end # For large x, answers are all 1 or zero. x >= maxintfloat(T) && return copysign(zero(T), _x) @@ -814,15 +818,19 @@ function sinpi(_x::T) where T<:IEEEFloat return ifelse(signbit(_x), -res, res) end """ - cospi(x) + cospi(x::T) where T -> float(T) Compute ``\\cos(\\pi x)`` more accurately than `cos(pi*x)`, especially for large `x`. + +Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. + +See also: [`cispi`](@ref), [`sincosd`](@ref), [`cospi`](@ref). """ function cospi(x::T) where T<:IEEEFloat x = abs(x) if !isfinite(x) isnan(x) && return x - throw(DomainError(x, "`x` cannot be infinite.")) + throw(DomainError(x, "`cospi(x)` is only defined for finite `x`.")) end # For large x, answers are all 1 or zero. x >= maxintfloat(T) && return one(T) @@ -842,11 +850,13 @@ function cospi(x::T) where T<:IEEEFloat end end """ - sincospi(x) + sincospi(x::T) where T -> float(T) Simultaneously compute [`sinpi(x)`](@ref) and [`cospi(x)`](@ref) (the sine and cosine of `π*x`, where `x` is in radians), returning a tuple `(sine, cosine)`. +Throw a [`DomainError`](@ref) if `isinf(x)`, return a `(T(NaN), T(NaN))` tuple if `isnan(x)`. + !!! compat "Julia 1.6" This function requires Julia 1.6 or later. @@ -856,7 +866,7 @@ function sincospi(_x::T) where T<:IEEEFloat x = abs(_x) if !isfinite(x) isnan(x) && return x, x - throw(DomainError(x, "`x` cannot be infinite.")) + throw(DomainError(x, "`sincospi(x)` is only defined for finite `x`.")) end # For large x, answers are all 1 or zero. x >= maxintfloat(T) && return (copysign(zero(T), _x), one(T)) @@ -880,10 +890,12 @@ function sincospi(_x::T) where T<:IEEEFloat end """ - tanpi(x) + tanpi(x::T) where T -> float(T) Compute ``\\tan(\\pi x)`` more accurately than `tan(pi*x)`, especially for large `x`. +Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. + !!! compat "Julia 1.10" This function requires at least Julia 1.10. @@ -895,7 +907,7 @@ function tanpi(_x::T) where T<:IEEEFloat x = abs(_x) if !isfinite(x) isnan(x) && return x - throw(DomainError(x, "`x` cannot be infinite.")) + throw(DomainError(x, "`tanpi(x)` is only defined for finite `x`.")) end # For large x, answers are all zero. # All integer values for floats larger than maxintfloat are even. @@ -1063,10 +1075,12 @@ isinf_real(x::Complex) = isinf(real(x)) && isfinite(imag(x)) isinf_real(x::Number) = false """ - sinc(x) + sinc(x::T) where {T <: Number} -> float(T) Compute normalized sinc function ``\\operatorname{sinc}(x) = \\sin(\\pi x) / (\\pi x)`` if ``x \\neq 0``, and ``1`` if ``x = 0``. +Return a `T(NaN)` if `isnan(x)`. + See also [`cosc`](@ref), its derivative. """ sinc(x::Number) = _sinc(float(x)) @@ -1080,11 +1094,13 @@ _sinc(x::Float16) = Float16(_sinc(Float32(x))) _sinc(x::ComplexF16) = ComplexF16(_sinc(ComplexF32(x))) """ - cosc(x) + cosc(x::T) where {T <: Number} -> float(T) Compute ``\\cos(\\pi x) / x - \\sin(\\pi x) / (\\pi x^2)`` if ``x \\neq 0``, and ``0`` if ``x = 0``. This is the derivative of `sinc(x)`. +Return a `T(NaN)` if `isnan(x)`. + See also [`sinc`](@ref). """ cosc(x::Number) = _cosc(float(x)) @@ -1129,19 +1145,25 @@ for (finv, f, finvh, fh, finvd, fd, fn) in ((:sec, :cos, :sech, :cosh, :secd, :c dname = string(finvd) @eval begin @doc """ - $($name)(x) + $($name)(x::T) where {T <: Number} -> float(T) Compute the $($fn) of `x`, where `x` is in radians. + + Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. """ ($finv)(z::Number) = inv(($f)(z)) @doc """ - $($hname)(x) + $($hname)(x::T) where {T <: Number} -> float(T) Compute the hyperbolic $($fn) of `x`. + + Return a `T(NaN)` if `isnan(x)`. """ ($finvh)(z::Number) = inv(($fh)(z)) @doc """ - $($dname)(x) + $($dname)(x::T) where {T <: Number} -> float(T) Compute the $($fn) of `x`, where `x` is in degrees. + + Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. """ ($finvd)(z::Number) = inv(($fd)(z)) end end @@ -1153,11 +1175,15 @@ for (tfa, tfainv, hfa, hfainv, fn) in ((:asec, :acos, :asech, :acosh, "secant"), hname = string(hfa) @eval begin @doc """ - $($tname)(x) - Compute the inverse $($fn) of `x`, where the output is in radians. """ ($tfa)(y::Number) = ($tfainv)(inv(y)) + $($tname)(x::T) where {T <: Number} -> float(T) + + Compute the inverse $($fn) of `x`, where the output is in radians. + """ ($tfa)(y::Number) = ($tfainv)(inv(y)) @doc """ - $($hname)(x) - Compute the inverse hyperbolic $($fn) of `x`. """ ($hfa)(y::Number) = ($hfainv)(inv(y)) + $($hname)(x::T) where {T <: Number} -> float(T) + + Compute the inverse hyperbolic $($fn) of `x`. + """ ($hfa)(y::Number) = ($hfainv)(inv(y)) end end @@ -1182,7 +1208,7 @@ deg2rad_ext(x::Real) = deg2rad(x) # Fallback function sind(x::Real) if isinf(x) - return throw(DomainError(x, "`x` cannot be infinite.")) + return throw(DomainError(x, "`sind(x)` is only defined for finite `x`.")) elseif isnan(x) return x end @@ -1213,7 +1239,7 @@ end function cosd(x::Real) if isinf(x) - return throw(DomainError(x, "`x` cannot be infinite.")) + return throw(DomainError(x, "`cosd(x)` is only defined for finite `x`.")) elseif isnan(x) return x end @@ -1240,10 +1266,12 @@ end tand(x::Real) = sind(x) / cosd(x) """ - sincosd(x) + sincosd(x::T) where T -> float(T) Simultaneously compute the sine and cosine of `x`, where `x` is in degrees. +Throw a [`DomainError`](@ref) if `isinf(x)`, return a `(T(NaN), T(NaN))` tuple if `isnan(x)`. + !!! compat "Julia 1.3" This function requires at least Julia 1.3. """ @@ -1258,11 +1286,13 @@ for (fd, f, fn) in ((:sind, :sin, "sine"), (:cosd, :cos, "cosine"), (:tand, :tan name = string(fd) @eval begin @doc """ - $($name)(x) + $($name)(x::T) where T -> float(T) Compute $($fn) of `x`, where `x` is in $($un). If `x` is a matrix, `x` needs to be a square matrix. + Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. + !!! compat "Julia 1.7" Matrix arguments require Julia 1.7 or later. """ ($fd)(x) = ($f)(($fu).(x)) @@ -1290,11 +1320,15 @@ for (fd, f, fn) in ((:asind, :asin, "sine"), (:acosd, :acos, "cosine"), end """ - atand(y) - atand(y,x) + atand(y::T) where T -> float(T) + atand(y::T, x::S) where {T,S} -> promote_type(T,S) + atand(y::AbstractMatrix{T}) where T -> AbstractMatrix{Complex{float(T)}} Compute the inverse tangent of `y` or `y/x`, respectively, where the output is in degrees. +Return a `NaN` if `isnan(y)` or `isnan(x)`. The returned `NaN` is either a `T` in the single +argument version, or a `promote_type(T,S)` in the two argument version. + !!! compat "Julia 1.7" The one-argument method supports square matrix arguments as of Julia 1.7. """ From 989c4db50e0ca59b890af2f1800004ecf91c0faf Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 29 Feb 2024 18:04:39 -0500 Subject: [PATCH 18/33] add _readdirx for returning more object info gathered during dir scan (#53377) --- base/file.jl | 85 ++++++++++++++++++++++++++++++++++++++++++++++++---- test/file.jl | 8 +++++ test/misc.jl | 5 ++-- 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/base/file.jl b/base/file.jl index 6c4f7bc5fb8f3..659c39d7415f8 100644 --- a/base/file.jl +++ b/base/file.jl @@ -929,7 +929,79 @@ julia> readdir(abspath("base"), join=true) "/home/JuliaUser/dev/julia/base/weakkeydict.jl" ``` """ -function readdir(dir::AbstractString; join::Bool=false, sort::Bool=true) +readdir(; join::Bool=false, kwargs...) = readdir(join ? pwd() : "."; join, kwargs...)::Vector{String} +readdir(dir::AbstractString; kwargs...) = _readdir(dir; return_objects=false, kwargs...)::Vector{String} + +# this might be better as an Enum but they're not available here +# UV_DIRENT_T +const UV_DIRENT_UNKNOWN = Cint(0) +const UV_DIRENT_FILE = Cint(1) +const UV_DIRENT_DIR = Cint(2) +const UV_DIRENT_LINK = Cint(3) +const UV_DIRENT_FIFO = Cint(4) +const UV_DIRENT_SOCKET = Cint(5) +const UV_DIRENT_CHAR = Cint(6) +const UV_DIRENT_BLOCK = Cint(7) + +""" + DirEntry + +A type representing a filesystem entry that contains the name of the entry, the directory, and +the raw type of the entry. The full path of the entry can be obtained lazily by accessing the +`path` field. The type of the entry can be checked for by calling [`isfile`](@ref), [`isdir`](@ref), +[`islink`](@ref), [`isfifo`](@ref), [`issocket`](@ref), [`ischardev`](@ref), and [`isblockdev`](@ref) +""" +struct DirEntry + dir::String + name::String + rawtype::Cint +end +function Base.getproperty(obj::DirEntry, p::Symbol) + if p === :path + return joinpath(obj.dir, obj.name) + else + return getfield(obj, p) + end +end +Base.propertynames(::DirEntry) = (:dir, :name, :path, :rawtype) +Base.isless(a::DirEntry, b::DirEntry) = a.dir == b.dir ? isless(a.name, b.name) : isless(a.dir, b.dir) +Base.hash(o::DirEntry, h::UInt) = hash(o.dir, hash(o.name, hash(o.rawtype, h))) +Base.:(==)(a::DirEntry, b::DirEntry) = a.name == b.name && a.dir == b.dir && a.rawtype == b.rawtype +joinpath(obj::DirEntry, args...) = joinpath(obj.path, args...) +isunknown(obj::DirEntry) = obj.rawtype == UV_DIRENT_UNKNOWN +islink(obj::DirEntry) = isunknown(obj) ? islink(obj.path) : obj.rawtype == UV_DIRENT_LINK +isfile(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isfile(obj.path) : obj.rawtype == UV_DIRENT_FILE +isdir(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isdir(obj.path) : obj.rawtype == UV_DIRENT_DIR +isfifo(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isfifo(obj.path) : obj.rawtype == UV_DIRENT_FIFO +issocket(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? issocket(obj.path) : obj.rawtype == UV_DIRENT_SOCKET +ischardev(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? ischardev(obj.path) : obj.rawtype == UV_DIRENT_CHAR +isblockdev(obj::DirEntry) = (isunknown(obj) || islink(obj)) ? isblockdev(obj.path) : obj.rawtype == UV_DIRENT_BLOCK +realpath(obj::DirEntry) = realpath(obj.path) + +""" + _readdirx(dir::AbstractString=pwd(); sort::Bool = true) -> Vector{DirEntry} + +Return a vector of [`DirEntry`](@ref) objects representing the contents of the directory `dir`, +or the current working directory if not given. If `sort` is true, the returned vector is +sorted by name. + +Unlike [`readdir`](@ref), `_readdirx` returns [`DirEntry`](@ref) objects, which contain the name of the +file, the directory it is in, and the type of the file which is determined during the +directory scan. This means that calls to [`isfile`](@ref), [`isdir`](@ref), [`islink`](@ref), [`isfifo`](@ref), +[`issocket`](@ref), [`ischardev`](@ref), and [`isblockdev`](@ref) can be made on the +returned objects without further stat calls. However, for some filesystems, the type of the file +cannot be determined without a stat call. In these cases the `rawtype` field of the [`DirEntry`](@ref)) +object will be 0 (`UV_DIRENT_UNKNOWN`) and [`isfile`](@ref) etc. will fall back to a `stat` call. + +```julia +for obj in _readdirx() + isfile(obj) && println("\$(obj.name) is a file with path \$(obj.path)") +end +``` +""" +_readdirx(dir::AbstractString=pwd(); sort::Bool=true) = _readdir(dir; return_objects=true, sort)::Vector{DirEntry} + +function _readdir(dir::AbstractString; return_objects::Bool=false, join::Bool=false, sort::Bool=true) # Allocate space for uv_fs_t struct req = Libc.malloc(_sizeof_uv_fs) try @@ -939,11 +1011,16 @@ function readdir(dir::AbstractString; join::Bool=false, sort::Bool=true) err < 0 && uv_error("readdir($(repr(dir)))", err) # iterate the listing into entries - entries = String[] + entries = return_objects ? DirEntry[] : String[] ent = Ref{uv_dirent_t}() while Base.UV_EOF != ccall(:uv_fs_scandir_next, Cint, (Ptr{Cvoid}, Ptr{uv_dirent_t}), req, ent) name = unsafe_string(ent[].name) - push!(entries, join ? joinpath(dir, name) : name) + if return_objects + rawtype = ent[].typ + push!(entries, DirEntry(dir, name, rawtype)) + else + push!(entries, join ? joinpath(dir, name) : name) + end end # Clean up the request string @@ -957,8 +1034,6 @@ function readdir(dir::AbstractString; join::Bool=false, sort::Bool=true) Libc.free(req) end end -readdir(; join::Bool=false, sort::Bool=true) = - readdir(join ? pwd() : ".", join=join, sort=sort) """ walkdir(dir; topdown=true, follow_symlinks=false, onerror=throw) diff --git a/test/file.jl b/test/file.jl index 1ce7a5cdb440b..a5ab29a3bf8bc 100644 --- a/test/file.jl +++ b/test/file.jl @@ -31,6 +31,8 @@ if !Sys.iswindows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER symlink(subdir, dirlink) @test stat(dirlink) == stat(subdir) @test readdir(dirlink) == readdir(subdir) + @test map(o->o.names, Base.Filesystem._readdirx(dirlink)) == map(o->o.names, Base.Filesystem._readdirx(subdir)) + @test realpath.(Base.Filesystem._readdirx(dirlink)) == realpath.(Base.Filesystem._readdirx(subdir)) # relative link relsubdirlink = joinpath(subdir, "rel_subdirlink") @@ -38,6 +40,7 @@ if !Sys.iswindows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER symlink(reldir, relsubdirlink) @test stat(relsubdirlink) == stat(subdir2) @test readdir(relsubdirlink) == readdir(subdir2) + @test Base.Filesystem._readdirx(relsubdirlink) == Base.Filesystem._readdirx(subdir2) # creation of symlink to directory that does not yet exist new_dir = joinpath(subdir, "new_dir") @@ -56,6 +59,7 @@ if !Sys.iswindows() || Sys.windows_version() >= Sys.WINDOWS_VISTA_VER mkdir(new_dir) touch(foo_file) @test readdir(new_dir) == readdir(nedlink) + @test realpath.(Base.Filesystem._readdirx(new_dir)) == realpath.(Base.Filesystem._readdirx(nedlink)) rm(foo_file) rm(new_dir) @@ -1441,6 +1445,10 @@ rm(dirwalk, recursive=true) touch(randstring()) end @test issorted(readdir()) + @test issorted(Base.Filesystem._readdirx()) + @test map(o->o.name, Base.Filesystem._readdirx()) == readdir() + @test map(o->o.path, Base.Filesystem._readdirx()) == readdir(join=true) + @test count(isfile, readdir(join=true)) == count(isfile, Base.Filesystem._readdirx()) end end end diff --git a/test/misc.jl b/test/misc.jl index 6597ecf8a8428..e870c7f491c13 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -1363,9 +1363,10 @@ end @test isdefined(KwdefWithEsc_TestModule, :Struct) @testset "exports of modules" begin - for (_, mod) in Base.loaded_modules + @testset "$mod" for (_, mod) in Base.loaded_modules mod === Main && continue # Main exports everything - for v in names(mod) + @testset "$v" for v in names(mod) + isdefined(mod, v) || @error "missing $v in $mod" @test isdefined(mod, v) end end From 2501e37e31a763ba64fc95c1e48b2bd35cb8ebb1 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 1 Mar 2024 09:43:00 +0900 Subject: [PATCH 19/33] post-opt analysis: fix conditional successor visitation logic (#53518) Previously `conditional_successors_may_throw` performed post-domination analysis not on the initially specified `bb` (which was given as the argument), but on those BBs being analyzed that were popped from the work-queue at the time. As a result, there were cases where not all conditional successors were visited. fixes #53508 --- base/compiler/optimize.jl | 21 ++++++++++++++------- test/compiler/effects.jl | 5 +++++ test/compiler/ssair.jl | 29 +++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index d3c680f9308d3..1615781727669 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -531,17 +531,22 @@ function any_stmt_may_throw(ir::IRCode, bb::Int) return false end -function conditional_successors_may_throw(lazypostdomtree::LazyPostDomtree, ir::IRCode, bb::Int) +function visit_conditional_successors(callback, lazypostdomtree::LazyPostDomtree, ir::IRCode, bb::Int) visited = BitSet((bb,)) worklist = Int[bb] while !isempty(worklist) - thisbb = pop!(worklist) + thisbb = popfirst!(worklist) for succ in ir.cfg.blocks[thisbb].succs succ in visited && continue push!(visited, succ) - postdominates(get!(lazypostdomtree), succ, thisbb) && continue - any_stmt_may_throw(ir, succ) && return true - push!(worklist, succ) + if postdominates(get!(lazypostdomtree), succ, bb) + # this successor is not conditional, so no need to visit it further + continue + elseif callback(succ) + return true + else + push!(worklist, succ) + end end end return false @@ -833,8 +838,10 @@ function ((; sv)::ScanStmt)(inst::Instruction, lstmt::Int, bb::Int) # inconsistent region. if !sv.result.ipo_effects.terminates sv.all_retpaths_consistent = false - elseif conditional_successors_may_throw(sv.lazypostdomtree, sv.ir, bb) - # Check if there are potential throws that require + elseif visit_conditional_successors(sv.lazypostdomtree, sv.ir, bb) do succ::Int + return any_stmt_may_throw(sv.ir, succ) + end + # check if this `GotoIfNot` leads to conditional throws, which taints consistency sv.all_retpaths_consistent = false else (; cfg, domtree) = get!(sv.lazyagdomtree) diff --git a/test/compiler/effects.jl b/test/compiler/effects.jl index f5ecb8a0d347c..fa70c8de9d853 100644 --- a/test/compiler/effects.jl +++ b/test/compiler/effects.jl @@ -1382,3 +1382,8 @@ let; Base.Experimental.@force_compile; func52843(); end # pointerref nothrow for invalid pointer @test !Core.Compiler.intrinsic_nothrow(Core.Intrinsics.pointerref, Any[Type{Ptr{Vector{Int64}}}, Int, Int]) @test !Core.Compiler.intrinsic_nothrow(Core.Intrinsics.pointerref, Any[Type{Ptr{T}} where T, Int, Int]) + +# post-opt :consistent-cy analysis correctness +# https://github.com/JuliaLang/julia/issues/53508 +@test !Core.Compiler.is_consistent(Base.infer_effects(getindex, (UnitRange{Int},Int))) +@test !Core.Compiler.is_consistent(Base.infer_effects(getindex, (Base.OneTo{Int},Int))) diff --git a/test/compiler/ssair.jl b/test/compiler/ssair.jl index 3a90ee6308d53..0a53dec80f732 100644 --- a/test/compiler/ssair.jl +++ b/test/compiler/ssair.jl @@ -238,6 +238,35 @@ let ci = code_lowered(()->@isdefined(_not_def_37919_), ())[1] @test Core.Compiler.verify_ir(ir) === nothing end +let code = Any[ + # block 1 + GotoIfNot(Argument(2), 4), + # block 2 + GotoNode(3), + # block 3 + Expr(:call, throw, "potential throw"), + # block 4 + Expr(:call, Core.Intrinsics.add_int, Argument(3), Argument(4)), + GotoNode(6), + # block 5 + ReturnNode(SSAValue(4)) + ] + ir = make_ircode(code; slottypes=Any[Any,Bool,Int,Int]) + lazypostdomtree = Core.Compiler.LazyPostDomtree(ir) + visited = BitSet() + @test !Core.Compiler.visit_conditional_successors(lazypostdomtree, ir, #=bb=#1) do succ::Int + push!(visited, succ) + return false + end + @test 2 ∈ visited + @test 3 ∈ visited + @test 4 ∉ visited + @test 5 ∉ visited + oc = Core.OpaqueClosure(ir) + @test oc(false, 1, 1) == 2 + @test_throws "potential throw" oc(true, 1, 1) +end + # Test dynamic update of domtree with edge insertions and deletions in the # following CFG: # From 95f54c4dcb58e4f15f53f8a13a930db8e035ea89 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 29 Feb 2024 21:38:38 -0500 Subject: [PATCH 20/33] fix InteractiveUtils call in Base.runtests on failure (#53525) Noticed in CI that `Base.runtests(["fail"])` fails here instead of throwing the correct error later, since #53326. --- base/util.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/base/util.jl b/base/util.jl index 847d89e895af4..c91f04e8baf96 100644 --- a/base/util.jl +++ b/base/util.jl @@ -694,11 +694,9 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), nothing catch buf = PipeBuffer() - original_load_path = copy(Base.LOAD_PATH); empty!(Base.LOAD_PATH); pushfirst!(Base.LOAD_PATH, "@stdlib") - let InteractiveUtils = Base.require_stdlib(Base, :InteractiveUtils) + let InteractiveUtils = Base.require_stdlib(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")) @invokelatest InteractiveUtils.versioninfo(buf) end - empty!(Base.LOAD_PATH); append!(Base.LOAD_PATH, original_load_path) error("A test has failed. Please submit a bug report (https://github.com/JuliaLang/julia/issues)\n" * "including error messages above and the output of versioninfo():\n$(read(buf, String))") end From 136f018e7473e48ebb90ed5bc8dc6ad3dd109d6f Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 1 Mar 2024 00:32:36 -0600 Subject: [PATCH 21/33] Allow GlobalRef and module qualified names in macro definitions (#53535) The following is currently an error: ``` julia> module MyMacroModule macro mymacro end end Main.MyMacroModule julia> macro MyMacroModule.mymacro() 1 end ERROR: syntax: invalid macro definition around REPL[2]:1 Stacktrace: [1] top-level scope @ REPL[2]:1 ``` Discussing with Jeff, we didn't think there was any good reason not to allow this, just a missing case in lowering. It's probably not particularly useful (unlike the corresponding case for functions that is used all the time), but it came up in writing a test case for #53515. --- src/julia-parser.scm | 12 ++++++++++-- src/julia-syntax.scm | 11 +++++++---- test/syntax.jl | 13 +++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/julia-parser.scm b/src/julia-parser.scm index fb20717add65d..891a26bb0ea49 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -2610,15 +2610,23 @@ (define (valid-modref? e) (and (length= e 3) (eq? (car e) '|.|) (pair? (caddr e)) - (eq? (car (caddr e)) 'quote) (symbol? (cadr (caddr e))) + (or (eq? (car (caddr e)) 'quote) + (eq? (car (caddr e)) 'inert)) + (symbol? (cadr (caddr e))) (or (symbol? (cadr e)) (valid-modref? (cadr e))))) (define (macroify-name e . suffixes) (cond ((symbol? e) (symbol (apply string #\@ e suffixes))) + ((and (pair? e) (eq? (car e) 'quote)) + `(quote ,(apply macroify-name (cadr e) suffixes))) + ((and (pair? e) (eq? (car e) 'inert)) + `(inert ,(apply macroify-name (cadr e) suffixes))) + ((globalref? e) + `(globalref ,(cadr e) ,(apply macroify-name (caddr e) suffixes))) ((valid-modref? e) `(|.| ,(cadr e) - (quote ,(apply macroify-name (cadr (caddr e)) suffixes)))) + ,(apply macroify-name (caddr e) suffixes))) (else (error (string "invalid macro usage \"@(" (deparse e) ")\"" ))))) (define (macroify-call s call startloc) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 7d9f8711ec714..d3d4f7dfe3a56 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1347,15 +1347,18 @@ (else (error "invalid let syntax")))) (else (error "invalid let syntax"))))))))) +(define (valid-macro-def-name? e) + (or (symbol? e) (valid-modref? e) (globalref? e))) + (define (expand-macro-def e) (cond ((and (pair? (cadr e)) (eq? (car (cadr e)) 'call) - (symbol? (cadr (cadr e)))) + (valid-macro-def-name? (cadr (cadr e)))) (let ((anames (remove-empty-parameters (cddr (cadr e))))) (if (has-parameters? anames) (error "macros cannot accept keyword arguments")) (expand-forms - `(function (call ,(symbol (string #\@ (cadr (cadr e)))) + `(function (call ,(macroify-name (cadr (cadr e))) (|::| __source__ (core LineNumberNode)) (|::| __module__ (core Module)) ,@(map (lambda (v) @@ -1364,8 +1367,8 @@ v)) anames)) ,@(cddr e))))) - ((and (length= e 2) (symbol? (cadr e))) - (expand-forms `(function ,(symbol (string #\@ (cadr e)))))) + ((and (length= e 2) (valid-macro-def-name? (cadr e))) + (expand-forms `(function ,(macroify-name (cadr e))))) (else (error "invalid macro definition")))) diff --git a/test/syntax.jl b/test/syntax.jl index 25d3e7282818a..423d1e3d04bfb 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -3637,3 +3637,16 @@ end @test array == [7] @test execs == 4 end + +# Allow GlobalRefs in macro definition +module MyMacroModule + macro mymacro end +end +macro MyMacroModule.mymacro() + 1 +end +@eval macro $(GlobalRef(MyMacroModule, :mymacro))(x) + 2 +end +@test (@MyMacroModule.mymacro) == 1 +@test (@MyMacroModule.mymacro(a)) == 2 From 0918cf12d549a53da02265ec1005aec7fce87e5f Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 1 Mar 2024 00:34:00 -0600 Subject: [PATCH 22/33] Allow sysimage build without the doc system (#53533) The earliest bootstrapping code has a definition of `atdoc` that is just supposed to ignore the doc string and pass the defining code through. This function is then replaced by the actual docsystem once that is available. For testing, I wanted to build the whole system image without the doc system using this boostrap definition. However, this turns out not to be possible, because there's a few doc syntax semantics that do not actually just ignore the doc string. In particular: ``` """ I am a doc for a particular signature """ foo(x::Int, y::Float64) ``` Does not acutally result in a call to `foo`. And similarly ``` """ I am a doc for a global binding """ MyModule.foo ``` Does not require `MyModule.foo` to actually have a value, since it only documents the binding. This PR allows both of those cases in the boostrap version of `atdoc` so that we can bootstrap without the doc system if we wanted to. --- base/boot.jl | 13 +++++++++++-- base/shell.jl | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index f8ed86b0fb6b7..e041488cdb74c 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -648,8 +648,17 @@ end macro __doc__(x) return Expr(:escape, Expr(:block, Expr(:meta, :doc), x)) end -atdoc = (source, mod, str, expr) -> Expr(:escape, expr) -atdoc!(λ) = global atdoc = λ + +isbasicdoc(@nospecialize x) = (isa(x, Expr) && x.head === :.) || isa(x, Union{QuoteNode, Symbol}) +iscallexpr(ex::Expr) = (isa(ex, Expr) && ex.head === :where) ? iscallexpr(ex.args[1]) : (isa(ex, Expr) && ex.head === :call) +iscallexpr(ex) = false +function ignoredoc(source, mod, str, expr) + (isbasicdoc(expr) || iscallexpr(expr)) && return Expr(:escape, nothing) + Expr(:escape, expr) +end + +global atdoc = ignoredoc +atdoc!(λ) = global atdoc = λ # macros for big integer syntax macro int128_str end diff --git a/base/shell.jl b/base/shell.jl index 9f15550a1aa59..137150b585d86 100644 --- a/base/shell.jl +++ b/base/shell.jl @@ -4,7 +4,7 @@ const shell_special = "#{}()[]<>|&*?~;" -@doc raw""" +(@doc raw""" rstrip_shell(s::AbstractString) Strip trailing whitespace from a shell command string, while respecting a trailing backslash followed by a space ("\\ "). @@ -16,7 +16,7 @@ julia> Base.rstrip_shell("echo 'Hello World' \\ ") julia> Base.rstrip_shell("echo 'Hello World' ") "echo 'Hello World'" ``` -""" -> +""" function rstrip_shell(s::AbstractString) c_old = nothing for (i, c) in Iterators.reverse(pairs(s)) @@ -26,7 +26,7 @@ function rstrip_shell(s::AbstractString) c_old = c end SubString(s, 1, 0) -end +end) function shell_parse(str::AbstractString, interpolate::Bool=true; special::AbstractString="", filename="none") From ac41e2a0d446de8249733d902bb774695c9d61f7 Mon Sep 17 00:00:00 2001 From: Daniel Wennberg Date: Fri, 1 Mar 2024 16:24:30 +0100 Subject: [PATCH 23/33] Fix `hypot` promotion bug (#53541) Fixes #53505 --- base/math.jl | 4 ++-- test/math.jl | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/base/math.jl b/base/math.jl index 71a4ef9f34882..c3c15505c5f8f 100644 --- a/base/math.jl +++ b/base/math.jl @@ -817,8 +817,8 @@ true ``` """ hypot(x::Number) = abs(float(x)) -hypot(x::Number, y::Number) = _hypot(promote(float(x), y)...) -hypot(x::Number, y::Number, xs::Number...) = _hypot(promote(float(x), y, xs...)) +hypot(x::Number, y::Number) = _hypot(float.(promote(x, y))...) +hypot(x::Number, y::Number, xs::Number...) = _hypot(float.(promote(x, y, xs...))) function _hypot(x, y) # preserves unit axu = abs(x) diff --git a/test/math.jl b/test/math.jl index f3c6dc5bb103f..f551bb3a5d4b7 100644 --- a/test/math.jl +++ b/test/math.jl @@ -1355,6 +1355,16 @@ end # hypot on Complex returns Real @test (@inferred hypot(3, 4im)) === 5.0 @test (@inferred hypot(3, 4im, 12)) === 13.0 + @testset "promotion, issue #53505" begin + @testset "Int,$T" for T in (Float16, Float32, Float64, BigFloat) + for args in ((3, 4), (3, 4, 12)) + for i in eachindex(args) + targs = ntuple(j -> (j == i) ? T(args[j]) : args[j], length(args)) + @test (@inferred hypot(targs...)) isa float(eltype(promote(targs...))) + end + end + end + end end struct BadFloatWrapper <: AbstractFloat From c703a1781543e7e82db981a1e49df8c7c6fe5dbb Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Fri, 1 Mar 2024 11:58:09 -0600 Subject: [PATCH 24/33] Update package naming guidelines to encourage consideration of the global namespace (#53514) Co-authored-by: Daniel Wennberg Co-authored-by: Matt Bauman --- doc/src/tutorials/creating-packages.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/src/tutorials/creating-packages.md b/doc/src/tutorials/creating-packages.md index d1f79e8c88a9d..bd5c9b32d34b3 100644 --- a/doc/src/tutorials/creating-packages.md +++ b/doc/src/tutorials/creating-packages.md @@ -605,9 +605,22 @@ may fit your package better. * `CPLEX.jl` wraps the `CPLEX` library, which can be identified easily in a web search. * `MATLAB.jl` provides an interface to call the MATLAB engine from within Julia. + 7. Avoid naming a package closely to an existing package * `Websocket` is too close to `WebSockets` and can be confusing to users. Rather use a new name such as `SimpleWebsockets`. +8. Avoid using a distinctive name that is already in use in a well known, unrelated project. + * Don't use the names `Tkinter.jl`, `TkinterGUI.jl`, etc. for a package that is unrelated + to the popular `tkinter` python package, even if it provides bindings to Tcl/Tk. + A package name of `Tkinter.jl` would only be appropriate if the package used Python's + library to accomplish its work or was spearheaded by the same community of developers. + * It's okay to name a package `HTTP.jl` even though it is unrelated to the popular rust + crate `http` because in most usages the name "http" refers to the hypertext transfer + protocol, not to the `http` rust crate. + * It's okay to name a package `OpenSSL.jl` if it provides an interface to the OpenSSL + library, even without explicit affiliation with the creators of the OpenSSL (provided + there's no copyright or trademark infringement etc.) + ## Registering packages Once a package is ready it can be registered with the [General Registry](https://github.com/JuliaRegistries/General#registering-a-package-in-general) (see also the [FAQ](https://github.com/JuliaRegistries/General#faq)). From 188e386e090fb83e89fb527d68fd23afee94c3b2 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 1 Mar 2024 18:03:41 -0500 Subject: [PATCH 25/33] use more efficient `_readdirx` for tab completion (#53540) Fixes https://github.com/JuliaLang/julia/issues/53153 --- stdlib/REPL/src/REPLCompletions.jl | 35 +++++++++++++++--------------- stdlib/REPL/src/docview.jl | 8 ++++--- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 609825b772d47..5ffa718f38604 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -8,6 +8,7 @@ using Core: Const const CC = Core.Compiler using Base.Meta using Base: propertynames, something, IdSet +using Base.Filesystem: _readdirx abstract type Completion end @@ -317,8 +318,8 @@ function cache_PATH() continue end - filesinpath = try - readdir(pathdir) + path_entries = try + _readdirx(pathdir) catch e # Bash allows dirs in PATH that can't be read, so we should as well. if isa(e, Base.IOError) || isa(e, Base.ArgumentError) @@ -328,13 +329,13 @@ function cache_PATH() rethrow() end end - for file in filesinpath + for entry in path_entries # In a perfect world, we would filter on whether the file is executable # here, or even on whether the current user can execute the file in question. try - if isfile(joinpath(pathdir, file)) - @lock PATH_cache_lock push!(PATH_cache, file) - push!(this_PATH_cache, file) + if isfile(entry) + @lock PATH_cache_lock push!(PATH_cache, entry.name) + push!(this_PATH_cache, entry.name) end catch e # `isfile()` can throw in rare cases such as when probing a @@ -378,11 +379,11 @@ function complete_path(path::AbstractString; else dir, prefix = splitdir(path) end - files = try + entries = try if isempty(dir) - readdir() + _readdirx() elseif isdir(dir) - readdir(dir) + _readdirx(dir) else return Completion[], dir, false end @@ -392,11 +393,10 @@ function complete_path(path::AbstractString; end matches = Set{String}() - for file in files - if startswith(file, prefix) - p = joinpath(dir, file) - is_dir = try isdir(p) catch ex; ex isa Base.IOError ? false : rethrow() end - push!(matches, is_dir ? file * "/" : file) + for entry in entries + if startswith(entry.name, prefix) + is_dir = try isdir(entry) catch ex; ex isa Base.IOError ? false : rethrow() end + push!(matches, is_dir ? entry.name * "/" : entry.name) end end @@ -1349,14 +1349,15 @@ function completions(string::String, pos::Int, context_module::Module=Main, shif append!(suggestions, project_deps_get_completion_candidates(s, dir)) end isdir(dir) || continue - for pname in readdir(dir) + for entry in _readdirx(dir) + pname = entry.name if pname[1] != '.' && pname != "METADATA" && pname != "REQUIRE" && startswith(pname, s) # Valid file paths are # .jl # /src/.jl # .jl/src/.jl - if isfile(joinpath(dir, pname)) + if isfile(entry) endswith(pname, ".jl") && push!(suggestions, PackageCompletion(pname[1:prevind(pname, end-2)])) else @@ -1365,7 +1366,7 @@ function completions(string::String, pos::Int, context_module::Module=Main, shif else pname end - if isfile(joinpath(dir, pname, "src", + if isfile(joinpath(entry, "src", "$mod_name.jl")) push!(suggestions, PackageCompletion(mod_name)) end diff --git a/stdlib/REPL/src/docview.jl b/stdlib/REPL/src/docview.jl index 79cc6b48d2f4d..feeeecbd97165 100644 --- a/stdlib/REPL/src/docview.jl +++ b/stdlib/REPL/src/docview.jl @@ -11,6 +11,8 @@ import Base.Docs: doc, formatdoc, parsedoc, apropos using Base: with_output_color, mapany, isdeprecated, isexported +using Base.Filesystem: _readdirx + import REPL using InteractiveUtils: subtypes @@ -379,9 +381,9 @@ function find_readme(m::Module)::Union{String, Nothing} path = dirname(mpath) top_path = pkgdir(m) while true - for file in readdir(path; join=true, sort=true) - isfile(file) && (basename(lowercase(file)) in ["readme.md", "readme"]) || continue - return file + for entry in _readdirx(path; sort=true) + isfile(entry) && (lowercase(entry.name) in ["readme.md", "readme"]) || continue + return entry.path end path == top_path && break # go no further than pkgdir path = dirname(path) # work up through nested modules From 2b9595601e6e81b0c81376d7942497af22e222dd Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 1 Mar 2024 18:04:35 -0500 Subject: [PATCH 26/33] use `_readdirx` for `walkdir` (#53545) On a M2 Mac there is some benefit, but assumed to be much greater on slower filesystems. ``` # master julia> @btime collect(walkdir(expanduser("~/Downloads"))); 380.086 ms (310696 allocations: 25.29 MiB) # This PR julia> @btime collect(walkdir(expanduser("~/Downloads"))); 289.747 ms (103300 allocations: 7.50 MiB) ``` The implementations appear to produce the same result ``` julia> collect(walkdir(expanduser("~/Downloads"))) == collect(walkdirx(expanduser("~/Downloads"))) true ``` --- base/file.jl | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/base/file.jl b/base/file.jl index 659c39d7415f8..b7adb8e1a3fbd 100644 --- a/base/file.jl +++ b/base/file.jl @@ -1089,18 +1089,16 @@ function walkdir(root; topdown=true, follow_symlinks=false, onerror=throw) end return end - content = tryf(readdir, root) - content === nothing && return - dirs = Vector{eltype(content)}() - files = Vector{eltype(content)}() - for name in content - path = joinpath(root, name) - + entries = tryf(_readdirx, root) + entries === nothing && return + dirs = Vector{String}() + files = Vector{String}() + for entry in entries # If we're not following symlinks, then treat all symlinks as files - if (!follow_symlinks && something(tryf(islink, path), true)) || !something(tryf(isdir, path), false) - push!(files, name) + if (!follow_symlinks && something(tryf(islink, entry), true)) || !something(tryf(isdir, entry), false) + push!(files, entry.name) else - push!(dirs, name) + push!(dirs, entry.name) end end From 8bf6a073a2bec5c015a8da3a8d5bb22d129d15ca Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 2 Mar 2024 11:49:08 -0500 Subject: [PATCH 27/33] revert "Add @create_log_macro for making custom styled logging macros (#52196)" (#53551) --- HISTORY.md | 3 --- base/logging.jl | 18 ++++++--------- stdlib/Logging/docs/src/index.md | 14 +----------- stdlib/Logging/src/ConsoleLogger.jl | 1 - stdlib/Logging/src/Logging.jl | 35 ----------------------------- stdlib/Logging/test/runtests.jl | 20 +++++------------ 6 files changed, 14 insertions(+), 77 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index cdcbe01481ba6..3eaec3c6d0774 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -148,9 +148,6 @@ Standard library changes * `lu` and `issuccess(::LU)` now accept an `allowsingular` keyword argument. When set to `true`, a valid factorization with rank-deficient U factor will be treated as success instead of throwing an error. Such factorizations are now shown by printing the factors together with a "rank-deficient" note rather than printing a "Failed Factorization" message ([#52957]). #### Logging -* New `@create_log_macro` macro for creating new log macros like `@info`, `@warn` etc. For instance - `@create_log_macro MyLog 1500 :magenta` will create `@mylog` to be used like `@mylog "hello"` which - will show as `┌ MyLog: hello` etc. ([#52196]) #### Printf diff --git a/base/logging.jl b/base/logging.jl index d0f612c31eeae..bef8a89118371 100644 --- a/base/logging.jl +++ b/base/logging.jl @@ -172,18 +172,14 @@ const AboveMaxLevel = LogLevel( 1000001) # Global log limiting mechanism for super fast but inflexible global log limiting. const _min_enabled_level = Ref{LogLevel}(Debug) -# stored as LogLevel => (name, color) -const custom_log_levels = Dict{LogLevel,Tuple{Symbol,Union{Symbol,Int}}}() - function show(io::IO, level::LogLevel) - if haskey(custom_log_levels, level) print(io, custom_log_levels[level][1]) - elseif level == BelowMinLevel print(io, "BelowMinLevel") - elseif level == Debug print(io, "Debug") - elseif level == Info print(io, "Info") - elseif level == Warn print(io, "Warn") - elseif level == Error print(io, "Error") - elseif level == AboveMaxLevel print(io, "AboveMaxLevel") - else print(io, "LogLevel($(level.level))") + if level == BelowMinLevel print(io, "BelowMinLevel") + elseif level == Debug print(io, "Debug") + elseif level == Info print(io, "Info") + elseif level == Warn print(io, "Warn") + elseif level == Error print(io, "Error") + elseif level == AboveMaxLevel print(io, "AboveMaxLevel") + else print(io, "LogLevel($(level.level))") end end diff --git a/stdlib/Logging/docs/src/index.md b/stdlib/Logging/docs/src/index.md index 6d0af3ac21321..c2bde11720f4c 100644 --- a/stdlib/Logging/docs/src/index.md +++ b/stdlib/Logging/docs/src/index.md @@ -62,7 +62,7 @@ automatically extracted. Let's examine the user-defined data first: * The *log level* is a broad category for the message that is used for early filtering. There are several standard levels of type [`LogLevel`](@ref); user-defined levels are also possible. - Each built-in log level is distinct in purpose: + Each is distinct in purpose: - [`Logging.Debug`](@ref) (log level -1000) is information intended for the developer of the program. These events are disabled by default. - [`Logging.Info`](@ref) (log level 0) is for general information to the user. @@ -74,17 +74,6 @@ automatically extracted. Let's examine the user-defined data first: Often this log-level is unneeded as throwing an exception can convey all the required information. - You can create logging macros for custom log levels. For instance: - ```julia-repl - julia> using Logging - - julia> @create_log_macro MyLog 200 :magenta - @mylog (macro with 1 method) - - julia> @mylog "hello" - [ MyLog: hello - ``` - * The *message* is an object describing the event. By convention `AbstractString`s passed as messages are assumed to be in markdown format. Other types will be displayed using `print(io, obj)` or `string(obj)` for @@ -315,7 +304,6 @@ Logging.Warn Logging.Error Logging.BelowMinLevel Logging.AboveMaxLevel -Logging.@create_log_macro ``` ### [Processing events with AbstractLogger](@id AbstractLogger-interface) diff --git a/stdlib/Logging/src/ConsoleLogger.jl b/stdlib/Logging/src/ConsoleLogger.jl index 2950ad1b8cf95..08e4a8c6b2efe 100644 --- a/stdlib/Logging/src/ConsoleLogger.jl +++ b/stdlib/Logging/src/ConsoleLogger.jl @@ -58,7 +58,6 @@ end showvalue(io, ex::Exception) = showerror(io, ex) function default_logcolor(level::LogLevel) - level in keys(custom_log_levels) ? custom_log_levels[level][2] : level < Info ? :log_debug : level < Warn ? :log_info : level < Error ? :log_warn : diff --git a/stdlib/Logging/src/Logging.jl b/stdlib/Logging/src/Logging.jl index c05d3b7227c34..3822bde2e630b 100644 --- a/stdlib/Logging/src/Logging.jl +++ b/stdlib/Logging/src/Logging.jl @@ -23,7 +23,6 @@ for sym in [ Symbol("@warn"), Symbol("@error"), Symbol("@logmsg"), - :custom_log_levels, :with_logger, :current_logger, :global_logger, @@ -32,39 +31,6 @@ for sym in [ @eval const $sym = Base.CoreLogging.$sym end -""" - @create_log_macro(name::Symbol, level::Int, face::Union{Symbol, StyledStrings.Face}) - -Creates a custom log macro like `@info`, `@warn` etc. with a given `name`, -`level` to be displayed with `face`. The macro created is named with the -lowercase form of `name` but the given form is used for the printing. - -```julia-repl -julia> @create_log_macro(:MyLog, 200, :magenta) -@mylog (macro with 1 method) - -julia> @mylog "hello" -[ MyLog: hello -``` -""" -macro create_log_macro(name, level, color) - macro_name = Symbol(lowercase(string(name))) - macro_string = QuoteNode(name) - loglevel = LogLevel(level) - if loglevel in (BelowMinLevel, Debug, Info, Warn, Error, AboveMaxLevel) - throw(ArgumentError("Cannot use the same log level as a built in log macro")) - end - if haskey(custom_log_levels, loglevel) - throw(ArgumentError("Custom log macro already exists for given log level")) - end - quote - $(custom_log_levels)[$(esc(loglevel))] = ($(macro_string), $(esc(color))) - macro $(esc(macro_name))(exs...) - $(Base.CoreLogging.logmsg_code)(($(Base.CoreLogging.@_sourceinfo))..., $(esc(loglevel)), exs...) - end - end -end - # LogLevel aliases (re-)documented here (JuliaLang/julia#40978) """ Debug @@ -115,7 +81,6 @@ export @warn, @error, @logmsg, - @create_log_macro, with_logger, current_logger, global_logger, diff --git a/stdlib/Logging/test/runtests.jl b/stdlib/Logging/test/runtests.jl index 57866ff0a18d2..a244facee3468 100644 --- a/stdlib/Logging/test/runtests.jl +++ b/stdlib/Logging/test/runtests.jl @@ -7,8 +7,8 @@ import Logging: min_enabled_level, shouldlog, handle_message @noinline func1() = backtrace() # see "custom log macro" testset -@create_log_macro CustomLog1 -500 :magenta -@create_log_macro CustomLog2 1500 1 +CustomLog = LogLevel(-500) +macro customlog(exs...) Base.CoreLogging.logmsg_code((Base.CoreLogging.@_sourceinfo)..., esc(CustomLog), exs...) end @testset "Logging" begin @@ -289,24 +289,16 @@ end end @testset "custom log macro" begin - llevel = LogLevel(-500) - - @test_logs (llevel, "foo") min_level=llevel @customlog1 "foo" + @test_logs (CustomLog, "a") min_level=CustomLog @customlog "a" buf = IOBuffer() io = IOContext(buf, :displaysize=>(30,80), :color=>false) - logger = ConsoleLogger(io, llevel) - - with_logger(logger) do - @customlog1 "foo" - end - @test occursin("CustomLog1: foo", String(take!(buf))) - + logger = ConsoleLogger(io, CustomLog) with_logger(logger) do - @customlog2 "hello" + @customlog "a" end - @test occursin("CustomLog2: hello", String(take!(buf))) + @test occursin("LogLevel(-500): a", String(take!(buf))) end @testset "Docstrings" begin From e3b2462ec925a8d445db0386a47429e8410de36e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateus=20Ara=C3=BAjo?= Date: Sat, 2 Mar 2024 20:02:45 +0100 Subject: [PATCH 28/33] fix number of chunks (#53413) The manual claims that `a` is split into `nthreads()` chunks, but this is not true in general. As it was you could get an error, if `length(a) < nthreads()`, or a number of chunks larger than `nthreads()`, if `nthreads()` is smaller than `length(a)` but does not divide it. With `cld`, on the other hand, you always get at most `nthreads()` chunks. --- doc/src/manual/multi-threading.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/manual/multi-threading.md b/doc/src/manual/multi-threading.md index dd02da8798f06..d16407efc3dcf 100644 --- a/doc/src/manual/multi-threading.md +++ b/doc/src/manual/multi-threading.md @@ -227,11 +227,11 @@ julia> sum_multi_bad(1:1_000_000) Note that the result is not `500000500000` as it should be, and will most likely change each evaluation. To fix this, buffers that are specific to the task may be used to segment the sum into chunks that are race-free. -Here `sum_single` is reused, with its own internal buffer `s`. The input vector `a` is split into `nthreads()` +Here `sum_single` is reused, with its own internal buffer `s`. The input vector `a` is split into at most `nthreads()` chunks for parallel work. We then use `Threads.@spawn` to create tasks that individually sum each chunk. Finally, we sum the results from each task using `sum_single` again: ```julia-repl julia> function sum_multi_good(a) - chunks = Iterators.partition(a, length(a) ÷ Threads.nthreads()) + chunks = Iterators.partition(a, cld(length(a), Threads.nthreads())) tasks = map(chunks) do chunk Threads.@spawn sum_single(chunk) end From 86f5b218ca8c99e9eea6b23208e6bd5569f8b504 Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Sun, 3 Mar 2024 02:16:23 +0100 Subject: [PATCH 29/33] Make dump print `const` before const fields (#53492) This provides more information to the user when dumping types, and also makes the output of dump slightly more similar to the type definition syntax. EDIT: This has been changed to print: * The kind of type before the type name `abstract type`, `mutable struct`, etc. * `const` only for `const` fields of `mutable struct` New behaviour ``` julia> dump(Float32) primitive type Float32 <: AbstractFloat julia> dump(Signed) abstract type Signed <: Integer julia> dump(Pair{Int, String}) struct Pair{Int64, String} <: Any first::Int64 second::String julia> dump(BitSet) mutable struct BitSet <: AbstractSet{Int64} const bits::Vector{UInt64} offset::Int64 julia> dump(Set) UnionAll var: TypeVar name: Symbol T lb: Union{} ub: abstract type Any body: struct Set{T} <: AbstractSet{T} dict::Dict{T, Nothing} ``` --------- Co-authored-by: Shuhei Kadowaki --- base/show.jl | 11 +++- doc/src/devdocs/types.md | 12 ++--- test/show.jl | 108 +++++++++++++++++++-------------------- 3 files changed, 70 insertions(+), 61 deletions(-) diff --git a/base/show.jl b/base/show.jl index 217b22ab39ef8..df429830b16fd 100644 --- a/base/show.jl +++ b/base/show.jl @@ -2986,6 +2986,13 @@ end # Types function dump(io::IOContext, x::DataType, n::Int, indent) + # For some reason, tuples are structs + is_struct = isstructtype(x) && !(x <: Tuple) + is_mut = is_struct && ismutabletype(x) + is_mut && print(io, "mutable ") + is_struct && print(io, "struct ") + isprimitivetype(x) && print(io, "primitive type ") + isabstracttype(x) && print(io, "abstract type ") print(io, x) if x !== Any print(io, " <: ", supertype(x)) @@ -3007,7 +3014,9 @@ function dump(io::IOContext, x::DataType, n::Int, indent) fieldtypes = datatype_fieldtypes(x) for idx in 1:length(fields) println(io) - print(io, indent, " ", fields[idx]) + print(io, indent, " ") + is_mut && isconst(x, idx) && print(io, "const ") + print(io, fields[idx]) if isassigned(fieldtypes, idx) print(io, "::") print(tvar_io, fieldtypes[idx]) diff --git a/doc/src/devdocs/types.md b/doc/src/devdocs/types.md index 120c948c6cadc..7a5ede63b1989 100644 --- a/doc/src/devdocs/types.md +++ b/doc/src/devdocs/types.md @@ -93,13 +93,13 @@ UnionAll var: TypeVar name: Symbol T lb: Union{} - ub: Any + ub: abstract type Any body: UnionAll var: TypeVar name: Symbol N lb: Union{} - ub: Any - body: Array{T, N} <: DenseArray{T, N} + ub: abstract type Any + body: mutable struct Array{T, N} <: DenseArray{T, N} ref::MemoryRef{T} size::NTuple{N, Int64} ``` @@ -181,13 +181,13 @@ TypeName var: TypeVar name: Symbol T lb: Union{} - ub: Any + ub: abstract type Any body: UnionAll var: TypeVar name: Symbol N lb: Union{} - ub: Any - body: Array{T, N} <: DenseArray{T, N} + ub: abstract type Any + body: mutable struct Array{T, N} <: DenseArray{T, N} cache: SimpleVector ... diff --git a/test/show.jl b/test/show.jl index 064ae76b6fe1f..8d6b32a3fd663 100644 --- a/test/show.jl +++ b/test/show.jl @@ -703,7 +703,7 @@ let oldout = stdout, olderr = stderr redirect_stderr(olderr) close(wrout) close(wrerr) - @test fetch(out) == "Int64 <: Signed\nTESTA\nTESTB\nΑ1Β2\"A\"\nA\n123\"C\"\n" + @test fetch(out) == "primitive type Int64 <: Signed\nTESTA\nTESTB\nΑ1Β2\"A\"\nA\n123\"C\"\n" @test fetch(err) == "TESTA\nTESTB\nΑ1Β2\"A\"\n" finally redirect_stdout(oldout) @@ -1259,64 +1259,64 @@ end close(s2) end -let repr = sprint(dump, :(x = 1)) - @test repr == "Expr\n head: Symbol =\n args: Array{Any}((2,))\n 1: Symbol x\n 2: $Int 1\n" -end -let repr = sprint(dump, Pair{String,Int64}) - @test repr == "Pair{String, Int64} <: Any\n first::String\n second::Int64\n" -end -let repr = sprint(dump, Tuple) - @test repr == "Tuple <: Any\n" -end -let repr = sprint(dump, Int64) - @test repr == "Int64 <: Signed\n" -end -let repr = sprint(dump, Any) - @test length(repr) == 4 - @test occursin(r"^Any\n", repr) - @test endswith(repr, '\n') -end -let repr = sprint(dump, Integer) - @test occursin("Integer <: Real", repr) - @test !occursin("Any", repr) -end -let repr = sprint(dump, Union{Integer, Float32}) - @test repr == "Union{Integer, Float32}\n" || repr == "Union{Float32, Integer}\n" -end module M30442 struct T end end -let repr = sprint(show, Union{String, M30442.T}) - @test repr == "Union{$(curmod_prefix)M30442.T, String}" || - repr == "Union{String, $(curmod_prefix)M30442.T}" -end -let repr = sprint(dump, Ptr{UInt8}(UInt(1))) - @test repr == "Ptr{UInt8} @$(Base.repr(UInt(1)))\n" -end -let repr = sprint(dump, Core.svec()) - @test repr == "empty SimpleVector\n" -end -let repr = sprint(dump, sin) - @test repr == "sin (function of type typeof(sin))\n" -end -let repr = sprint(dump, Test) - @test repr == "Module Test\n" -end -let repr = sprint(dump, nothing) - @test repr == "Nothing nothing\n" -end -let a = Vector{Any}(undef, 10000) - a[2] = "elemA" - a[4] = "elemB" - a[11] = "elemC" - repr = sprint(dump, a; context=(:limit => true), sizehint=0) - @test repr == "Array{Any}((10000,))\n 1: #undef\n 2: String \"elemA\"\n 3: #undef\n 4: String \"elemB\"\n 5: #undef\n ...\n 9996: #undef\n 9997: #undef\n 9998: #undef\n 9999: #undef\n 10000: #undef\n" -end -@test occursin("NamedTuple", sprint(dump, NamedTuple)) +@testset "Dump types" begin + let repr = sprint(dump, :(x = 1)) + @test repr == "Expr\n head: Symbol =\n args: Array{Any}((2,))\n 1: Symbol x\n 2: $Int 1\n" + end + let repr = sprint(dump, Pair{String,Int64}) + @test repr == "struct Pair{String, Int64} <: Any\n first::String\n second::Int64\n" + end + let repr = sprint(dump, Tuple) + @test repr == "Tuple <: Any\n" + end + let repr = sprint(dump, Int64) + @test repr == "primitive type Int64 <: Signed\n" + end + let repr = sprint(dump, Any) + @test repr == "abstract type Any\n" + end + let repr = sprint(dump, Integer) + @test occursin("abstract type Integer <: Real", repr) + @test !occursin("Any", repr) + end + let repr = sprint(dump, Union{Integer, Float32}) + @test repr == "Union{Integer, Float32}\n" || repr == "Union{Float32, Integer}\n" + end -# issue 36495, dumping a partial NamedTupled shouldn't error -@test occursin("NamedTuple", sprint(dump, NamedTuple{(:foo,:bar)})) + let repr = sprint(show, Union{String, M30442.T}) + @test repr == "Union{$(curmod_prefix)M30442.T, String}" || + repr == "Union{String, $(curmod_prefix)M30442.T}" + end + let repr = sprint(dump, Ptr{UInt8}(UInt(1))) + @test repr == "Ptr{UInt8} @$(Base.repr(UInt(1)))\n" + end + let repr = sprint(dump, Core.svec()) + @test repr == "empty SimpleVector\n" + end + let repr = sprint(dump, sin) + @test repr == "sin (function of type typeof(sin))\n" + end + let repr = sprint(dump, Test) + @test repr == "Module Test\n" + end + let repr = sprint(dump, nothing) + @test repr == "Nothing nothing\n" + end + let a = Vector{Any}(undef, 10000) + a[2] = "elemA" + a[4] = "elemB" + a[11] = "elemC" + repr = sprint(dump, a; context=(:limit => true), sizehint=0) + @test repr == "Array{Any}((10000,))\n 1: #undef\n 2: String \"elemA\"\n 3: #undef\n 4: String \"elemB\"\n 5: #undef\n ...\n 9996: #undef\n 9997: #undef\n 9998: #undef\n 9999: #undef\n 10000: #undef\n" + end + @test occursin("NamedTuple", sprint(dump, NamedTuple)) + # issue 36495, dumping a partial NamedTupled shouldn't error + @test occursin("NamedTuple", sprint(dump, NamedTuple{(:foo,:bar)})) +end # issue #17338 @test repr(Core.svec(1, 2)) == "svec(1, 2)" From a586d3cc6b02c7ae72e2161c8674aa3800d8da1f Mon Sep 17 00:00:00 2001 From: Mason Protter Date: Sun, 3 Mar 2024 02:16:51 +0100 Subject: [PATCH 30/33] Clarify docstring for `zero`, `one`, and `oneunit`. (#52107) spurred by a conversation on Slack where someone was thrown off wondering why `zero(Vector{Int})` errors, but `zero(Int)` is fine. --------- Co-authored-by: Sukera <11753998+Seelengrab@users.noreply.github.com> Co-authored-by: Max Horn Co-authored-by: Simon Byrne --- base/number.jl | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/base/number.jl b/base/number.jl index 923fc907d4038..39908222027c1 100644 --- a/base/number.jl +++ b/base/number.jl @@ -287,7 +287,12 @@ map(f, x::Number, ys::Number...) = f(x, ys...) zero(x) zero(::Type) -Get the additive identity element for the type of `x` (`x` can also specify the type itself). +Get the additive identity element for `x`. If the additive identity can be deduced +from the type alone, then a type may be given as an argument to `zero`. + +For example, `zero(Int)` will work because the additive identity is the same for all +instances of `Int`, but `zero(Vector{Int})` is not defined because vectors of different +lengths have different additive identities. See also [`iszero`](@ref), [`one`](@ref), [`oneunit`](@ref), [`oftype`](@ref). @@ -314,9 +319,12 @@ zero(::Type{Union{}}, slurp...) = Union{}(0) one(T::type) Return a multiplicative identity for `x`: a value such that -`one(x)*x == x*one(x) == x`. Alternatively `one(T)` can -take a type `T`, in which case `one` returns a multiplicative -identity for any `x` of type `T`. +`one(x)*x == x*one(x) == x`. If the multiplicative identity can +be deduced from the type alone, then a type may be given as +an argument to `one` (e.g. `one(Int)` will work because the +multiplicative identity is the same for all instances of `Int`, +but `one(Matrix{Int})` is not defined because matrices of +different shapes have different multiplicative identities.) If possible, `one(x)` returns a value of the same type as `x`, and `one(T)` returns a value of type `T`. However, this may @@ -354,9 +362,10 @@ one(::Type{Union{}}, slurp...) = Union{}(1) oneunit(x::T) oneunit(T::Type) -Return `T(one(x))`, where `T` is either the type of the argument or -(if a type is passed) the argument. This differs from [`one`](@ref) for -dimensionful quantities: `one` is dimensionless (a multiplicative identity) +Return `T(one(x))`, where `T` is either the type of the argument, or +the argument itself in cases where the `oneunit` can be deduced from +the type alone. This differs from [`one`](@ref) for dimensionful +quantities: `one` is dimensionless (a multiplicative identity) while `oneunit` is dimensionful (of the same type as `x`, or of type `T`). # Examples From e7734ea0889c940c7dc57ba4e4401c93747fb428 Mon Sep 17 00:00:00 2001 From: Abel Soares Siqueira Date: Sun, 3 Mar 2024 02:23:18 +0100 Subject: [PATCH 31/33] Add a GitHub Actions workflow for automatic validation of the citation metadata file (`CITATION.CFF`) (#44062) Hello! We noticed that your `CITATION.cff` had a small issue and fixed it. In addition to the fix, this Pull Request automates validation of that file using the [cffconvert GitHub Action](https://github.com/marketplace/actions/cffconvert). That way, it's a little bit easier to be robust against future changes to the `CITATION.cff` file. BTW it's perfectly fine if you don't feel like accepting this Pull Request for whatever reason -- we just thought it might be helpful is all. We found your repository using a partially automated workflow; if you have any questions about that, feel free to create an issue over at https://github.com/cffbots/filtering/issues/ On behalf of the cffbots team, @abelsiqueira / @fdiblen / @jspaaks --------- Co-authored-by: Max Horn --- .github/workflows/cffconvert.yml | 33 ++++++++++++++++++++++++++++++++ CITATION.cff | 1 + 2 files changed, 34 insertions(+) create mode 100644 .github/workflows/cffconvert.yml diff --git a/.github/workflows/cffconvert.yml b/.github/workflows/cffconvert.yml new file mode 100644 index 0000000000000..751476865ae4c --- /dev/null +++ b/.github/workflows/cffconvert.yml @@ -0,0 +1,33 @@ +name: cffconvert + +on: + push: + branches: + - 'master' + - 'release-*' + paths: + - CITATION.cff + pull_request: + branches: + - 'master' + - 'release-*' + paths: + - CITATION.cff + +permissions: + contents: read + +jobs: + validate: + name: "validate" + runs-on: ubuntu-latest + steps: + - name: Check out a copy of the repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check whether the citation metadata from CITATION.cff is valid + uses: citation-file-format/cffconvert-github-action@2.0.0 + with: + args: "--validate" diff --git a/CITATION.cff b/CITATION.cff index c88727bcfa311..878ab94a4d86a 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,3 +1,4 @@ +# Official format description at https://citation-file-format.github.io cff-version: 1.2.0 message: "Cite this paper whenever you use Julia" authors: From 0f902bf2f7ea6dcf0955b5dc37f00a2c1447af28 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Sun, 3 Mar 2024 04:14:06 +0100 Subject: [PATCH 32/33] Provide better error hint when `UndefVarError` results from name clashes (#53469) We can detect this since we set the `usingfailed` bit when the clash occurs (to avoid printing the `WARNING` multiple times). In this case, typos or missing imports (the current message) isn't quite as clear as it could be, because in fact the name is probably spelled totally right, it's just that there is a missing explicit import or the name should be qualified. This code will stop working if we change the flags in `Core.Binding`, but the test I added should catch that. However if REPL is supposed to be independent of Base and not depend on internals there, there could be an issue. In that case we should probably add an API to Base to inspect this `usingfailed` bit so we can use it in the REPL. --------- Co-authored-by: Jameson Nash Co-authored-by: Alex Arslan --- stdlib/REPL/src/REPL.jl | 11 ++++++++++- stdlib/REPL/test/repl.jl | 29 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index c5c45dee34a01..bb5915c192b6e 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -44,7 +44,16 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) if C_NULL == owner # No global of this name exists in this module. # This is the common case, so do not print that information. - print(io, "\nSuggestion: check for spelling errors or missing imports.") + # It could be the binding was exported by two modules, which we can detect + # by the `usingfailed` flag in the binding: + if isdefined(bnd, :flags) && Bool(bnd.flags >> 4 & 1) # magic location of the `usingfailed` flag + print(io, "\nHint: It looks like two or more modules export different ", + "bindings with this name, resulting in ambiguity. Try explicitly ", + "importing it from a particular module, or qualifying the name ", + "with the module it should come from.") + else + print(io, "\nSuggestion: check for spelling errors or missing imports.") + end owner = bnd else owner = unsafe_pointer_to_objref(owner)::Core.Binding diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 83df34a056578..12f3f8956122e 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -1696,6 +1696,35 @@ finally empty!(Base.Experimental._hint_handlers) end +try # test the functionality of `UndefVarError_hint` against import clashes + @assert isempty(Base.Experimental._hint_handlers) + Base.Experimental.register_error_hint(REPL.UndefVarError_hint, UndefVarError) + + @eval module X + + module A + export x + x = 1 + end # A + + module B + export x + x = 2 + end # B + + using .A, .B + + end # X + + expected_message = string("\nHint: It looks like two or more modules export different ", + "bindings with this name, resulting in ambiguity. Try explicitly ", + "importing it from a particular module, or qualifying the name ", + "with the module it should come from.") + @test_throws expected_message X.x +finally + empty!(Base.Experimental._hint_handlers) +end + # Hints for tab completes fake_repl() do stdin_write, stdout_read, repl From 7179050c81cfcaca75419f4948bf59a025b50469 Mon Sep 17 00:00:00 2001 From: erich-9 Date: Sun, 3 Mar 2024 04:14:28 +0100 Subject: [PATCH 33/33] Always return a value in 1-d circshift! of abstractarray.jl (#53554) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sukera <11753998+Seelengrab@users.noreply.github.com> Co-authored-by: Mosè Giordano --- base/abstractarray.jl | 4 ++-- test/arrayops.jl | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index bdbcbe4d081a4..a9431197932dd 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -3656,9 +3656,9 @@ end ## 1-d circshift ## function circshift!(a::AbstractVector, shift::Integer) n = length(a) - n == 0 && return + n == 0 && return a shift = mod(shift, n) - shift == 0 && return + shift == 0 && return a l = lastindex(a) reverse!(a, firstindex(a), l-shift) reverse!(a, l-shift+1, lastindex(a)) diff --git a/test/arrayops.jl b/test/arrayops.jl index a07e631e639f9..566dd44b8dcd9 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -797,6 +797,14 @@ end oa = OffsetVector(copy(a), -1) @test circshift!(oa, 1) === oa @test oa == circshift(OffsetVector(a, -1), 1) + + # 1d circshift! (#53554) + a = [] + @test circshift!(a, 1) === a + @test circshift!(a, 1) == [] + a = [1:5;] + @test circshift!(a, 10) === a + @test circshift!(a, 10) == 1:5 end @testset "circcopy" begin