[release 1.5] more backports for 1.5-rc2

NEWS.md

@@ -232,6 +232,7 @@ Tooling Improvements
 [#25930]: https://github.com/JuliaLang/julia/issues/25930
 [#26872]: https://github.com/JuliaLang/julia/issues/26872
 [#28789]: https://github.com/JuliaLang/julia/issues/28789
+[#28811]: https://github.com/JuliaLang/julia/issues/28811
 [#29240]: https://github.com/JuliaLang/julia/issues/29240
 [#29333]: https://github.com/JuliaLang/julia/issues/29333
 [#29411]: https://github.com/JuliaLang/julia/issues/29411
@@ -247,6 +248,7 @@ Tooling Improvements
 [#33864]: https://github.com/JuliaLang/julia/issues/33864
 [#33886]: https://github.com/JuliaLang/julia/issues/33886
 [#33937]: https://github.com/JuliaLang/julia/issues/33937
+[#34126]: https://github.com/JuliaLang/julia/issues/34126
 [#34149]: https://github.com/JuliaLang/julia/issues/34149
 [#34199]: https://github.com/JuliaLang/julia/issues/34199
 [#34200]: https://github.com/JuliaLang/julia/issues/34200
@@ -273,9 +275,12 @@ Tooling Improvements
 [#34896]: https://github.com/JuliaLang/julia/issues/34896
 [#34953]: https://github.com/JuliaLang/julia/issues/34953
 [#35001]: https://github.com/JuliaLang/julia/issues/35001
+[#35057]: https://github.com/JuliaLang/julia/issues/35057
 [#35078]: https://github.com/JuliaLang/julia/issues/35078
+[#35085]: https://github.com/JuliaLang/julia/issues/35085
 [#35094]: https://github.com/JuliaLang/julia/issues/35094
 [#35108]: https://github.com/JuliaLang/julia/issues/35108
+[#35113]: https://github.com/JuliaLang/julia/issues/35113
 [#35124]: https://github.com/JuliaLang/julia/issues/35124
 [#35132]: https://github.com/JuliaLang/julia/issues/35132
 [#35138]: https://github.com/JuliaLang/julia/issues/35138
@@ -290,3 +295,4 @@ Tooling Improvements
 [#35522]: https://github.com/JuliaLang/julia/issues/35522
 [#35554]: https://github.com/JuliaLang/julia/issues/35554
 [#35626]: https://github.com/JuliaLang/julia/issues/35626
+[#36070]: https://github.com/JuliaLang/julia/issues/36070

base/array.jl

@@ -211,11 +211,9 @@ julia> Base.bitsunionsize(Union{Float64, UInt8, Int128})
 ```
 """
 function bitsunionsize(u::Union)
-    sz = Ref{Csize_t}(0)
-    algn = Ref{Csize_t}(0)
-    isunboxed = ccall(:jl_islayout_inline, Cint, (Any, Ptr{Csize_t}, Ptr{Csize_t}), u, sz, algn)
-    @assert isunboxed != Cint(0)
-    return sz[]
+    isinline, sz, _ = uniontype_layout(u)
+    @assert isinline
+    return sz
 end
 
 length(a::Array) = arraylen(a)

base/c.jl

@@ -681,7 +681,7 @@ with a Julia variable named `s`. See also `ccall`.
 
 Varargs are supported with the following convention:
 
-    @ccall sprintf("%s = %d"::Cstring ; "foo"::Cstring, foo::Cint)::Cint
+    @ccall printf("%s = %d"::Cstring ; "foo"::Cstring, foo::Cint)::Cint
 
 The semicolon is used to separate required arguments (of which there
 must be at least one) from variadic arguments.

base/channels.jl

@@ -419,6 +419,7 @@ on a [`put!`](@ref).
 """
 isready(c::Channel) = n_avail(c) > 0
 n_avail(c::Channel) = isbuffered(c) ? length(c.data) : length(c.cond_put.waitq)
+isempty(c::Channel) = isbuffered(c) ? isempty(c.data) : isempty(c.cond_put.waitq)
 
 lock(c::Channel) = lock(c.cond_take)
 unlock(c::Channel) = unlock(c.cond_take)

base/compiler/tfuncs.jl

@@ -306,16 +306,34 @@ function sizeof_nothrow(@nospecialize(x))
         end
     elseif isa(x, Conditional)
         return true
-    else
-        x = widenconst(x)
     end
     if isa(x, Union)
         return sizeof_nothrow(x.a) && sizeof_nothrow(x.b)
     end
-    isconstType(x) && (x = x.parameters[1]) # since sizeof(typeof(x)) == sizeof(x)
-    x === DataType && return false
-    return isconcretetype(x) || isprimitivetype(x)
+    t, exact = instanceof_tfunc(x)
+    if !exact
+        # Could always be bottom at runtime, which throws
+        return false
+    end
+    if t !== Bottom
+        t === DataType && return true
+        x = t
+        x = unwrap_unionall(x)
+        if isa(x, Union)
+            isinline, sz, _ = uniontype_layout(x)
+            return isinline
+        end
+        isa(x, DataType) || return false
+        x.layout == C_NULL && return false
+        (datatype_nfields(x) == 0 && !datatype_pointerfree(x)) && return false
+        return true
+    else
+        x = widenconst(x)
+        x === DataType && return false
+        return isconcretetype(x) || isprimitivetype(x)
+    end
 end
+
 function _const_sizeof(@nospecialize(x))
     # Constant Vector does not have constant size
     isa(x, Vector) && return Int
@@ -334,12 +352,27 @@ function sizeof_tfunc(@nospecialize(x),)
     isa(x, Const) && return _const_sizeof(x.val)
     isa(x, Conditional) && return _const_sizeof(Bool)
     isconstType(x) && return _const_sizeof(x.parameters[1])
-    x = widenconst(x)
     if isa(x, Union)
         return tmerge(sizeof_tfunc(x.a), sizeof_tfunc(x.b))
     end
-    x !== DataType && isconcretetype(x) && return _const_sizeof(x)
-    isprimitivetype(x) && return _const_sizeof(x)
+    # Core.sizeof operates on either a type or a value. First check which
+    # case we're in.
+    t, exact = instanceof_tfunc(x)
+    if t !== Bottom
+        # The value corresponding to `x` at runtime could be a type.
+        # Normalize the query to ask about that type.
+        x = unwrap_unionall(t)
+        if isa(x, Union)
+            isinline, sz, _ = uniontype_layout(x)
+            return isinline ? Const(Int(sz)) : (exact ? Bottom : Int)
+        end
+        isa(x, DataType) || return Int
+        (isconcretetype(x) || isprimitivetype(x)) && return _const_sizeof(x)
+    else
+        x = widenconst(x)
+        x !== DataType && isconcretetype(x) && return _const_sizeof(x)
+        isprimitivetype(x) && return _const_sizeof(x)
+    end
     return Int
 end
 add_tfunc(Core.sizeof, 1, 1, sizeof_tfunc, 0)

base/deprecated.jl

@@ -207,4 +207,6 @@ macro get!(h, key0, default)
     end
 end
 
+pointer(V::SubArray{<:Any,<:Any,<:Array,<:Tuple{Vararg{RangeIndex}}}, is::Tuple) = pointer(V, CartesianIndex(is))
+
 # END 1.5 deprecations

base/iddict.jl

@@ -1,3 +1,5 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
 """
     IdDict([itr])
 

base/idset.jl

@@ -1,3 +1,5 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
 # Like Set, but using IdDict
 mutable struct IdSet{T} <: AbstractSet{T}
     dict::IdDict{T,Nothing}

base/io.jl

@@ -652,18 +652,18 @@ function write(s::IO, a::SubArray{T,N,<:Array}) where {T,N}
     if !isbitstype(T) || !isa(a, StridedArray)
         return invoke(write, Tuple{IO, AbstractArray}, s, a)
     end
-    elsz = sizeof(T)
+    elsz = elsize(a)
     colsz = size(a,1) * elsz
     GC.@preserve a if stride(a,1) != 1
         for idxs in CartesianIndices(size(a))
-            unsafe_write(s, pointer(a, idxs.I), elsz)
+            unsafe_write(s, pointer(a, idxs), elsz)
         end
         return elsz * length(a)
     elseif N <= 1
         return unsafe_write(s, pointer(a, 1), colsz)
     else
-        for idxs in CartesianIndices((1, size(a)[2:end]...))
-            unsafe_write(s, pointer(a, idxs.I), colsz)
+        for colstart in CartesianIndices((1, size(a)[2:end]...))
+            unsafe_write(s, pointer(a, colstart), colsz)
         end
         return colsz * trailingsize(a,2)
     end

base/multidimensional.jl

@@ -999,7 +999,9 @@ See also [`circshift`](@ref).
     axes(dest) == inds || throw(ArgumentError("indices of src and dest must match (got $inds and $(axes(dest)))"))
     _circshift!(dest, (), src, (), inds, fill_to_length(shiftamt, 0, Val(N)))
 end
-circshift!(dest::AbstractArray, src, shiftamt) = circshift!(dest, src, (shiftamt...,))
+
+circshift!(dest::AbstractArray, src, shiftamt) =
+    circshift!(dest, src, map(Integer, (shiftamt...,)))
 
 # For each dimension, we copy the first half of src to the second half
 # of dest, and the second half of src to the first half of dest. This

base/reflection.jl

@@ -338,15 +338,19 @@ function datatype_alignment(dt::DataType)
     return Int(alignment)
 end
 
+function uniontype_layout(T::Type)
+    sz = RefValue{Csize_t}(0)
+    algn = RefValue{Csize_t}(0)
+    isinline = ccall(:jl_islayout_inline, Cint, (Any, Ptr{Csize_t}, Ptr{Csize_t}), T, sz, algn) != 0
+    (isinline, sz[], algn[])
+end
+
 # amount of total space taken by T when stored in a container
 function aligned_sizeof(T)
     @_pure_meta
     if isbitsunion(T)
-        sz = Ref{Csize_t}(0)
-        algn = Ref{Csize_t}(0)
-        ccall(:jl_islayout_inline, Cint, (Any, Ptr{Csize_t}, Ptr{Csize_t}), T, sz, algn)
-        al = algn[]
-        return (sz[] + al - 1) & -al
+        _, sz, al = uniontype_layout(T)
+        return (sz + al - 1) & -al
     elseif allocatedinline(T)
         al = datatype_alignment(T)
         return (Core.sizeof(T) + al - 1) & -al
@@ -372,6 +376,19 @@ function datatype_haspadding(dt::DataType)
     return flags & 1 == 1
 end
 
+"""
+    Base.datatype_nfields(dt::DataType) -> Bool
+
+Return the number of fields known to this datatype's layout.
+Can be called on any `isconcretetype`.
+"""
+function datatype_nfields(dt::DataType)
+    @_pure_meta
+    dt.layout == C_NULL && throw(UndefRefError())
+    return unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).nfields
+end
+
+
 """
     Base.datatype_pointerfree(dt::DataType) -> Bool
 

base/subarray.jl

@@ -64,7 +64,10 @@ size(V::SubArray) = (@_inline_meta; map(n->Int(unsafe_length(n)), axes(V)))
 
 similar(V::SubArray, T::Type, dims::Dims) = similar(V.parent, T, dims)
 
-sizeof(V::SubArray) = length(V) * elsize(V.parent)
+sizeof(V::SubArray) = length(V) * sizeof(eltype(V))
+sizeof(V::SubArray{<:Any,<:Any,<:Array}) = length(V) * elsize(V.parent)
+
+elsize(::Type{<:SubArray{<:Any,<:Any,P}}) where {P<:Array} = elsize(P)
 
 copy(V::SubArray) = V.parent[V.indices...]
 
@@ -405,6 +408,15 @@ end
 pointer(V::FastSubArray, i::Int) = pointer(V.parent, V.offset1 + V.stride1*i)
 pointer(V::FastContiguousSubArray, i::Int) = pointer(V.parent, V.offset1 + i)
 
+function pointer(V::SubArray{<:Any,<:Any,<:Array,<:Tuple{Vararg{RangeIndex}}}, is::AbstractCartesianIndex{N}) where {N}
+    index = first_index(V)
+    strds = strides(V)
+    for d = 1:N
+        index += (is[d]-1)*strds[d]
+    end
+    return pointer(V.parent, index)
+end
+
 # indices are taken from the range/vector
 # Since bounds-checking is performance-critical and uses
 # indices, it's worth optimizing these implementations thoroughly

base/ttyhascolor.jl

@@ -1,3 +1,5 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
 if Sys.iswindows()
     ttyhascolor(term_type = nothing) = true
 else

contrib/mac/app/notarize_check.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# This file is a part of Julia. License is MIT: https://julialang.org/license
 
 # Note that you need to have exported `APPLEID` and `APPLEID_PASSWORD` for this to work.
 

contrib/mac/app/renotarize_dmg.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# This file is a part of Julia. License is MIT: https://julialang.org/license
 
 # We need a URL
 if [[ -z "$1" ]]; then

doc/src/base/c.md

@@ -1,6 +1,7 @@
 # C Interface
 
 ```@docs
+@ccall
 ccall
 Core.Intrinsics.cglobal
 Base.@cfunction

doc/src/manual/multi-threading.md

@@ -1,5 +1,8 @@
 # [Multi-Threading](@id man-multithreading)
 
+Visit this [blog post](https://julialang.org/blog/2019/07/multithreading/) for a presentation
+of Julia multi-threading features.
+
 ## Starting Julia with multiple threads
 
 By default, Julia starts up with a single thread of execution. This can be verified by using the
@@ -301,47 +304,47 @@ rather pointless as a finalizer). This leads us to a bit of a conundrum.
 There are a few approaches to dealing with this problem:
 
 1. When single-threaded, code could call the internal `jl_gc_enable_finalizers`
-C function to prevent finalizers from being scheduled
-inside a critical region. Internally, this is used inside some functions (such
-as our C locks) to prevent recursion when doing certain operations (incremental
-package loading, codegen, etc.). The combination of a lock and this flag
-can be used to make finalizers safe.
+   C function to prevent finalizers from being scheduled
+   inside a critical region. Internally, this is used inside some functions (such
+   as our C locks) to prevent recursion when doing certain operations (incremental
+   package loading, codegen, etc.). The combination of a lock and this flag
+   can be used to make finalizers safe.
 
 2. A second strategy, employed by Base in a couple places, is to explicitly
-delay a finalizer until it may be able to acquire its lock non-recursively.
-The following example demonstrates how this strategy could be applied to
-`Distributed.finalize_ref`:
-
-```
-function finalize_ref(r::AbstractRemoteRef)
-    if r.where > 0 # Check if the finalizer is already run
-        if islocked(client_refs) || !trylock(client_refs)
-            # delay finalizer for later if we aren't free to acquire the lock
-            finalizer(finalize_ref, r)
-            return nothing
-        end
-        try # `lock` should always be followed by `try`
-            if r.where > 0 # Must check again here
-                # Do actual cleanup here
-                r.where = 0
-            end
-        finally
-            unlock(client_refs)
-        end
-    end
-    nothing
-end
-```
+   delay a finalizer until it may be able to acquire its lock non-recursively.
+   The following example demonstrates how this strategy could be applied to
+   `Distributed.finalize_ref`:
+
+   ```
+   function finalize_ref(r::AbstractRemoteRef)
+       if r.where > 0 # Check if the finalizer is already run
+           if islocked(client_refs) || !trylock(client_refs)
+               # delay finalizer for later if we aren't free to acquire the lock
+               finalizer(finalize_ref, r)
+               return nothing
+           end
+           try # `lock` should always be followed by `try`
+               if r.where > 0 # Must check again here
+                   # Do actual cleanup here
+                   r.where = 0
+               end
+           finally
+               unlock(client_refs)
+           end
+       end
+       nothing
+   end
+   ```
 
 3. A related third strategy is to use a yield-free queue. We don't currently
-have a lock-free queue implemented in Base, but
-`Base.InvasiveLinkedListSynchronized{T}` is suitable. This can frequently be a
-good strategy to use for code with event loops. For example, this strategy is
-employed by `Gtk.jl` to manage lifetime ref-counting. In this approach, we
-don't do any explicit work inside the `finalizer`, and instead add it to a queue
-to run at a safer time. In fact, Julia's task scheduler already uses this, so
-defining the finalizer as `x -> @spawn do_cleanup(x)` is one example of this
-approach. Note however that this doesn't control which thread `do_cleanup`
-runs on, so `do_cleanup` would still need to acquire a lock. That
-doesn't need to be true if you implement your own queue, as you can explicitly
-only drain that queue from your thread.
+   have a lock-free queue implemented in Base, but
+   `Base.InvasiveLinkedListSynchronized{T}` is suitable. This can frequently be a
+   good strategy to use for code with event loops. For example, this strategy is
+   employed by `Gtk.jl` to manage lifetime ref-counting. In this approach, we
+   don't do any explicit work inside the `finalizer`, and instead add it to a queue
+   to run at a safer time. In fact, Julia's task scheduler already uses this, so
+   defining the finalizer as `x -> @spawn do_cleanup(x)` is one example of this
+   approach. Note however that this doesn't control which thread `do_cleanup`
+   runs on, so `do_cleanup` would still need to acquire a lock. That
+   doesn't need to be true if you implement your own queue, as you can explicitly
+   only drain that queue from your thread.