perf: optimize append_any function (#30248)

Reimplement a larger portion of the optimizations in jl_f__apply
in the fallback function, so we can reduce the performance wall in more cases.

General fix for #29133-like performance issues

Author: vtjnash

base/compiler/abstractinterpretation.jl

@@ -149,7 +149,7 @@ function abstract_call_method_with_const_args(@nospecialize(f), argtypes::Vector
     length(argtypes) >= nargs || return Any
     haveconst = false
     for a in argtypes
-        a = maybe_widen_conditional(a)
+        a = widenconditional(a)
         if has_nontrivial_const_info(a)
             # have new information from argtypes that wasn't available from the signature
             if !isa(a, Const) || (isa(a.val, Symbol) || isa(a.val, Type) || (!isa(a.val, String) && isimmutable(a.val)))
@@ -181,7 +181,7 @@ function abstract_call_method_with_const_args(@nospecialize(f), argtypes::Vector
         if !istopfunction(f, :getproperty) && !istopfunction(f, :setproperty!)
             # in this case, see if all of the arguments are constants
             for a in argtypes
-                a = maybe_widen_conditional(a)
+                a = widenconditional(a)
                 if !isa(a, Const) && !isconstType(a)
                     return Any
                 end
@@ -518,7 +518,7 @@ end
 
 function pure_eval_call(@nospecialize(f), argtypes::Vector{Any}, @nospecialize(atype), sv::InferenceState)
     for i = 2:length(argtypes)
-        a = maybe_widen_conditional(argtypes[i])
+        a = widenconditional(argtypes[i])
         if !(isa(a, Const) || isconstType(a))
             return false
         end
@@ -537,7 +537,7 @@ function pure_eval_call(@nospecialize(f), argtypes::Vector{Any}, @nospecialize(a
         return false
     end
 
-    args = Any[ (a = maybe_widen_conditional(argtypes[i]); isa(a, Const) ? a.val : a.parameters[1]) for i in 2:length(argtypes) ]
+    args = Any[ (a = widenconditional(argtypes[i]); isa(a, Const) ? a.val : a.parameters[1]) for i in 2:length(argtypes) ]
     try
         value = Core._apply_pure(f, args)
         # TODO: add some sort of edge(s)
@@ -1059,7 +1059,7 @@ function typeinf_local(frame::InferenceState)
                 end
             elseif hd === :return
                 pc´ = n + 1
-                rt = maybe_widen_conditional(abstract_eval(stmt.args[1], s[pc], frame))
+                rt = widenconditional(abstract_eval(stmt.args[1], s[pc], frame))
                 if !isa(rt, Const) && !isa(rt, Type) && (!isa(rt, PartialTuple) || frame.cached)
                     # only propagate information we know we can store
                     # and is valid inter-procedurally

base/compiler/inferenceresult.jl

@@ -36,7 +36,7 @@ function matching_cache_argtypes(linfo::MethodInstance, given_argtypes::Vector)
     @assert isa(linfo.def, Method) # ensure the next line works
     nargs::Int = linfo.def.nargs
     @assert length(given_argtypes) >= (nargs - 1)
-    given_argtypes = anymap(maybe_widen_conditional, given_argtypes)
+    given_argtypes = anymap(widenconditional, given_argtypes)
     if linfo.def.isva
         isva_given_argtypes = Vector{Any}(undef, nargs)
         for i = 1:(nargs - 1)

base/compiler/tfuncs.jl

@@ -226,8 +226,8 @@ end
 add_tfunc(ifelse, 3, 3, ifelse_tfunc, 1)
 
 function egal_tfunc(@nospecialize(x), @nospecialize(y))
-    xx = maybe_widen_conditional(x)
-    yy = maybe_widen_conditional(y)
+    xx = widenconditional(x)
+    yy = widenconditional(y)
     if isa(x, Conditional) && isa(yy, Const)
         yy.val === false && return Conditional(x.var, x.elsetype, x.vtype)
         yy.val === true && return x
@@ -900,7 +900,7 @@ function apply_type_nothrow(argtypes::Array{Any, 1}, @nospecialize(rt))
     u = headtype
     for i = 2:length(argtypes)
         isa(u, UnionAll) || return false
-        ai = maybe_widen_conditional(argtypes[i])
+        ai = widenconditional(argtypes[i])
         if ai === TypeVar
             # We don't know anything about the bounds of this typevar, but as
             # long as the UnionAll is not constrained, that's ok.
@@ -981,7 +981,7 @@ function apply_type_tfunc(@nospecialize(headtypetype), @nospecialize args...)
     tparams = Any[]
     outervars = Any[]
     for i = 1:largs
-        ai = maybe_widen_conditional(args[i])
+        ai = widenconditional(args[i])
         if isType(ai)
             aip1 = ai.parameters[1]
             canconst &= !has_free_typevars(aip1)
@@ -1069,7 +1069,7 @@ end
 # convert the dispatch tuple type argtype to the real (concrete) type of
 # the tuple of those values
 function tuple_tfunc(atypes::Vector{Any})
-    atypes = anymap(maybe_widen_conditional, atypes)
+    atypes = anymap(widenconditional, atypes)
     all_are_const = true
     for i in 1:length(atypes)
         if !isa(atypes[i], Const)

base/compiler/typeinfer.jl

@@ -223,17 +223,6 @@ function widen_all_consts!(src::CodeInfo)
     return src
 end
 
-maybe_widen_conditional(@nospecialize vt) = vt
-function maybe_widen_conditional(vt::Conditional)
-    if vt.vtype === Bottom
-        return Const(false)
-    elseif vt.elsetype === Bottom
-        return Const(true)
-    else
-        return Bool
-    end
-end
-
 function annotate_slot_load!(e::Expr, vtypes::VarTable, sv::InferenceState, undefs::Array{Bool,1})
     head = e.head
     i0 = 1
@@ -256,7 +245,7 @@ end
 function visit_slot_load!(sl::Slot, vtypes::VarTable, sv::InferenceState, undefs::Array{Bool,1})
     id = slot_id(sl)
     s = vtypes[id]
-    vt = maybe_widen_conditional(s.typ)
+    vt = widenconditional(s.typ)
     if s.undef
         # find used-undef variables
         undefs[id] = true
@@ -312,7 +301,7 @@ function type_annotate!(sv::InferenceState)
         if gt[j] === NOT_FOUND
             gt[j] = Union{}
         end
-        gt[j] = maybe_widen_conditional(gt[j])
+        gt[j] = widenconditional(gt[j])
     end
 
     # compute the required type for each slot

base/compiler/typelattice.jl

@@ -217,6 +217,7 @@ end
 @inline tchanged(@nospecialize(n), @nospecialize(o)) = o === NOT_FOUND || (n !== NOT_FOUND && !(n ⊑ o))
 @inline schanged(@nospecialize(n), @nospecialize(o)) = (n !== o) && (o === NOT_FOUND || (n !== NOT_FOUND && !issubstate(n, o)))
 
+widenconditional(@nospecialize typ) = typ
 function widenconditional(typ::Conditional)
     if typ.vtype == Union{}
         return Const(false)

base/essentials.jl

@@ -415,27 +415,6 @@ Stacktrace:
 """
 sizeof(x) = Core.sizeof(x)
 
-function append_any(xs...)
-    # used by apply() and quote
-    # must be a separate function from append(), since apply() needs this
-    # exact function.
-    out = Vector{Any}(undef, 4)
-    l = 4
-    i = 1
-    for x in xs
-        for y in x
-            if i > l
-                _growend!(out, 16)
-                l += 16
-            end
-            arrayset(true, out, y, i)
-            i += 1
-        end
-    end
-    _deleteend!(out, l-i+1)
-    out
-end
-
 # simple Array{Any} operations needed for bootstrap
 @eval setindex!(A::Array{Any}, @nospecialize(x), i::Int) = arrayset($(Expr(:boundscheck)), A, x, i)
 
@@ -641,6 +620,72 @@ function isassigned(v::SimpleVector, i::Int)
     return x != C_NULL
 end
 
+
+# used by ... syntax to access the `iterate` function from inside the Core._apply implementation
+# must be a separate function from append(), since Core._apply needs this exact function
+function append_any(xs...)
+    @nospecialize
+    lx = length(xs)
+    l = 4
+    i = 1
+    out = Vector{Any}(undef, l)
+    for xi in 1:lx
+        x = @inbounds xs[xi]
+        # handle some common cases, where we know the length
+        # and can inline the iterator because the runtime
+        # has an optimized version of the iterator
+        if x isa SimpleVector
+            lx = length(x)
+            if i + lx - 1 > l
+                ladd = lx > 16 ? lx : 16
+                _growend!(out, ladd)
+                l += ladd
+            end
+            for j in 1:lx
+                y = @inbounds x[j]
+                arrayset(true, out, y, i)
+                i += 1
+            end
+        elseif x isa Tuple
+            lx = length(x)
+            if i + lx - 1 > l
+                ladd = lx > 16 ? lx : 16
+                _growend!(out, ladd)
+                l += ladd
+            end
+            for j in 1:lx
+                y = @inbounds x[j]
+                arrayset(true, out, y, i)
+                i += 1
+            end
+        elseif x isa Array
+            lx = length(x)
+            if i + lx - 1 > l
+                ladd = lx > 16 ? lx : 16
+                _growend!(out, ladd)
+                l += ladd
+            end
+            for j in 1:lx
+                y = arrayref(true, x, j)
+                arrayset(true, out, y, i)
+                i += 1
+            end
+        else
+            for y in x
+                if i > l
+                    _growend!(out, 16)
+                    l += 16
+                end
+                arrayset(true, out, y, i)
+                i += 1
+            end
+        end
+    end
+    _deleteend!(out, l - i + 1)
+    return out
+end
+
+
 """
     Colon()
 

src/cgutils.cpp

@@ -1087,7 +1087,7 @@ static std::pair<Value*, bool> emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x,
     }
 
     // intersection with Type needs to be handled specially
-    if (jl_has_intersect_type_not_kind(type)) {
+    if (jl_has_intersect_type_not_kind(type) || jl_has_intersect_type_not_kind(intersected_type)) {
         Value *vx = maybe_decay_untracked(boxed(ctx, x));
         Value *vtyp = maybe_decay_untracked(literal_pointer_val(ctx, type));
         if (msg && *msg == "typeassert") {
@@ -1131,6 +1131,16 @@ static std::pair<Value*, bool> emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x,
         return std::make_pair(ctx.builder.CreateICmpEQ(emit_typeof_boxed(ctx, x),
             maybe_decay_untracked(literal_pointer_val(ctx, intersected_type))), false);
     }
+    jl_datatype_t *dt = (jl_datatype_t*)jl_unwrap_unionall(intersected_type);
+    if (jl_is_datatype(dt) && !dt->abstract && jl_subtype(dt->name->wrapper, type)) {
+        // intersection is a supertype of all instances of its constructor,
+        // so the isa test reduces to a comparison of the typename by pointer
+        return std::make_pair(
+                ctx.builder.CreateICmpEQ(
+                    mark_callee_rooted(emit_datatype_name(ctx, emit_typeof_boxed(ctx, x))),
+                    mark_callee_rooted(literal_pointer_val(ctx, (jl_value_t*)dt->name))),
+                false);
+    }
     // everything else can be handled via subtype tests
     return std::make_pair(ctx.builder.CreateICmpNE(
             ctx.builder.CreateCall(prepare_call(jlsubtype_func),