implement Method edge encoding

This allows us to distinguish between edges from failed
`abstract_call_method` that don’t need to be recorded and those from
`abstract_applicable` and `_hasmethod_tfunc` that do need to be
recorded as backedges.

Author: aviatesk

base/compiler/stmtinfo.jl

@@ -44,7 +44,8 @@ struct MethodMatchInfo <: CallInfo
         return new(results, mt, atype, fullmatch, edges)
     end
 end
-function add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo)
+add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo) = _add_edges_impl(edges, info)
+function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Bool=false)
     if !fully_covering(info)
         # add legacy-style missing backedge info also
         exists = false
@@ -66,7 +67,7 @@ function add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo)
         edge = info.edges[1]
         m = info.results[1]
         if edge === nothing
-            mi = specialize_method(m)
+            mi = specialize_method(m) # don't allow `Method`-edge for this optimized format
             edge = mi
         else
             mi = edge.def
@@ -88,13 +89,11 @@ function add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo)
         edge = info.edges[i]
         m = info.results[i]
         if edge === nothing
-            # push!(edges, m.method)
-            mi = specialize_method(m)
-            push!(edges, mi)
+            edge = mi_edge ? specialize_method(m) : m.method
         else
             @assert edge.def.def === m.method
-            push!(edges, edge)
         end
+        push!(edges, edge)
     end
     nothing
 end
@@ -112,11 +111,11 @@ function add_one_edge!(edges::Vector{Any}, edge::MethodInstance)
 end
 function add_one_edge!(edges::Vector{Any}, edge::CodeInstance)
     for i in 1:length(edges)
-        edgeᵢ = edges[i]
+        edgeᵢ_orig = edgeᵢ = edges[i]
         edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def)
         edgeᵢ isa MethodInstance || continue
         if edgeᵢ === edge.def && !(i > 1 && edges[i-1] isa Type)
-            if edges[i] isa MethodInstance
+            if edgeᵢ_orig isa MethodInstance
                 # found edge we can upgrade
                 edges[i] = edge
                 return
@@ -149,7 +148,9 @@ struct UnionSplitInfo <: CallInfo
     split::Vector{MethodMatchInfo}
 end
 add_edges_impl(edges::Vector{Any}, info::UnionSplitInfo) =
-    for split in info.split; add_edges!(edges, split); end
+    _add_edges_impl(edges, info)
+_add_edges_impl(edges::Vector{Any}, info::UnionSplitInfo, mi_edge::Bool=false) =
+    for split in info.split; _add_edges_impl(edges, split, mi_edge); end
 nsplit_impl(info::UnionSplitInfo) = length(info.split)
 getsplit_impl(info::UnionSplitInfo, idx::Int) = getsplit(info.split[idx], 1)
 getresult_impl(::UnionSplitInfo, ::Int) = nothing
@@ -294,21 +295,21 @@ struct InvokeCallInfo <: CallInfo
     result::Union{Nothing,ConstResult}
     atype # ::Type
 end
-function add_edges_impl(edges::Vector{Any}, info::InvokeCallInfo)
+add_edges_impl(edges::Vector{Any}, info::InvokeCallInfo) =
+    _add_edges_impl(edges, info)
+function _add_edges_impl(edges::Vector{Any}, info::InvokeCallInfo, mi_edge::Bool=false)
     edge = info.edge
     if edge === nothing
-        mi = specialize_method(info.match)
-        add_invoke_edge!(edges, info.atype, mi)
-    else
-        add_invoke_edge!(edges, info.atype, edge)
+        edge = mi_edge ? specialize_method(info.match) : info.match.method
     end
+    add_invoke_edge!(edges, info.atype, edge)
     nothing
 end
-function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), edge::MethodInstance)
+function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), edge::Union{MethodInstance,Method})
     for i in 2:length(edges)
         edgeᵢ = edges[i]
         edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def)
-        edgeᵢ isa MethodInstance || continue
+        edgeᵢ isa MethodInstance || edgeᵢ isa Method || continue
         if edgeᵢ === edge
             edge_minus_1 = edges[i-1]
             if edge_minus_1 isa Type && edge_minus_1 == atype
@@ -321,13 +322,13 @@ function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), edge::Method
 end
 function add_invoke_edge!(edges::Vector{Any}, @nospecialize(atype), edge::CodeInstance)
     for i in 2:length(edges)
-        edgeᵢ = edges[i]
+        edgeᵢ_orig = edgeᵢ = edges[i]
         edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def)
-        edgeᵢ isa MethodInstance || continue
-        if edgeᵢ === edge.def
+        if ((edgeᵢ isa MethodInstance && edgeᵢ === edge.def) ||
+            (edgeᵢ isa Method && edgeᵢ === edge.def.def))
             edge_minus_1 = edges[i-1]
             if edge_minus_1 isa Type && edge_minus_1 == atype
-                if edges[i] isa MethodInstance
+                if edgeᵢ_orig isa MethodInstance || edgeᵢ_orig isa Method
                     # found edge we can upgrade
                     edges[i] = edge
                     return
@@ -426,8 +427,9 @@ end
 add_edges_impl(edges::Vector{Any}, info::ModifyOpInfo) = add_edges!(edges, info.info)
 
 struct VirtualMethodMatchInfo <: CallInfo
-    info::CallInfo
+    info::Union{MethodMatchInfo,UnionSplitInfo,InvokeCallInfo}
 end
-add_edges_impl(edges::Vector{Any}, info::VirtualMethodMatchInfo) = add_edges!(edges, info.info)
+add_edges_impl(edges::Vector{Any}, info::VirtualMethodMatchInfo) =
+    _add_edges_impl(edges, info.info, #=mi_edge=#true)
 
 @specialize

base/compiler/utilities.jl

@@ -263,21 +263,32 @@ function iterate(iter::BackedgeIterator, i::Int=1)
     while true
         i > length(backedges) && return nothing
         item = backedges[i]
-        item isa Int && (i += 2; continue) # ignore the query information if present
-        # TODO: These `MethodInstance`s can be categorized into two types:
-        # those that should be recorded as forward edges but are unnecessary as backedges
-        # (such as cases where `abstract_call_method` fails), and
-        # those that should be recorded as backedges, such as those provided by
-        # `abstract_applicable` or user-provided edges.
-        # Ideally, we should avoid recording the former cases as backedges and instead
-        # prepare a special token to represent them.
-        # isa(item, MethodInstance) && (i += 1; continue) # ignore edges which don't contribute info (future style edges)
-        isa(item, CodeInstance) && (item = item.def)
-        isa(item, MethodInstance) && return BackedgePair(nothing, item), i+1      # regular dispatch
-        isa(item, MethodTable) && return BackedgePair(backedges[i+1], item), i+2  # abstract dispatch (legacy style edges)
-        target = backedges[i+1]
-        isa(target, CodeInstance) && (target = target.def)
-        return BackedgePair(item, target::MethodInstance), i+2                    # `invoke` calls
+        if item isa Int
+            i += 2
+            continue # ignore the query information if present
+        elseif isa(item, Method)
+            # ignore `Method`-edges (from e.g. failed `abstract_call_method`)
+            i += 1
+            continue
+        end
+        if isa(item, CodeInstance)
+            item = item.def
+        end
+        if isa(item, MethodInstance) # regular dispatch
+            return BackedgePair(nothing, item), i+1
+        elseif isa(item, MethodTable) # abstract dispatch (legacy style edges)
+            return BackedgePair(backedges[i+1], item), i+2
+        else # `invoke` call
+            callee = backedges[i+1]
+            if isa(callee, Method)
+                i += 2
+                continue
+            end
+            if isa(callee, CodeInstance)
+                callee = callee.def
+            end
+            return BackedgePair(item, callee::MethodInstance), i+2
+        end
     end
 end
 

src/staticdata_utils.c

@@ -750,20 +750,19 @@ static void jl_copy_roots(jl_array_t *method_roots_list, uint64_t key)
     }
 }
 
-static size_t verify_invokesig(jl_value_t *invokesig, jl_method_instance_t *expected, size_t minworld)
+static size_t verify_invokesig(jl_value_t *invokesig, jl_method_t *expected, size_t minworld)
 {
     assert(jl_is_type(invokesig));
-    assert(jl_is_method_instance(expected));
-    jl_method_t *m = ((jl_method_instance_t*)expected)->def.method;
+    assert(jl_is_method(expected));
     size_t min_valid = 0;
     size_t max_valid = ~(size_t)0;
-    if (jl_egal(invokesig, m->sig)) {
-        // the invoke match is `m` for `m->sig`, unless `m` is invalid
-        if (jl_atomic_load_relaxed(&m->deleted_world) < max_valid)
+    if (jl_egal(invokesig, expected->sig)) {
+        // the invoke match is `expected` for `expected->sig`, unless `expected` is invalid
+        if (jl_atomic_load_relaxed(&expected->deleted_world) < max_valid)
             max_valid = 0;
     }
     else {
-        jl_methtable_t *mt = jl_method_get_table(m);
+        jl_methtable_t *mt = jl_method_get_table(expected);
         if ((jl_value_t*)mt == jl_nothing) {
             max_valid = 0;
         }
@@ -773,7 +772,7 @@ static size_t verify_invokesig(jl_value_t *invokesig, jl_method_instance_t *expe
                 max_valid = 0;
             }
             else {
-                if (((jl_method_match_t*)matches)->method != m) {
+                if (((jl_method_match_t*)matches)->method != expected) {
                     max_valid = 0;
                 }
             }
@@ -808,8 +807,14 @@ static size_t verify_call(jl_value_t *sig, jl_svec_t *expecteds, size_t i, size_
                 jl_value_t *t = jl_svecref(expecteds, j + i);
                 if (jl_is_code_instance(t))
                     t = (jl_value_t*)((jl_code_instance_t*)t)->def;
-                assert(jl_is_method_instance(t));
-                if (match == ((jl_method_instance_t*)t)->def.method)
+                jl_method_t *meth;
+                if (jl_is_method(t))
+                    meth = (jl_method_t*)t;
+                else {
+                    assert(jl_is_method_instance(t));
+                    meth = ((jl_method_instance_t*)t)->def.method;
+                }
+                if (match == meth)
                     break;
             }
             if (j == n) {
@@ -853,6 +858,7 @@ static size_t jl_verify_method(jl_code_instance_t *codeinst, size_t minworld, ar
     for (size_t j = 0; j < jl_svec_len(callees); ) {
         jl_value_t *edge = jl_svecref(callees, j);
         size_t max_valid2;
+        assert(!jl_is_method(edge)); // `Method`-edge isn't allowed for the optimized one-edge format
         if (jl_is_code_instance(edge))
             edge = (jl_value_t*)((jl_code_instance_t*)edge)->def;
         if (jl_is_method_instance(edge)) {
@@ -875,10 +881,17 @@ static size_t jl_verify_method(jl_code_instance_t *codeinst, size_t minworld, ar
             continue;
         }
         else {
-            jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(callees, j + 1);
-            if (jl_is_code_instance(mi))
-                mi = ((jl_code_instance_t*)mi)->def;
-            max_valid2 = verify_invokesig(edge, mi, minworld);
+            jl_method_instance_t *callee = (jl_method_instance_t*)jl_svecref(callees, j + 1);
+            jl_method_t *meth;
+            if (jl_is_code_instance(callee))
+                callee = ((jl_code_instance_t*)callee)->def;
+            if (jl_is_method_instance(callee))
+                meth = callee->def.method;
+            else {
+                assert(jl_is_method(callee));
+                meth = (jl_method_t*)callee;
+            }
+            max_valid2 = verify_invokesig(edge, meth, minworld);
             j += 2;
         }
         if (max_valid2 < max_valid)
@@ -1069,16 +1082,20 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_ci_list, size
                 // if this callee is still valid, add all the backedges
                 for (size_t j = 0; j < jl_svec_len(callees); ) {
                     jl_value_t *edge = jl_svecref(callees, j);
+                    if (jl_is_long(edge)) {
+                        j += 2; // skip over signature and count but not methods
+                        continue;
+                    }
+                    else if (jl_is_method(edge)) {
+                        j += 1;
+                        continue;
+                    }
                     if (jl_is_code_instance(edge))
                         edge = (jl_value_t*)((jl_code_instance_t*)edge)->def;
                     if (jl_is_method_instance(edge)) {
                         jl_method_instance_add_backedge((jl_method_instance_t*)edge, NULL, codeinst);
                         j += 1;
                     }
-                    else if (jl_is_long(edge)) {
-                        j += 2; // skip over signature and count but not methods
-                        continue;
-                    }
                     else if (jl_is_mtable(edge)) {
                         jl_methtable_t *mt = (jl_methtable_t*)edge;
                         jl_value_t *sig = jl_svecref(callees, j + 1);
@@ -1089,6 +1106,10 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_ci_list, size
                         jl_value_t *callee = jl_svecref(callees, j + 1);
                         if (jl_is_code_instance(callee))
                             callee = (jl_value_t*)((jl_code_instance_t*)callee)->def;
+                        else if (jl_is_method(callee)) {
+                            j += 2;
+                            continue;
+                        }
                         jl_method_instance_add_backedge((jl_method_instance_t*)callee, edge, codeinst);
                         j += 2;
                     }