JIT: RBO can now infer nested isinst outcomes from class relationships

src/coreclr/jit/compiler.h

@@ -8300,6 +8300,7 @@ class Compiler
     bool          optRelopTryInferWithOneEqualOperand(const VNFuncApp&      domApp,
                                                       const VNFuncApp&      treeApp,
                                                       RelopImplicationInfo* rii);
+    bool          optRelopTryInferFromTypeCheck(const VNFuncApp& domApp, RelopImplicationInfo* rii);
 
     /**************************************************************************
      *               Value/Assertion propagation

src/coreclr/jit/redundantbranchopts.cpp

@@ -501,6 +501,16 @@ void Compiler::optRelopImpliesRelop(RelopImplicationInfo* rii)
         }
     }
 
+    // See if dominating and tree compares are both of the form
+    //   EQ/NE(VNF_IsInstanceOf(clsA, obj), null)
+    // with the same object VN. Then we can infer the tree's value from the
+    // relationship between the two class handles.
+    //
+    if (optRelopTryInferFromTypeCheck(domApp, rii))
+    {
+        return;
+    }
+
     // See if dominating compare is a compound comparison that might
     // tell us the value of the tree compare.
     //
@@ -714,6 +724,201 @@ bool Compiler::optRelopTryInferWithOneEqualOperand(const VNFuncApp&      domApp,
     return true;
 }
 
+//------------------------------------------------------------------------
+// optRelopTryInferFromTypeCheck: infer the tree relop's value when both
+//   dom and tree relops compare a VNF_IsInstanceOf against null on the
+//   same object VN, using the class-handle relationship.
+//
+// Arguments:
+//   domApp - The dominating relop's VN func app
+//   rii    - [in/out] struct with relop implication information
+//            (rii->treeNormVN is the tree relop's VN)
+//
+// Returns:
+//   True if we set canInfer.
+//
+// Recognizes:
+//     dom:  EQ/NE(VNF_IsInstanceOf(clsA, obj), null)
+//    tree:  EQ/NE(VNF_IsInstanceOf(clsB, obj), null)
+//
+// Uses compareTypesForCast in both directions to infer whichever holds:
+//   * clsA derives from clsB     : obj is-a clsA implies obj is-a clsB.
+//   * clsB derives from clsA     : obj is-NOT-a clsA implies obj is-NOT-a clsB.
+//   * clsA and clsB unrelated    : obj is-a clsA implies obj is-NOT-a clsB.
+//
+bool Compiler::optRelopTryInferFromTypeCheck(const VNFuncApp& domApp, RelopImplicationInfo* rii)
+{
+    // Both dom and tree must be EQ or NE.
+    genTreeOps const domOper = genTreeOps(domApp.GetFunc());
+    if (!GenTree::StaticOperIs(domOper, GT_EQ, GT_NE))
+    {
+        return false;
+    }
+
+    VNFuncApp treeApp;
+    if (!vnStore->GetVNFunc(rii->treeNormVN, &treeApp))
+    {
+        return false;
+    }
+
+    genTreeOps const treeOper = genTreeOps(treeApp.GetFunc());
+    if (!GenTree::StaticOperIs(treeOper, GT_EQ, GT_NE))
+    {
+        return false;
+    }
+
+    // Canonicalize: put the (potentially) IsInstanceOf side on arg 0 and null side on arg 1.
+    auto extractIsInst = [this](const VNFuncApp& app, ValueNum* clsVN, ValueNum* objVN) -> bool {
+        for (unsigned side = 0; side < 2; side++)
+        {
+            ValueNum lhs = app.GetArg(side);
+            ValueNum rhs = app.GetArg(1 - side);
+
+            if (rhs != vnStore->VNForNull())
+            {
+                continue;
+            }
+
+            VNFuncApp isInstApp;
+            if (vnStore->GetVNFunc(lhs, &isInstApp) && (isInstApp.GetFunc() == VNF_IsInstanceOf))
+            {
+                *clsVN = isInstApp.GetArg(0);
+                *objVN = isInstApp.GetArg(1);
+                return true;
+            }
+        }
+        return false;
+    };
+
+    ValueNum domClsVN;
+    ValueNum domObjVN;
+    ValueNum treeClsVN;
+    ValueNum treeObjVN;
+    if (!extractIsInst(domApp, &domClsVN, &domObjVN) || !extractIsInst(treeApp, &treeClsVN, &treeObjVN))
+    {
+        return false;
+    }
+
+    // Must be on the same object VN.
+    if (domObjVN != treeObjVN)
+    {
+        return false;
+    }
+
+    // Same class handle - direct VN match should have handled this; be defensive.
+    if (domClsVN == treeClsVN)
+    {
+        return false;
+    }
+
+    // Both class-handle VNs must be constant type handles we can extract.
+    CORINFO_CLASS_HANDLE domCls  = NO_CLASS_HANDLE;
+    CORINFO_CLASS_HANDLE treeCls = NO_CLASS_HANDLE;
+    if (!vnStore->IsVNTypeHandle(domClsVN, &domCls) || !vnStore->IsVNTypeHandle(treeClsVN, &treeCls))
+    {
+        return false;
+    }
+
+    // Ask the EE how the two class handles relate. `compareTypesForCast(A, B)` answers
+    // CanCastTo(A, B), i.e., "does A derive from / implement B?". We combine forward and
+    // reverse queries to figure out which "obj is-a" implication is sound:
+    //
+    //   forwardCast == Must:                   domCls derives from treeCls.
+    //                                            obj is-a domCls -> obj is-a treeCls.
+    //   forwardCast, reverseCast == MustNot:   domCls and treeCls are unrelated types.
+    //                                            obj is-a domCls -> obj is-NOT-a treeCls.
+    //                                            (Sound only for non-interface classes:
+    //                                            interfaces can be co-implemented, and a
+    //                                            treeCls interface is subject to IDCC.)
+    //   reverseCast == Must:                   treeCls derives from domCls.
+    //                                            obj is-a treeCls -> obj is-a domCls, so
+    //                                            (contrapositive) obj is-NOT-a domCls ->
+    //                                            obj is-NOT-a treeCls. (Sound if treeCls is
+    //                                            not an interface: IDCC could otherwise make
+    //                                            an unrelated object claim to be treeCls.)
+    //
+    // The Must case gives us information when the dom relop asserts "obj is-a domCls";
+    // the reverseCast==Must case gives us information when it asserts "obj is-NOT-a domCls".
+    //
+    const unsigned domClsAttribs   = info.compCompHnd->getClassAttribs(domCls);
+    const unsigned treeClsAttribs  = info.compCompHnd->getClassAttribs(treeCls);
+    const bool     domIsInterface  = (domClsAttribs & CORINFO_FLG_INTERFACE) != 0;
+    const bool     treeIsInterface = (treeClsAttribs & CORINFO_FLG_INTERFACE) != 0;
+
+    TypeCompareState const forwardCast = info.compCompHnd->compareTypesForCast(domCls, treeCls);
+    TypeCompareState const reverseCast = info.compCompHnd->compareTypesForCast(treeCls, domCls);
+
+    // Pick the sound inference direction, if any.
+    //
+    //   inferredCondIsObjIsA:  true if the inference applies when "obj is-a domCls",
+    //                          false if it applies when "obj is-NOT-a domCls".
+    //   objIsBUnderCond:       value of "obj is-a treeCls" under that condition.
+    //
+    bool inferredCondIsObjIsA;
+    bool objIsBUnderCond;
+
+    if (forwardCast == TypeCompareState::Must)
+    {
+        inferredCondIsObjIsA = true;
+        objIsBUnderCond      = true;
+    }
+    else if ((forwardCast == TypeCompareState::MustNot) && (reverseCast == TypeCompareState::MustNot))
+    {
+        if (domIsInterface || treeIsInterface)
+        {
+            return false;
+        }
+        inferredCondIsObjIsA = true;
+        objIsBUnderCond      = false;
+    }
+    else if (reverseCast == TypeCompareState::Must)
+    {
+        if (treeIsInterface)
+        {
+            return false;
+        }
+        inferredCondIsObjIsA = false;
+        objIsBUnderCond      = false;
+    }
+    else
+    {
+        return false;
+    }
+
+    // Translate to the {canInferFromTrue, canInferFromFalse, reverseSense} tuple used by RBO.
+    //
+    //   dom relop = "EQ(IsInst(A,obj), null)" is true  when "obj is NOT A" and false when "obj is A"
+    //   dom relop = "NE(IsInst(A,obj), null)" is true  when "obj is A"     and false when "obj is NOT A"
+    //  tree relop = "EQ(IsInst(B,obj), null)" is true  when "obj is NOT B" and false when "obj is B"
+    //  tree relop = "NE(IsInst(B,obj), null)" is true  when "obj is B"     and false when "obj is NOT B"
+    //
+    // The RBO consumer applies:
+    //   relopValue = !reverseSense  when using canInferFromTrue  (dom relop is true)
+    //   relopValue =  reverseSense  when using canInferFromFalse (dom relop is false)
+    // so we set reverseSense accordingly.
+    //
+    bool const condHoldsWhenDomTrue = inferredCondIsObjIsA ? (domOper == GT_NE) : (domOper == GT_EQ);
+    bool const treeTrueMeansObjIsB  = (treeOper == GT_NE);
+    bool const treeValueUnderCond   = (objIsBUnderCond == treeTrueMeansObjIsB);
+
+    rii->canInfer          = true;
+    rii->vnRelation        = ValueNumStore::VN_RELATION_KIND::VRK_Inferred;
+    rii->canInferFromTrue  = condHoldsWhenDomTrue;
+    rii->canInferFromFalse = !condHoldsWhenDomTrue;
+    rii->reverseSense      = condHoldsWhenDomTrue ? !treeValueUnderCond : treeValueUnderCond;
+
+    JITDUMP("Can infer tree IsInstanceOf outcome from dominating IsInstanceOf: forwardCast = %s, "
+            "reverseCast = %s, canInferFromTrue = %s, canInferFromFalse = %s, reverseSense = %s\n",
+            (forwardCast == TypeCompareState::Must) ? "Must"
+                                                    : ((forwardCast == TypeCompareState::MustNot) ? "MustNot" : "May"),
+            (reverseCast == TypeCompareState::Must) ? "Must"
+                                                    : ((reverseCast == TypeCompareState::MustNot) ? "MustNot" : "May"),
+            rii->canInferFromTrue ? "true" : "false", rii->canInferFromFalse ? "true" : "false",
+            rii->reverseSense ? "true" : "false");
+
+    return true;
+}
+
 //------------------------------------------------------------------------
 // optRedundantDominatingBranch: see if we can optimize a branch in a
 //    dominating block.