JIT: fold isinst using reaching VNs and compareTypesForCast

src/coreclr/jit/assertionprop.cpp

@@ -2386,24 +2386,58 @@ AssertionIndex Compiler::optAssertionIsSubrange(GenTree* tree, IntegralRange ran
     return NO_ASSERTION_INDEX;
 }
 
-/**********************************************************************************
- *
- * Given a "tree" that is usually arg1 of a isinst/cast kind of GT_CALL (a class
- * handle), and "methodTableArg" which is a const int (a class handle), then search
- * if there is an assertion in "assertions", that asserts the equality of the two
- * class handles and then returns the index of the assertion. If one such assertion
- * could not be found, then it returns NO_ASSERTION_INDEX.
- *
- */
-AssertionIndex Compiler::optAssertionIsSubtype(GenTree* tree, GenTree* methodTableArg, ASSERT_VALARG_TP assertions)
+//------------------------------------------------------------------------
+// optAssertionVNIsSubtype: see if a VN is known to be a subtype of castTo
+//    using the given assertion set, VN-level type info, and assertions that
+//    reach via PHI definitions.
+//
+// Arguments:
+//   objVN      - VN to check
+//   castToVN   - VN representing the type handle being cast to.
+//   assertions - set of live assertions
+//   budget     - limits the depth of recursion when chasing assertions across
+//                phi-def reaching VNs.
+//
+// Return Value:
+//   True if the VN is known to be a subtype of castTo.
+//
+bool Compiler::optAssertionVNIsSubtype(ValueNum objVN, ValueNum castToVN, ASSERT_VALARG_TP assertions, int budget)
 {
+    if ((budget <= 0) || (objVN == ValueNumStore::NoVN))
+    {
+        return false;
+    }
+
+    bool isExact;
+    bool isNonNull;
+
+    CORINFO_CLASS_HANDLE castTo;
+    if (!vnStore->IsVNTypeHandle(castToVN, &castTo))
+    {
+        return false;
+    }
+    assert(castTo != NO_CLASS_HANDLE);
+
+    // First, try the VN's version of gtGetClassHandle over the vn itself, e.g. vn being
+    // Jit_NewObj(MyClass) while we're trying to prove "Jit_NewObj(MyClass) is MyClass".
+    CORINFO_CLASS_HANDLE castFromVN = vnStore->GetObjectType(objVN, &isExact, &isNonNull);
+    if ((castFromVN != NO_CLASS_HANDLE) &&
+        (info.compCompHnd->compareTypesForCast(castFromVN, castTo) == TypeCompareState::Must))
+    {
+        return true;
+    }
+
+    // Now look through assertions directly on the VN.
+    // We're looking for "vn is (exactly/subtype) cls" assertions that can help us prove the cast.
     BitVecOps::Iter iter(apTraits, assertions);
     unsigned        bvIndex = 0;
     while (iter.NextElem(&bvIndex))
     {
         AssertionIndex const index        = GetAssertionIndex(bvIndex);
         const AssertionDsc&  curAssertion = optGetAssertion(index);
-        if (!curAssertion.KindIs(OAK_EQUAL) || !curAssertion.GetOp1().KindIs(O1K_SUBTYPE, O1K_EXACT_TYPE))
+
+        if (!curAssertion.KindIs(OAK_EQUAL) || !curAssertion.GetOp1().KindIs(O1K_SUBTYPE, O1K_EXACT_TYPE) ||
+            (curAssertion.GetOp1().GetVN() != objVN))
         {
             // TODO-CQ: We might benefit from OAK_NOT_EQUAL assertion as well, e.g.:
             // if (obj is not MyClass) // obj is known to be never of MyClass class
@@ -2414,27 +2448,28 @@ AssertionIndex Compiler::optAssertionIsSubtype(GenTree* tree, GenTree* methodTab
             continue;
         }
 
-        if ((curAssertion.GetOp1().GetVN() != vnStore->VNConservativeNormalValue(tree->gtVNPair) ||
-             !curAssertion.GetOp2().KindIs(O2K_CONST_INT)))
+        // Extract CORINFO_CLASS_HANDLE from curAssertion.GetOp2()
+        CORINFO_CLASS_HANDLE cls;
+        if (!vnStore->IsVNTypeHandle(curAssertion.GetOp2().GetVN(), &cls))
         {
             continue;
         }
 
-        ssize_t      methodTableVal = 0;
-        GenTreeFlags iconFlags      = GTF_EMPTY;
-        if (!optIsTreeKnownIntValue(!optLocalAssertionProp, methodTableArg, &methodTableVal, &iconFlags))
+        // Now we have "objVN is (exactly/subtype) cls" assertion.
+        // We want to see if this implies "objVN is (exactly/subtype) castTo".
+        if (info.compCompHnd->compareTypesForCast(cls, castTo) == TypeCompareState::Must)
         {
-            continue;
-        }
-
-        if (curAssertion.GetOp2().GetIntConstant() == methodTableVal)
-        {
-            // TODO-CQ: if they don't match, we might still be able to prove that the result is foldable via
-            // compareTypesForCast.
-            return index;
+            // The assertion implies the cast is always successful.
+            return true;
         }
     }
-    return NO_ASSERTION_INDEX;
+
+    // For PHI-defs, walk reaching assertions/VNs and recursively check.
+    return optVisitReachingAssertions(objVN,
+                                      [this, castToVN, budget](ValueNum reachingVN, ASSERT_TP reachingAssertions) {
+        return optAssertionVNIsSubtype(reachingVN, castToVN, reachingAssertions, budget - 1) ? AssertVisit::Continue
+                                                                                             : AssertVisit::Abort;
+    }) == AssertVisit::Continue;
 }
 
 //------------------------------------------------------------------------------
@@ -5361,11 +5396,12 @@ GenTree* Compiler::optAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCal
             CallArg* objCallArg    = call->gtArgs.GetArgByIndex(1);
             GenTree* castToArg     = castToCallArg->GetNode();
             GenTree* objArg        = objCallArg->GetNode();
+            ValueNum objVN         = optConservativeNormalVN(objArg);
+            ValueNum castToVN      = optConservativeNormalVN(castToArg);
 
-            const unsigned index = optAssertionIsSubtype(objArg, castToArg, assertions);
-            if (index != NO_ASSERTION_INDEX)
+            if (optAssertionVNIsSubtype(objVN, castToVN, assertions))
             {
-                JITDUMP("\nDid VN based subtype prop for index #%02u in " FMT_BB ":\n", index, compCurBB->bbNum);
+                JITDUMP("\nDid VN based subtype prop in " FMT_BB ":\n", compCurBB->bbNum);
                 DISPTREE(call);
 
                 // if castObjArg is not simple, we replace the arg with a temp assignment and

src/coreclr/jit/compiler.h

@@ -9075,9 +9075,9 @@ class Compiler
 
     // Used for respective assertion propagations.
     AssertionIndex optAssertionIsSubrange(GenTree* tree, IntegralRange range, ASSERT_VALARG_TP assertions);
-    AssertionIndex optAssertionIsSubtype(GenTree* tree, GenTree* methodTableArg, ASSERT_VALARG_TP assertions);
-    bool           optAssertionVNIsNonNull(ValueNum vn, ASSERT_VALARG_TP assertions, int budget = 10);
-    bool           optAssertionIsNonNull(GenTree* op, ASSERT_VALARG_TP assertions);
+    bool optAssertionVNIsSubtype(ValueNum objVN, ValueNum castToVN, ASSERT_VALARG_TP assertions, int budget = 10);
+    bool optAssertionVNIsNonNull(ValueNum vn, ASSERT_VALARG_TP assertions, int budget = 10);
+    bool optAssertionIsNonNull(GenTree* op, ASSERT_VALARG_TP assertions);
 
     AssertionIndex optGlobalAssertionIsEqualOrNotEqual(ASSERT_VALARG_TP assertions, GenTree* op1, GenTree* op2);
     AssertionIndex optLocalAssertionIsEqualOrNotEqual(

src/coreclr/jit/compiler.hpp

@@ -2101,7 +2101,8 @@ void GenTree::BashToConst(T value, var_types type /* = TYP_UNDEF */)
             }
 
             AsIntCon()->SetIconValue(static_cast<ssize_t>(value));
-            AsIntCon()->gtFieldSeq = nullptr;
+            AsIntCon()->gtFieldSeq          = nullptr;
+            AsIntCon()->gtCompileTimeHandle = 0;
             break;
 
 #if !defined(TARGET_64BIT)

src/coreclr/jit/valuenum.cpp

@@ -7174,6 +7174,30 @@ bool ValueNumStore::IsVNTypeHandle(ValueNum vn)
     return IsVNHandle(vn, GTF_ICON_CLASS_HDL);
 }
 
+//------------------------------------------------------------------------
+// IsVNTypeHandle: check whether a VN represents a class type handle and,
+//    if so, recover the underlying compile-time class handle.
+//
+// Arguments:
+//    vn   - the VN to inspect
+//    pCls - [out] compile-time class handle
+//
+// Return Value:
+//    True if vn is a constant class-handle VN and its compile-time class
+//    handle was found in the embedded-handle map; false otherwise.
+//
+bool ValueNumStore::IsVNTypeHandle(ValueNum vn, CORINFO_CLASS_HANDLE* pCls)
+{
+    ssize_t handle = 0;
+    if (IsVNTypeHandle(vn) && EmbeddedHandleMapLookup(ConstantValue<ssize_t>(vn), &handle) && (handle != 0))
+    {
+        *pCls = reinterpret_cast<CORINFO_CLASS_HANDLE>(handle);
+        return true;
+    }
+    *pCls = NO_CLASS_HANDLE;
+    return false;
+}
+
 //------------------------------------------------------------------------
 // SwapRelop: return VNFunc for swapped relop
 //
@@ -12263,8 +12287,12 @@ void Compiler::fgValueNumberTreeConst(GenTree* tree)
                 const GenTreeIntCon* cns         = tree->AsIntCon();
                 const GenTreeFlags   handleFlags = tree->GetIconHandleFlag();
                 tree->gtVNPair.SetBoth(vnStore->VNForHandle(cns->IconValue(), handleFlags));
-                if (handleFlags == GTF_ICON_CLASS_HDL)
+                if ((handleFlags == GTF_ICON_CLASS_HDL) && (cns->gtCompileTimeHandle != 0))
                 {
+                    // Skip registration when gtCompileTimeHandle is unknown (e.g., the node was created by
+                    // BashToConst+gtFlags|=GTF_ICON_CLASS_HDL in optConstantAssertionProp). Overwriting an
+                    // existing valid mapping with 0 would poison the map for any constant assertion-based
+                    // re-flagging that happens to share the same iconValue as a real embedded handle node.
                     vnStore->AddToEmbeddedHandleMap(cns->IconValue(), cns->gtCompileTimeHandle);
                 }
             }
@@ -14406,9 +14434,11 @@ bool Compiler::fgValueNumberSpecialIntrinsic(GenTreeCall* call)
             ValueNum clsVN     = typeHandleFuncApp.m_args[0];
             ssize_t  clsHandle = 0;
 
-            // NOTE: EmbeddedHandleMapLookup may return 0 for non-0 embedded handle
-            if (!vnStore->EmbeddedHandleMapLookup(vnStore->ConstantValue<ssize_t>(clsVN), &clsHandle) &&
-                (clsHandle != 0))
+            // EmbeddedHandleMapLookup may return false (no entry) or true with a 0 handle when the
+            // backing iconNode was produced by BashToConst (i.e., it has no compile-time handle).
+            // Either way, we cannot resolve the runtime type, so bail.
+            if (!vnStore->EmbeddedHandleMapLookup(vnStore->ConstantValue<ssize_t>(clsVN), &clsHandle) ||
+                (clsHandle == 0))
             {
                 break;
             }
@@ -15849,17 +15879,15 @@ CORINFO_CLASS_HANDLE ValueNumStore::GetObjectType(ValueNum vn, bool* pIsExact, b
     const VNFunc func = funcApp.m_func;
     if ((func == VNF_CastClass) || (func == VNF_IsInstanceOf) || (func == VNF_JitNew))
     {
-        ssize_t  clsHandle = 0;
-        ValueNum clsVN     = funcApp.m_args[0];
+        ValueNum clsVN = funcApp.m_args[0];
 
-        // NOTE: EmbeddedHandleMapLookup may return 0 for non-0 embedded handle
-        if (IsVNTypeHandle(clsVN) && EmbeddedHandleMapLookup(ConstantValue<ssize_t>(clsVN), &clsHandle) &&
-            (clsHandle != 0))
+        CORINFO_CLASS_HANDLE clsHandle;
+        if (IsVNTypeHandle(clsVN, &clsHandle))
         {
             // JitNew returns an exact and non-null obj, castclass and isinst do not have this guarantee.
             *pIsNonNull = func == VNF_JitNew;
             *pIsExact   = func == VNF_JitNew;
-            return (CORINFO_CLASS_HANDLE)clsHandle;
+            return clsHandle;
         }
     }
 

src/coreclr/jit/valuenum.h

@@ -1133,6 +1133,12 @@ class ValueNumStore
     // Returns true iff the VN represents a Type handle constant.
     bool IsVNTypeHandle(ValueNum vn);
 
+    // Returns true iff the VN represents a Type handle constant. If so,
+    // *pCls is set to the resolved compile-time class handle (looked up
+    // through the embedded-handle map so AOT/R2R-encoded handles are
+    // mapped back). On failure *pCls is set to NO_CLASS_HANDLE.
+    bool IsVNTypeHandle(ValueNum vn, CORINFO_CLASS_HANDLE* pCls);
+
     // Returns true iff the VN represents a relop
     bool IsVNRelop(ValueNum vn, VNFuncApp* pFuncApp = nullptr);
 

src/tests/JIT/opt/AssertionPropagation/IsInstReachingVN.cs

@@ -0,0 +1,92 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT License.
+
+// Regression test for assertion-prop / VN folding of `isinst` across reaching
+// definitions (PHIs of typed allocations).
+//
+// These tests exercise the path in optAssertionVNIsSubtype where the cast
+// target type is proven from reaching VNs (VNF_JitNew / VNF_JitNewArr) for
+// PHI inputs. The JIT must:
+//   * Fold `isinst BaseClass` to true when every reaching VN is a subtype.
+//   * NOT fold to true when any reaching VN can be null or an unrelated type.
+
+using System;
+using System.Runtime.CompilerServices;
+using Xunit;
+
+public class IsInstReachingVN
+{
+    public class BaseClass { }
+    public class DerivedClass1 : BaseClass { }
+    public class DerivedClass2 : BaseClass { }
+    public class Unrelated { }
+
+    public static object s_sink;
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static bool PhiOfDerived(int len)
+    {
+        object bc;
+        if (len > 100)
+            bc = new DerivedClass1();
+        else
+            bc = new DerivedClass2();
+
+        // Keep `bc` live so DCE cannot remove the allocations / cast.
+        s_sink = bc;
+        return bc is BaseClass;
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static bool PhiWithNull(int len)
+    {
+        object bc;
+        if (len > 100)
+            bc = new DerivedClass1();
+        else
+            bc = null;
+
+        s_sink = bc;
+        // Must NOT be folded to true: null reaches here.
+        return bc is BaseClass;
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static bool DirectAlloc()
+    {
+        object bc = new DerivedClass1();
+        s_sink = bc;
+        return bc is BaseClass;
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    public static bool PhiWithUnrelated(int len)
+    {
+        object bc;
+        if (len > 100)
+            bc = new DerivedClass1();
+        else
+            bc = new Unrelated();
+
+        s_sink = bc;
+        // Must NOT be folded to true: Unrelated is not a BaseClass.
+        return bc is BaseClass;
+    }
+
+    [Fact]
+    public static int TestEntryPoint()
+    {
+        if (!PhiOfDerived(200)) return 101;
+        if (!PhiOfDerived(50))  return 102;
+
+        if (PhiWithNull(50))    return 103;
+        if (!PhiWithNull(200))  return 104;
+
+        if (!DirectAlloc())     return 105;
+
+        if (!PhiWithUnrelated(200)) return 106;
+        if (PhiWithUnrelated(50))   return 107;
+
+        return 100;
+    }
+}

src/tests/JIT/opt/AssertionPropagation/IsInstReachingVN.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <DebugType>None</DebugType>
+    <Optimize>True</Optimize>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="IsInstReachingVN.cs" />
+  </ItemGroup>
+</Project>