Avoid commandeering typevar from the other side if it's not valid.

Fix #39088.

Author: N5N3

src/subtype.c

@@ -74,7 +74,13 @@ typedef struct jl_varbinding_t {
     // 1 - var.ub = ub; return var
     // 2 - either (var.ub = ub; return var), or return ub
     int8_t constraintkind;
-    int8_t intvalued;      // must be integer-valued; i.e. occurs as N in Vararg{_,N}
+    // intvalued: must be integer-valued; i.e. occurs as N in Vararg{_,N}
+    // 0: No restriction
+    // 1: must be unbounded/ or fixed to a `Int`/typevar
+    // 2: This var has been commandeered once.
+    // 3: This var has been commandeered more than once.
+    // 4: This var has been commandeered more than once, and it has no fixed ub/lb.
+    int8_t intvalued;
     int8_t limited;
     int16_t depth0;         // # of invariant constructors nested around the UnionAll type for this var
     // when this variable's integer value is compared to that of another,
@@ -2196,12 +2202,23 @@ static jl_value_t *bound_var_below(jl_tvar_t *tv, jl_varbinding_t *bb, jl_stenv_
     if (jl_is_long(bb->lb)) {
         size_t blb = jl_unbox_long(bb->lb);
         if (blb < bb->offset || blb < 0)
+        if (jl_unbox_long(bb->lb) < bb->offset || jl_unbox_long(bb->lb) < 0)
             return jl_bottom_type;
         // Here we always return the shorter `Vararg`'s length.
         if (bb->offset <= 0)
             return bb->lb;
         return jl_box_long(jl_unbox_long(bb->lb) - bb->offset);
     }
+    if (bb->intvalued == 1 && bb->offset > 0)
+        // We really should return `tv` here, but some subtype tests need it.
+        // As a temparay fix, increase `intvalued` to 2.
+        bb->intvalued = 2;
+    else if (bb->intvalued == 2)
+        // If bb has been commandeered, we should always increase `intvalued`.
+        bb->intvalued = 3;
+    else if (bb->intvalued == 4 && bb->offset > 0)
+        // bb has no fixed bounds, return `NULL` if `tv` is not correct.
+        return NULL;
     return (jl_value_t*)tv;
 }
 
@@ -2453,6 +2470,13 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind
         }
     }
 
+    // This typevar has been commandeered by the other side once. Which is allowed if it's unbounded.
+    if (vb->intvalued == 2 && vb->lb == jl_bottom_type && vb->ub == (jl_value_t*)jl_any_type)
+        vb->intvalued = 1;
+    // This typevar has been commandeered more than once. Set it to 4 if its not fixed.
+    if (vb->intvalued == 3 && !(varval && jl_is_long(varval)))
+        vb->intvalued = 4;
+
     // TODO: this can prevent us from matching typevar identities later
     if (!varval && (vb->lb != vb->var->lb || vb->ub != vb->var->ub))
         newvar = jl_new_typevar(vb->var->name, vb->lb, vb->ub);
@@ -2645,7 +2669,7 @@ static jl_value_t *intersect_unionall(jl_value_t *t, jl_unionall_t *u, jl_stenv_
             e->vars->limited = 1;
     }
     else if (res != jl_bottom_type) {
-        if (vb.concrete || vb.occurs_inv>1 || u->var->lb != jl_bottom_type || (vb.occurs_inv && vb.occurs_cov)) {
+        if (vb.concrete || vb.occurs_inv>1 || vb.intvalued > 1 || u->var->lb != jl_bottom_type || (vb.occurs_inv && vb.occurs_cov)) {
             restore_env(e, NULL, &se);
             vb.occurs_cov = vb.occurs_inv = 0;
             vb.constraintkind = vb.concrete ? 1 : 2;
@@ -2702,7 +2726,7 @@ static jl_value_t *intersect_varargs(jl_vararg_t *vmx, jl_vararg_t *vmy, size_t
     if (xp2 && jl_is_typevar(xp2)) {
         xb = lookup(e, (jl_tvar_t*)xp2);
         if (xb) {
-            xb->intvalued = 1;
+            if (xb->intvalued == 0) xb->intvalued = 1;
             xb->offset = offset;
         }
         if (!yp2)
@@ -2711,7 +2735,7 @@ static jl_value_t *intersect_varargs(jl_vararg_t *vmx, jl_vararg_t *vmy, size_t
     if (yp2 && jl_is_typevar(yp2)) {
         yb = lookup(e, (jl_tvar_t*)yp2);
         if (yb) {
-            yb->intvalued = 1;
+            if (yb->intvalued == 0) yb->intvalued = 1;
             yb->offset = -offset;
         }
         if (!xp2)

test/subtype.jl

@@ -2080,3 +2080,34 @@ let A = Tuple{NTuple{N,Any},Val{N}} where {N},
     @testintersect(A, Tuple{Tuple{Any,Any,Any,Any,Any,Vararg{Any}},Val{4}}, Union{})
     @testintersect(A, Tuple{Tuple{Any,Any,Any,Any,Any,Vararg{Any,N}} where {N},Val{4}}, Union{})
 end
+
+#issue #39088
+let
+    a() = c((1,), (1,1,1,1))
+    c(d::NTuple{T}, ::NTuple{T}) where T = d
+    c(d::NTuple{f}, b) where f = c((d..., f), b)
+    j(h::NTuple{T}, ::NTuple{T} = a()) where T = nothing
+    @test j((1,1,1,1)) === nothing
+end
+
+let A = Tuple{NTuple{N, Int}, NTuple{N, Int}} where N,
+    C = Tuple{NTuple{4, Int}, NTuple{4, Int}}
+    @testintersect(A, Tuple{Tuple{Int, Vararg{Any}}, NTuple{4, Int}}, C)
+    @testintersect(A, Tuple{Tuple{Int, Vararg{Any, N}} where {N}, NTuple{4, Int}}, C)
+    @testintersect(A, Tuple{Tuple{Int, Vararg{Any, N}}, NTuple{4, Int}} where {N}, C)
+
+    Bs = (Tuple{Tuple{Int, Vararg{Any}}, Tuple{Int, Int, Vararg{Any}}},
+          Tuple{Tuple{Int, Vararg{Any,N1}}, Tuple{Int, Int, Vararg{Any,N2}}} where {N1,N2},
+          Tuple{Tuple{Int, Vararg{Any,N}} where {N}, Tuple{Int, Int, Vararg{Any,N}} where {N}})
+    Cerr = Tuple{Tuple{Int, Vararg{Int, N}}, Tuple{Int, Int, Vararg{Int, N}}} where {N}
+    for B in Bs
+        C = typeintersect(A, B)
+        @test C == typeintersect(B, A) != Union{}
+        @test C != Cerr
+        # TODO: The idea result is Tuple{Tuple{Int, Int, Vararg{Int, N}}, Tuple{Int, Int, Vararg{Int, N}}} where {N}
+        @test_broken C != Tuple{Tuple{Int, Vararg{Int}}, Tuple{Int, Int, Vararg{Int}}}
+    end
+end
+
+# Example from pr#39098
+@testintersect(NTuple, Tuple{Any,Vararg}, Tuple{T, Vararg{T}} where {T})