Avoid invalid typevar setting in bound_var_below.

The added check failed to cover all cases.
It should be much safer to always return a  `Vararg` with unknown length.
We only need to make sure that `typeintersect(Val{<:Tuple{Any,Vararg{Any}}}, Val{Tuple{Vararg{Any,N}} where {N}})` doesn't gives `Val{Tuple{Any,Vararg{Any}}`.

Author: N5N3

src/subtype.c

@@ -77,9 +77,7 @@ typedef struct jl_varbinding_t {
     // intvalued: must be integer-valued; i.e. occurs as N in Vararg{_,N}
     // 0: No restriction
     // 1: must be unbounded/ or fixed to a `Int`/typevar
-    // 2: This var has been commandeered once.
-    // 3: This var has been commandeered more than once.
-    // 4: This var has been commandeered more than once, and it has no fixed ub/lb.
+    // 2: we have some imprecise vararg length intersection that can be improved if this var is const valued.
     int8_t intvalued;
     int8_t limited;
     int16_t depth0;         // # of invariant constructors nested around the UnionAll type for this var
@@ -2209,16 +2207,10 @@ static jl_value_t *bound_var_below(jl_tvar_t *tv, jl_varbinding_t *bb, jl_stenv_
             return bb->lb;
         return jl_box_long(jl_unbox_long(bb->lb) - bb->offset);
     }
-    if (bb->intvalued == 1 && bb->offset > 0)
-        // We really should return `tv` here, but some subtype tests need it.
-        // As a temparay fix, increase `intvalued` to 2.
+    if (bb->offset > 0) {
         bb->intvalued = 2;
-    else if (bb->intvalued == 2)
-        // If bb has been commandeered, we should always increase `intvalued`.
-        bb->intvalued = 3;
-    else if (bb->intvalued == 4 && bb->offset > 0)
-        // bb has no fixed bounds, return `NULL` if `tv` is not correct.
         return NULL;
+    }
     return (jl_value_t*)tv;
 }
 
@@ -2307,6 +2299,26 @@ static int check_unsat_bound(jl_value_t *t, jl_tvar_t *v, jl_stenv_t *e) JL_NOTS
     return 0;
 }
 
+static int has_null_vararg_length(jl_value_t *a) {
+    int is_unionall = 0;
+    while (jl_is_unionall(a)) {
+        a = ((jl_unionall_t *)a)->body;
+        is_unionall = 1;
+    }
+    if (jl_is_datatype(a) && jl_is_tuple_type((jl_datatype_t *)a)) {
+        size_t lx = jl_nparams((jl_datatype_t *)a);
+        if (lx > 0) {
+            jl_value_t *la = jl_tparam((jl_datatype_t *)a, lx-1);
+            if (jl_is_vararg(la)) {
+                jl_value_t *len = jl_unwrap_vararg_num((jl_vararg_t *)la);
+                if (!len) return 1;
+                if (!jl_is_long(len)) return is_unionall;
+            }
+        }
+    }
+    return 0;    
+}
+
 static jl_value_t *intersect_var(jl_tvar_t *b, jl_value_t *a, jl_stenv_t *e, int8_t R, int param)
 {
     jl_varbinding_t *bb = lookup(e, b);
@@ -2353,6 +2365,10 @@ static jl_value_t *intersect_var(jl_tvar_t *b, jl_value_t *a, jl_stenv_t *e, int
                 ub = (jl_value_t*)b;
             }
         }
+        if (has_null_vararg_length(ub) && !has_null_vararg_length(a)) {
+            bb->ub = ub;
+            ub = (jl_value_t*)b;
+        }
         if (ub != (jl_value_t*)b) {
             if (jl_has_free_typevars(ub)) {
                 if (check_unsat_bound(ub, b, e)) {
@@ -2470,12 +2486,9 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind
         }
     }
 
-    // This typevar has been commandeered by the other side once. Which is allowed if it's unbounded.
-    if (vb->intvalued == 2 && vb->lb == jl_bottom_type && vb->ub == (jl_value_t*)jl_any_type)
+    // vb is still unbounded.
+    if (vb->intvalued == 2 && !(varval && jl_is_long(varval)))
         vb->intvalued = 1;
-    // This typevar has been commandeered more than once. Set it to 4 if its not fixed.
-    if (vb->intvalued == 3 && !(varval && jl_is_long(varval)))
-        vb->intvalued = 4;
 
     // TODO: this can prevent us from matching typevar identities later
     if (!varval && (vb->lb != vb->var->lb || vb->ub != vb->var->ub))
@@ -3080,10 +3093,10 @@ static jl_value_t *intersect(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int pa
                 }
                 JL_GC_POP();
                 // Here we always return the shorter `Vararg`'s length.
-                if ((xx && xx->offset < 0) || (yy && yy->offset > 0))
+                if ((xx && xx->offset < 0) || (yy && yy->offset > 0)) {
+                    if (yy) yy->intvalued = 2;
                     return x;
-                if ((xx && xx->offset > 0) || (yy && yy->offset < 0))
-                    return y;
+                }
                 return y;
             }
             record_var_occurrence(xx, e, param);

test/subtype.jl

@@ -1043,7 +1043,12 @@ function test_intersection()
                    Type{Tuple{Int,T}} where T<:Integer)
     @testintersect(Type{<:Tuple{Any,Vararg{Any}}},
                    Type{Tuple{Vararg{Int,N}}} where N,
-                   Type{Tuple{Int,Vararg{Int,N}}} where N)
+                   !Union{})
+
+    @test typeintersect(Type{<:Tuple{Any,Vararg{Any}}}, Type{Tuple{Vararg{Int,N}}} where N) != Type{Tuple{Int,Vararg{Int}}}
+    @test_broken typeintersect(Type{<:Tuple{Any,Vararg{Any}}}, Type{Tuple{Vararg{Int,N}}} where N) == Type{Tuple{Int,Vararg{Int,N}}} where N
+    @test_broken typeintersect(Type{<:Tuple{Any,Vararg{Any}}}, Type{Tuple{Vararg{Int,N}}} where N) != Type{<:Tuple{Int,Vararg{Int}}}
+
     @testintersect(Type{<:Array},
                    Type{AbstractArray{T}} where T,
                    Bottom)
@@ -2109,5 +2114,24 @@ let A = Tuple{NTuple{N, Int}, NTuple{N, Int}} where N,
     end
 end
 
+struct __Vals{A,B} end
+let A = __Vals{NTuple{N, Int}, NTuple{N, Int}} where N,
+    C = __Vals{NTuple{4, Int}, NTuple{4, Int}}
+    @testintersect(A, __Vals{<:Tuple{Int, Vararg{Any}}, NTuple{4, Int}}, C)
+    @testintersect(A, __Vals{<:Tuple{Int, Vararg{Any, N}} where {N}, NTuple{4, Int}}, C)
+    @testintersect(A, __Vals{<:Tuple{Int, Vararg{Any, N}}, NTuple{4, Int}} where {N}, C)
+
+    Bs = (__Vals{<:Tuple{Int, Vararg{Int}}, <:Tuple{Int, Int, Vararg{Int}}},
+          __Vals{Tuple{Int, Vararg{Int,N1}}, Tuple{Int, Int, Vararg{Int,N2}}} where {N1,N2},
+          __Vals{<:Tuple{Int, Vararg{Int,N}} where {N}, <:Tuple{Int, Int, Vararg{Int,N}} where {N}})
+    Cerr = __Vals{Tuple{Int, Vararg{Int, N}}, Tuple{Int, Int, Vararg{Int, N}}} where {N}
+    for B in Bs
+        C = typeintersect(A, B)
+        @test C == typeintersect(B, A) != Union{}
+        @test C != Cerr
+        @test_broken C != B
+    end
+end
+
 # Example from pr#39098
 @testintersect(NTuple, Tuple{Any,Vararg}, Tuple{T, Vararg{T}} where {T})