[release/10.0] Fix issues with nulls in primitive collections

src/EFCore.Relational/Query/SqlNullabilityProcessor.cs

@@ -26,6 +26,9 @@ public class SqlNullabilityProcessor : ExpressionVisitor
     private static readonly bool UseOldBehavior37152 =
         AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue37152", out var enabled) && enabled;
 
+    private static readonly bool UseOldBehavior37537 =
+        AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue37537", out var enabled) && enabled;
+
     private readonly List<ColumnExpression> _nonNullableColumns;
     private readonly List<ColumnExpression> _nullValueColumns;
     private readonly ISqlExpressionFactory _sqlExpressionFactory;
@@ -1906,13 +1909,21 @@ protected virtual bool TryMakeNonNullable(
             var parameters = ParametersDecorator.GetAndDisableCaching();
 
             IList values;
+            Type elementClrType;
             if (UseOldBehavior37204)
             {
                 if (parameters[collectionParameter.Name] is not IList list)
                 {
                     throw new UnreachableException($"Parameter '{collectionParameter.Name}' is not an IList.");
                 }
                 values = list;
+                elementClrType = UseOldBehavior37537
+                    ? values.GetType().GetSequenceType()
+                    // We found the first null value - we need to start copying values to a new list which will be used for the rewritten parameter.
+                    // The type of the new list must match that of the original enumerable parameter, as there may be value converters involved which
+                    // rely on the precise element type (see #37605). We therefore get the type of the element from the original list if it implements
+                    // IEnumerable<T>, or default to object.
+                    : list.GetType().TryGetElementType(typeof(IEnumerable<>)) ?? typeof(object);
             }
             else
             {
@@ -1921,6 +1932,13 @@ protected virtual bool TryMakeNonNullable(
                     throw new UnreachableException($"Parameter '{collectionParameter.Name}' is not an IEnumerable.");
                 }
                 values = enumerable.Cast<object?>().ToList();
+                elementClrType = UseOldBehavior37537
+                    ? values.GetType().GetSequenceType()
+                    // We found the first null value - we need to start copying values to a new list which will be used for the rewritten parameter.
+                    // The type of the new list must match that of the original enumerable parameter, as there may be value converters involved which
+                    // rely on the precise element type (see #37605). We therefore get the type of the element from the original list if it implements
+                    // IEnumerable<T>, or default to object.
+                    : enumerable.GetType().TryGetElementType(typeof(IEnumerable<>)) ?? typeof(object);
             }
 
             IList? processedValues = null;
@@ -1933,7 +1951,6 @@ protected virtual bool TryMakeNonNullable(
                 {
                     if (processedValues is null)
                     {
-                        var elementClrType = values.GetType().GetSequenceType();
                         processedValues = (IList)Activator.CreateInstance(typeof(List<>).MakeGenericType(elementClrType), values.Count)!;
                         for (var j = 0; j < i; j++)
                         {

src/EFCore.SqlServer/Query/Internal/SqlServerSqlNullabilityProcessor.cs

@@ -27,6 +27,9 @@ public class SqlServerSqlNullabilityProcessor : SqlNullabilityProcessor
     private static readonly bool UseOldBehavior37336 =
         AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue37336", out var enabled) && enabled;
 
+    private static readonly bool UseOldBehavior37537 =
+        AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue37537", out var enabled) && enabled;
+
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
     ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
@@ -290,7 +293,9 @@ protected override SqlExpression VisitIn(InExpression inExpression, bool allowOp
                                             new ColumnExpression(
                                                 columnName,
                                                 openJson.Alias,
-                                                valuesParameter.Type.GetSequenceType(),
+                                                UseOldBehavior37537
+                                                    ? valuesParameter.Type.GetSequenceType()
+                                                    : valuesParameter.Type.GetSequenceType().UnwrapNullableType(),
                                                 elementTypeMapping,
                                                 containsNulls!.Value),
                                             columnName)

test/EFCore.Cosmos.FunctionalTests/Query/PrimitiveCollectionsQueryCosmosTest.cs

@@ -647,6 +647,20 @@ WHERE NOT(ARRAY_CONTAINS(@nullableInts, c["NullableInt"]))
 """);
     }
 
+    public override async Task Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter()
+    {
+        await base.Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter();
+
+        AssertSql(
+            """
+@nullableInts='[null,999]'
+
+SELECT VALUE c
+FROM root c
+WHERE ARRAY_CONTAINS(@nullableInts, c["NullableInt"])
+""");
+    }
+
     public override async Task Parameter_collection_of_strings_Contains_string()
     {
         await base.Parameter_collection_of_strings_Contains_string();

test/EFCore.Specification.Tests/Query/PrimitiveCollectionsQueryTestBase.cs

@@ -308,6 +308,16 @@ public virtual async Task Parameter_collection_of_nullable_ints_Contains_nullabl
         await AssertQuery(ss => ss.Set<PrimitiveCollectionsEntity>().Where(c => !nullableInts.Contains(c.NullableInt)));
     }
 
+    [ConditionalFact] // #37605
+    public virtual async Task Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter()
+    {
+        var nullableInts = new int?[] { null, 999 };
+
+        await AssertQuery(
+            ss => ss.Set<PrimitiveCollectionsEntity>().Where(c => EF.Parameter(nullableInts).Contains(c.NullableInt)),
+            ss => ss.Set<PrimitiveCollectionsEntity>().Where(c => nullableInts.Contains(c.NullableInt)));
+    }
+
     [ConditionalFact]
     public virtual async Task Parameter_collection_of_structs_Contains_struct()
     {

test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQueryOldSqlServerTest.cs

@@ -623,6 +623,14 @@ WHERE [p].[NullableInt] IS NOT NULL AND [p].[NullableInt] <> @nullableInts1
 """);
     }
 
+    public override async Task Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter()
+    {
+        // EF.Parameter() on primitive collection (OPENJSON on SQL Server) not supported on old versions of SQL Server.
+        await Assert.ThrowsAsync<InvalidOperationException>(base.Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter);
+
+        AssertSql();
+    }
+
     public override async Task Parameter_collection_of_strings_Contains_string()
     {
         await base.Parameter_collection_of_strings_Contains_string();

test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServer160Test.cs

@@ -631,6 +631,23 @@ WHERE [p].[NullableInt] IS NOT NULL AND [p].[NullableInt] <> @nullableInts1
 """);
     }
 
+    public override async Task Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter()
+    {
+        await base.Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter();
+
+        AssertSql(
+            """
+@nullableInts_without_nulls='[999]' (Size = 4000)
+
+SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[NullableString], [p].[NullableStrings], [p].[NullableWrappedId], [p].[NullableWrappedIdWithNullableComparer], [p].[String], [p].[Strings], [p].[WrappedId]
+FROM [PrimitiveCollectionsEntity] AS [p]
+WHERE [p].[NullableInt] IN (
+    SELECT [n].[value]
+    FROM OPENJSON(@nullableInts_without_nulls) AS [n]
+) OR [p].[NullableInt] IS NULL
+""");
+    }
+
     public override async Task Parameter_collection_of_strings_Contains_string()
     {
         await base.Parameter_collection_of_strings_Contains_string();

test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServerJsonTypeTest.cs

@@ -800,6 +800,23 @@ WHERE [p].[NullableInt] IS NOT NULL AND [p].[NullableInt] <> @nullableInts1
 """);
     }
 
+    public override async Task Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter()
+    {
+        await base.Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter();
+
+        AssertSql(
+            """
+@nullableInts_without_nulls='[999]' (Size = 5)
+
+SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[NullableString], [p].[NullableStrings], [p].[NullableWrappedId], [p].[NullableWrappedIdWithNullableComparer], [p].[String], [p].[Strings], [p].[WrappedId]
+FROM [PrimitiveCollectionsEntity] AS [p]
+WHERE [p].[NullableInt] IN (
+    SELECT [n].[value]
+    FROM OPENJSON(@nullableInts_without_nulls) AS [n]
+) OR [p].[NullableInt] IS NULL
+""");
+    }
+
     public override async Task Parameter_collection_of_structs_Contains_struct()
     {
         await base.Parameter_collection_of_structs_Contains_struct();

test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServerTest.cs

@@ -654,6 +654,23 @@ WHERE [p].[NullableInt] IS NOT NULL AND [p].[NullableInt] <> @nullableInts1
 """);
     }
 
+    public override async Task Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter()
+    {
+        await base.Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter();
+
+        AssertSql(
+            """
+@nullableInts_without_nulls='[999]' (Size = 4000)
+
+SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[NullableString], [p].[NullableStrings], [p].[NullableWrappedId], [p].[NullableWrappedIdWithNullableComparer], [p].[String], [p].[Strings], [p].[WrappedId]
+FROM [PrimitiveCollectionsEntity] AS [p]
+WHERE [p].[NullableInt] IN (
+    SELECT [n].[value]
+    FROM OPENJSON(@nullableInts_without_nulls) AS [n]
+) OR [p].[NullableInt] IS NULL
+""");
+    }
+
     public override async Task Parameter_collection_of_strings_Contains_string()
     {
         await base.Parameter_collection_of_strings_Contains_string();

test/EFCore.Sqlite.FunctionalTests/Query/PrimitiveCollectionsQuerySqliteTest.cs

@@ -642,6 +642,23 @@ public override async Task Parameter_collection_of_nullable_ints_Contains_nullab
 """);
     }
 
+    public override async Task Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter()
+    {
+        await base.Parameter_collection_of_nullable_ints_Contains_nullable_int_with_EF_Parameter();
+
+        AssertSql(
+            """
+@nullableInts_without_nulls='[999]' (Size = 5)
+
+SELECT "p"."Id", "p"."Bool", "p"."Bools", "p"."DateTime", "p"."DateTimes", "p"."Enum", "p"."Enums", "p"."Int", "p"."Ints", "p"."NullableInt", "p"."NullableInts", "p"."NullableString", "p"."NullableStrings", "p"."NullableWrappedId", "p"."NullableWrappedIdWithNullableComparer", "p"."String", "p"."Strings", "p"."WrappedId"
+FROM "PrimitiveCollectionsEntity" AS "p"
+WHERE "p"."NullableInt" IN (
+    SELECT "n"."value"
+    FROM json_each(@nullableInts_without_nulls) AS "n"
+) OR "p"."NullableInt" IS NULL
+""");
+    }
+
     public override async Task Parameter_collection_of_strings_Contains_string()
     {
         await base.Parameter_collection_of_strings_Contains_string();