InMemory: Index based bindings for projection

Resolves #17620

src/EFCore.Cosmos/Query/Internal/CosmosProjectionBindingExpressionVisitor.cs

@@ -42,8 +42,10 @@ private readonly IDictionary<ProjectionMember, Expression> _projectionMapping
 
         private readonly Stack<ProjectionMember> _projectionMembers = new();
 
+#pragma warning disable CS0618 // Type or member is obsolete
         private readonly IDictionary<ParameterExpression, CollectionShaperExpression> _collectionShaperMapping
             = new Dictionary<ParameterExpression, CollectionShaperExpression>();
+#pragma warning restore CS0618 // Type or member is obsolete
 
         private readonly Stack<INavigation> _includedNavigations
             = new();
@@ -366,7 +368,9 @@ protected override Expression VisitMember(MemberExpression memberExpression)
                             Expression.Convert(objectArrayProjectionExpression.InnerProjection, typeof(object)), typeof(ValueBuffer)),
                         nullable: true);
 
+#pragma warning disable CS0618 // Type or member is obsolete
                     return new CollectionShaperExpression(
+#pragma warning restore CS0618 // Type or member is obsolete
                         objectArrayProjectionExpression,
                         innerShaperExpression,
                         navigation,
@@ -570,7 +574,9 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
                                 Expression.Convert(objectArrayProjectionExpression.InnerProjection, typeof(object)), typeof(ValueBuffer)),
                             nullable: true);
 
+#pragma warning disable CS0618 // Type or member is obsolete
                         return new CollectionShaperExpression(
+#pragma warning restore CS0618 // Type or member is obsolete
                             objectArrayProjectionExpression,
                             innerShaperExpression,
                             navigation,
@@ -599,7 +605,9 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
 
                         case nameof(Queryable.Select)
                             when genericMethod == QueryableMethods.Select:
+#pragma warning disable CS0618 // Type or member is obsolete
                             if (!(visitedSource is CollectionShaperExpression shaper))
+#pragma warning restore CS0618 // Type or member is obsolete
                             {
                                 return null;
                             }

src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.CosmosProjectionBindingRemovingExpressionVisitorBase.cs

@@ -248,7 +248,9 @@ protected override Expression VisitExtension(Expression extensionExpression)
                             projectionBindingExpression.Type, (projection.Expression as SqlExpression)?.TypeMapping);
                     }
 
+#pragma warning disable CS0618 // Type or member is obsolete
                     case CollectionShaperExpression collectionShaperExpression:
+#pragma warning restore CS0618 // Type or member is obsolete
                     {
                         ObjectArrayProjectionExpression objectArrayProjection;
                         switch (collectionShaperExpression.Projection)

src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.JObjectInjectingExpressionVisitor.cs

@@ -53,7 +53,9 @@ protected override Expression VisitExtension(Expression extensionExpression)
                             expressions);
                     }
 
+#pragma warning disable CS0618 // Type or member is obsolete
                     case CollectionShaperExpression collectionShaperExpression:
+#pragma warning restore CS0618 // Type or member is obsolete
                     {
                         _currentEntityIndex++;
 

src/EFCore.InMemory/Properties/InMemoryStrings.resx

@@ -120,6 +120,9 @@
   <data name="DefaultIfEmptyAppliedAfterProjection" xml:space="preserve">
     <value>Cannot apply 'DefaultIfEmpty' after a client-evaluated projection. Consider applying 'DefaultIfEmpty' before last 'Select' or use 'AsEnumerable' before 'DefaultIfEmpty' to apply it on client-side.</value>
   </data>
+  <data name="DistinctOnSubqueryNotSupported" xml:space="preserve">
+    <value>Using 'Distinct' operation on a projection containing a subquery is not supported.</value>
+  </data>
   <data name="InvalidDerivedTypeInEntityProjection" xml:space="preserve">
     <value>The specified entity type '{derivedType}' is not derived from '{entityType}'.</value>
   </data>
@@ -140,6 +143,9 @@
   <data name="NullabilityErrorExceptionSensitive" xml:space="preserve">
     <value>Required properties '{requiredProperties}' are missing for the instance of entity type '{entityType}' with the key value '{keyValue}'.</value>
   </data>
+  <data name="SetOperationsNotAllowedAfterClientEvaluation" xml:space="preserve">
+    <value>Unable to translate set operation after client projection has been applied. Consider moving the set operation before the last 'Select' call.</value>
+  </data>
   <data name="UnableToBindMemberToEntityProjection" xml:space="preserve">
     <value>Unable to bind '{memberType}' '{member}' to entity projection of '{entityType}'.</value>
   </data>

src/EFCore.InMemory/Query/Internal/CollectionResultShaperExpression.cs

@@ -0,0 +1,128 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Query;
+using Microsoft.EntityFrameworkCore.Utilities;
+
+namespace Microsoft.EntityFrameworkCore.InMemory.Query.Internal
+{
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public class CollectionResultShaperExpression : Expression, IPrintableExpression
+    {
+        /// <summary>
+        ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+        ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+        ///     any release. You should only use it directly in your code with extreme caution and knowing that
+        ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+        /// </summary>
+        public CollectionResultShaperExpression(
+            Expression projection,
+            Expression innerShaper,
+            INavigationBase? navigation,
+            Type elementType)
+        {
+            Check.NotNull(projection, nameof(projection));
+            Check.NotNull(innerShaper, nameof(innerShaper));
+
+            Projection = projection;
+            InnerShaper = innerShaper;
+            Navigation = navigation;
+            ElementType = elementType;
+        }
+
+        /// <summary>
+        ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+        ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+        ///     any release. You should only use it directly in your code with extreme caution and knowing that
+        ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+        /// </summary>
+        public virtual Expression Projection { get; }
+
+        /// <summary>
+        ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+        ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+        ///     any release. You should only use it directly in your code with extreme caution and knowing that
+        ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+        /// </summary>
+        public virtual Expression InnerShaper { get; }
+
+        /// <summary>
+        ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+        ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+        ///     any release. You should only use it directly in your code with extreme caution and knowing that
+        ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+        /// </summary>
+        public virtual INavigationBase? Navigation { get; }
+
+        /// <summary>
+        ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+        ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+        ///     any release. You should only use it directly in your code with extreme caution and knowing that
+        ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+        /// </summary>
+        public virtual Type ElementType { get; }
+
+        /// <inheritdoc />
+        public sealed override ExpressionType NodeType
+            => ExpressionType.Extension;
+
+        /// <inheritdoc />
+        public override Type Type
+            => Navigation?.ClrType ?? typeof(List<>).MakeGenericType(ElementType);
+
+        /// <inheritdoc />
+        protected override Expression VisitChildren(ExpressionVisitor visitor)
+        {
+            Check.NotNull(visitor, nameof(visitor));
+
+            var projection = visitor.Visit(Projection);
+            var innerShaper = visitor.Visit(InnerShaper);
+
+            return Update(projection, innerShaper);
+        }
+
+        /// <summary>
+        ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+        ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+        ///     any release. You should only use it directly in your code with extreme caution and knowing that
+        ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+        /// </summary>
+        public virtual CollectionResultShaperExpression Update(
+            Expression projection,
+            Expression innerShaper)
+        {
+            Check.NotNull(projection, nameof(projection));
+            Check.NotNull(innerShaper, nameof(innerShaper));
+
+            return projection != Projection || innerShaper != InnerShaper
+                ? new CollectionResultShaperExpression(projection, innerShaper, Navigation, ElementType)
+                : this;
+        }
+
+        /// <inheritdoc />
+        void IPrintableExpression.Print(ExpressionPrinter expressionPrinter)
+        {
+            Check.NotNull(expressionPrinter, nameof(expressionPrinter));
+
+            expressionPrinter.AppendLine("CollectionResultShaperExpression:");
+            using (expressionPrinter.Indent())
+            {
+                expressionPrinter.Append("(");
+                expressionPrinter.Visit(Projection);
+                expressionPrinter.Append(", ");
+                expressionPrinter.Visit(InnerShaper);
+                expressionPrinter.AppendLine($", {Navigation?.Name}, {ElementType.ShortDisplayName()})");
+            }
+        }
+    }
+}

src/EFCore.InMemory/Query/Internal/InMemoryExpressionTranslatingExpressionVisitor.cs

@@ -278,12 +278,7 @@ protected override Expression VisitExtension(Expression extensionExpression)
 
                 case ProjectionBindingExpression projectionBindingExpression
                     when projectionBindingExpression.ProjectionMember != null:
-                    return ((InMemoryQueryExpression)projectionBindingExpression.QueryExpression)
-                        .GetMappedProjection(projectionBindingExpression.ProjectionMember);
-
-                //case ProjectionBindingExpression projectionBindingExpression
-                //    when projectionBindingExpression.Index is int index:
-                //    return ((InMemoryQueryExpression)projectionBindingExpression.QueryExpression).Projection[index];
+                    return ((InMemoryQueryExpression)projectionBindingExpression.QueryExpression).GetProjection(projectionBindingExpression);
 
                 case InMemoryGroupByShaperExpression inMemoryGroupByShaperExpression:
                     return new GroupingElementExpression(
@@ -747,9 +742,8 @@ static Expression RemapLambda(GroupingElementExpression groupingElement, LambdaE
                 }
 
                 return ProcessSingleResultScalar(
-                    subquery.ServerQueryExpression,
-                    subquery.GetMappedProjection(projectionBindingExpression.ProjectionMember),
-                    subquery.CurrentParameter,
+                    subquery,
+                    subquery.GetProjection(projectionBindingExpression),
                     methodCallExpression.Type);
             }
 
@@ -1209,60 +1203,54 @@ protected override Expression VisitUnary(UnaryExpression unaryExpression)
 
                 Expression readValueExpression;
                 var projectionBindingExpression = (ProjectionBindingExpression)entityShaper.ValueBufferExpression;
-                if (projectionBindingExpression.ProjectionMember != null)
-                {
-                    var entityProjectionExpression = (EntityProjectionExpression)inMemoryQueryExpression.GetMappedProjection(
-                        projectionBindingExpression.ProjectionMember);
-                    readValueExpression = entityProjectionExpression.BindProperty(property);
-                }
-                else
-                {
-                    // This has to be index map since entities cannot map to just integer index
-                    var index = projectionBindingExpression.IndexMap![property];
-                    readValueExpression = inMemoryQueryExpression.Projection[index];
-                }
+                var entityProjectionExpression = (EntityProjectionExpression)inMemoryQueryExpression.GetProjection(
+                    projectionBindingExpression);
+                readValueExpression = entityProjectionExpression.BindProperty(property);
 
                 return ProcessSingleResultScalar(
-                    inMemoryQueryExpression.ServerQueryExpression,
+                    inMemoryQueryExpression,
                     readValueExpression,
-                    inMemoryQueryExpression.CurrentParameter,
                     type);
             }
 
             return null;
         }
 
         private static Expression ProcessSingleResultScalar(
-            Expression serverQuery,
+            InMemoryQueryExpression inMemoryQueryExpression,
             Expression readValueExpression,
-            Expression valueBufferParameter,
             Type type)
         {
-            var singleResult = ((LambdaExpression)((NewExpression)serverQuery).Arguments[0]).Body;
-            if (readValueExpression is UnaryExpression unaryExpression
+            if (inMemoryQueryExpression.ServerQueryExpression is not NewExpression)
+            {
+                // The terminating operator is not applied
+                // It is of FirstOrDefault kind
+                // So we change to single column projection and then apply it.
+                inMemoryQueryExpression.ReplaceProjection(new Dictionary<ProjectionMember, Expression>
+                {
+                    { new ProjectionMember(), readValueExpression }
+                });
+                inMemoryQueryExpression.ApplyProjection();
+            }
+
+            var serverQuery = inMemoryQueryExpression.ServerQueryExpression;
+            serverQuery = ((LambdaExpression)((NewExpression)serverQuery).Arguments[0]).Body;
+            if (serverQuery is UnaryExpression unaryExpression
                 && unaryExpression.NodeType == ExpressionType.Convert
                 && unaryExpression.Type == typeof(object))
             {
-                readValueExpression = unaryExpression.Operand;
+                serverQuery = unaryExpression.Operand;
             }
 
             var valueBufferVariable = Expression.Variable(typeof(ValueBuffer));
-            var replacedReadExpression = ReplacingExpressionVisitor.Replace(
-                valueBufferParameter,
-                valueBufferVariable,
-                readValueExpression);
-
-            replacedReadExpression = replacedReadExpression.Type == type
-                ? replacedReadExpression
-                : Expression.Convert(replacedReadExpression, type);
-
+            var readExpression = valueBufferVariable.CreateValueBufferReadValueExpression(type, index: 0, property: null);
             return Expression.Block(
                 variables: new[] { valueBufferVariable },
-                Expression.Assign(valueBufferVariable, singleResult),
+                Expression.Assign(valueBufferVariable, serverQuery),
                 Expression.Condition(
                     Expression.MakeMemberAccess(valueBufferVariable, _valueBufferIsEmpty),
                     Expression.Default(type),
-                    replacedReadExpression));
+                    readExpression));
         }
 
         [UsedImplicitly]