Merge pull request matteobortolazzo#40 from matteobortolazzo/dev

Release 1.1.0

CHANGELOG.md

@@ -1,4 +1,9 @@
-# 1.0.2 (2019-05-02)
+# 1.1.0 (2019-05-05)
+
+## Features
+* **_find:** IQueryable methods that are not supported by CouchDB are evaluated in-memory using the IEnumerable counterpart, if possible.
+
+# 1.0.2 (2019-05-02)
 
 ## Bug Fixes
 * **_find:** Boolean member expressions converted to binary expressions in Where (Fix [#PR36](https://github.com/matteobortolazzo/couchdb-net/pull/36)).

LATEST_CHANGE.md

@@ -1,2 +1,2 @@
-## Bug Fixes
-* **_find:** Boolean member expression converted to binary expression in Where (Fix [#PR36](https://github.com/matteobortolazzo/couchdb-net/pull/36)).
+## Features
+* **_find:** IQueryable methods that are not supported by CouchDB are evaluated in-memory using the IEnumerable counterpart, if possible.

README.md

@@ -179,6 +179,31 @@ If the Where method is not called in the expression, it will at an empty selecto
 | execution_stats | IncludeExecutionStats()                              |
 | conflicts       | IncludeConflicts()                                   |
 
+### Composite methods
+
+Some methods that are not directly supported by CouchDB are converted to a composition of supported ones.
+
+| Input                             | Output                                |
+|:----------------------------------|:--------------------------------------|
+| Min(r => r.Age)                   | OrderBy(r => r.Age).Take(1)           |
+| Max(r => r.Age)                   | OrderByDescending(r => r.Age).Take(1) |
+| Single()                          | Take(1)								|
+| SingleOrDefault()                 | Take(1)								|
+| Single(r => r.Age == 19)			| Where(r => r.Age == 19).Take(1)       |
+| SingleOrDefault(r => r.Age == 19) | Where(r => r.Age == 19).Take(1)       |
+
+**WARN**: Do not call a method twice, for example: `Where(func).Single(func)` won't work.
+
+**WARN**: Since Max and Min use **sort**, an *index* must be created.
+
+
+### All other IQueryables
+
+IQueryable methods that are not natively supported by CouchDB are evaluated in-memory using the IEnumerable counterpart, if possible.
+
+For example: `All` `Any` `Avg` `Count` `DefaultIfEmpty` `ElementAt` `ElementAtOrDefault` `GroupBy` `Last` `Reverse` `SelectMany` `Sum`
+
+
 ## Client operations
 
 ```csharp

src/CouchDB.Driver/CouchClientAuthentication.cs

@@ -4,6 +4,7 @@
 using Flurl.Http;
 using Nito.AsyncEx;
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
 using System.Text.RegularExpressions;
@@ -25,7 +26,7 @@ protected virtual void OnBeforeCall(HttpCall httpCall)
                 case AuthenticationType.None:
                     break;
                 case AuthenticationType.Basic:
-                    httpCall.FlurlRequest.WithBasicAuth(_settings.Username, _settings.Password);
+                    httpCall.FlurlRequest = httpCall.FlurlRequest.WithBasicAuth(_settings.Username, _settings.Password);
                     break;
                 case AuthenticationType.Cookie:
                     var isTokenExpired =
@@ -35,7 +36,7 @@ protected virtual void OnBeforeCall(HttpCall httpCall)
                     {
                         AsyncContext.Run(() => LoginAsync());
                     }
-                    httpCall.FlurlRequest.EnableCookies().WithCookie("AuthSession", _cookieToken);
+                    httpCall.FlurlRequest = httpCall.FlurlRequest.EnableCookies().WithCookie("AuthSession", _cookieToken);
                     break;
                 default:
                     throw new NotSupportedException($"Authentication of type {_settings.AuthenticationType} is not supported.");
@@ -55,7 +56,7 @@ private async Task LoginAsync()
 
             _cookieCreationDate = DateTime.Now;
 
-            if (response.Headers.TryGetValues("Set-Cookie", out var values))
+            if (response.Headers.TryGetValues("Set-Cookie", out IEnumerable<string> values))
             {
                 var dirtyToken = values.First();
                 var regex = new Regex(@"^AuthSession=(.+); Version=1; .*Path=\/; HttpOnly$");

src/CouchDB.Driver/CouchDB.Driver.csproj

@@ -9,14 +9,13 @@
     <PackageProjectUrl>https://github.com/matteobortolazzo/couchdb-net</PackageProjectUrl>
     <RepositoryUrl>https://github.com/matteobortolazzo/couchdb-ne</RepositoryUrl>
     <PackageTags>couchdb,driver,nosql,netstandard,pouchdb,xamarin</PackageTags>
-    <PackageReleaseNotes>Complete rewrite.
-IQueryable support.
-All selectors support.</PackageReleaseNotes>
+    <PackageReleaseNotes></PackageReleaseNotes>
     <PackageIconUrl>http://couchdb.apache.org/image/couch@2x.png</PackageIconUrl>
     <ApplicationIcon />
     <OutputType>Library</OutputType>
     <StartupObject />
     <NeutralLanguage>en</NeutralLanguage>
+    <Version>1.1.0</Version>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">

src/CouchDB.Driver/CouchQueryProvider.cs

@@ -1,5 +1,5 @@
 using CouchDB.Driver.DTOs;
-using CouchDB.Driver.ExpressionVisitors;
+using CouchDB.Driver.CompositeExpressionsEvaluator;
 using CouchDB.Driver.Helpers;
 using CouchDB.Driver.Settings;
 using CouchDB.Driver.Types;
@@ -29,49 +29,51 @@ public CouchQueryProvider(IFlurlClient flurlClient, CouchSettings settings, stri
 
         public override string GetQueryText(Expression expression)
         {
-            return Translate(expression);
+            return Translate(ref expression);
         }
 
-        public override object Execute(Expression e, bool completeResponse)
+        public override object Execute(Expression expression, bool completeResponse)
         {
-            MethodInfo _filterMethodInfo = null;
-            Expression[] _filteringExpressions = Array.Empty<Expression>();
-            if (e is MethodCallExpression m)
+            // Remove from the expressions tree all IQueryable methods not supported by CouchDB and put them into the list
+            var unsupportedMethodCallExpressions = new List<MethodCallExpression>();
+            expression = RemoveUnsupportedMethodExpressions(expression, out var hasUnsupportedMethods, unsupportedMethodCallExpressions);
+
+            var body = Translate(ref expression);
+            Type elementType = TypeSystem.GetElementType(expression.Type);
+
+            // Create generic GetCouchList method and invoke it, sending the request to CouchDB
+            MethodInfo method = typeof(CouchQueryProvider).GetMethod(nameof(CouchQueryProvider.GetCouchList));
+            MethodInfo generic = method.MakeGenericMethod(elementType);
+            var result = generic.Invoke(this, new[] { body });
+
+            // If no unsupported methods, return the result
+            if (!hasUnsupportedMethods)
             {
-                if (
-                    m.Method.Name == "First" ||
-                    m.Method.Name == "FirstOrDefault" ||
-                    m.Method.Name == "Last" ||
-                    m.Method.Name == "LastOrDefault" ||
-                    m.Method.Name == "Single" ||
-                    m.Method.Name == "SingleOrDefault")
-                {
-                    _filterMethodInfo = m.Method;
-                    _filteringExpressions = m.Arguments.Skip(1).ToArray();
-                    e = m.Arguments[0];
-                }
+                return result;
             }
 
-            var body = Translate(e);
-            Type elementType = TypeSystem.GetElementType(e.Type);
-
-            MethodInfo method = typeof(CouchQueryProvider).GetMethod(nameof(CouchQueryProvider.GetCouchListOrFiltered));
-            MethodInfo generic = method.MakeGenericMethod(elementType);
-            var result = generic.Invoke(this, new[] { body, (object)_filterMethodInfo, _filteringExpressions });
+            // For every unsupported method expression, execute it on the result
+            foreach (MethodCallExpression inMemoryCall in unsupportedMethodCallExpressions)
+            {
+                result = InvokeUnsupportedMethodCallExpression(result, inMemoryCall);
+            }
             return result;
         }
 
-        private string Translate(Expression e)
+        private string Translate(ref Expression e)
         {
-            e = Evaluator.PartialEval(e);
-            var whereVisitor = new WhereExpressionVisitor();
+            e = Local.PartialEval(e);
+            var whereVisitor = new BoolMemberToConstantEvaluator();
             e = whereVisitor.Visit(e);
 
+            var pretranslator = new QueryPretranslator();
+            e = pretranslator.Visit(e);
+
             return new QueryTranslator(_settings).Translate(e);
         }
-        
-        public object GetCouchListOrFiltered<T>(string body, MethodInfo filteringMethodInfo, Expression[] filteringExpressions)
-        {
+
+        public object GetCouchList<T>(string body)
+        {            
             FindResult<T> result = _flurlClient
                 .Request(_connectionString)
                 .AppendPathSegments(_db, "_find")
@@ -80,49 +82,116 @@ public object GetCouchListOrFiltered<T>(string body, MethodInfo filteringMethodI
                 .SendRequest();
 
             var couchList = new CouchList<T>(result.Docs.ToList(), result.Bookmark, result.ExecutionStats);
+            return couchList;            
+        }
 
-            if (filteringMethodInfo == null)
+        private Expression RemoveUnsupportedMethodExpressions(Expression expression, out bool hasUnsupportedMethods, IList<MethodCallExpression> unsupportedMethodCallExpressions)
+        {
+            if (unsupportedMethodCallExpressions == null)
             {
-                return couchList;
+                throw new ArgumentNullException(nameof(unsupportedMethodCallExpressions));
             }
 
-            var filteringMethods = typeof(Enumerable).GetMethods()
-                .Where(m => 
-                    m.Name == filteringMethodInfo.Name && 
-                    m.GetParameters().Length - 1 == filteringExpressions.Length)
-                .OrderBy(m => m.GetParameters().Length).ToList();
-
-
-            var invokeParameter = new object[filteringExpressions.Length + 1];
-            invokeParameter[0] = couchList;
-
-            bool IsRightOverload(MethodInfo m)
+            // Search for method calls to run in-memory,
+            // Once one is found all method calls after that must run in-memory.
+            // The expression to translate in JSON ends with the last not in-memory call.
+            bool IsUnsupportedMethodCallExpression(Expression ex)
             {
-                ParameterInfo[] parameters = m.GetParameters();
-                for (var i = 0; i < filteringExpressions.Length; i++)
+                if (ex is MethodCallExpression m)
                 {
-                    var lamdaExpression = filteringExpressions[i] as UnaryExpression;
-                    if (lamdaExpression == null)
+                    Expression nextCall = m.Arguments[0];
+                    // Check if the next expression is unsupported
+                    var isUnsupported = IsUnsupportedMethodCallExpression(nextCall);
+                    if (isUnsupported)
                     {
-                        return false;
+                        unsupportedMethodCallExpressions.Add(m);
+                        return isUnsupported;
                     }
-
-                    if (lamdaExpression.Operand.Type != parameters[i + 1].ParameterType)
+                    // If the next call is supported and the current is in the composite list
+                    if (QueryTranslator.CompositeQueryableMethods.Contains(m.Method.Name))
                     {
-                        return false;
+                        unsupportedMethodCallExpressions.Add(m);
+                        return true;
+                    }
+                    // If the next call is supported and the current is not in the supported list
+                    if (!QueryTranslator.NativeQueryableMethods.Contains(m.Method.Name))
+                    {
+                        unsupportedMethodCallExpressions.Add(m);
+                        expression = nextCall;
+                        return true;
                     }
-                    invokeParameter[i + 1] = lamdaExpression.Operand;
                 }
-                return true;
+                return false;
             }
 
-            MethodInfo rightOverload = filteringMethods.Single(IsRightOverload);
+            hasUnsupportedMethods = IsUnsupportedMethodCallExpression(expression);
+            return expression;
+        }
 
-            MethodInfo enumerableGenericFilteringMethod = rightOverload.MakeGenericMethod(typeof(T));
+        private object InvokeUnsupportedMethodCallExpression(object result, MethodCallExpression methodCallExpression)
+        {
+            MethodInfo queryableMethodInfo = methodCallExpression.Method;
+            Expression[] queryableMethodArguments = methodCallExpression.Arguments.ToArray();
 
+            // Since Max and Min are not map 1 to 1 from Queryable to Enumerable 
+            // they need to be handled differently
+            MethodInfo FindEnumerableMethod()
+            {
+                if (queryableMethodInfo.Name == nameof(Queryable.Max) || queryableMethodInfo.Name == nameof(Queryable.Min))
+                {
+                    return FindEnumerableMinMax(queryableMethodInfo);
+                }
+                return typeof(Enumerable).GetMethods().Single(enumerableMethodInfo =>
+                {
+                    return
+                        queryableMethodInfo.Name == enumerableMethodInfo.Name &&
+                        ReflectionComparator.IsCompatible(queryableMethodInfo, enumerableMethodInfo);
+                });
+            }
 
-            var filtered = enumerableGenericFilteringMethod.Invoke(null, invokeParameter);
+            // Find the equivalent method in Enumerable
+            MethodInfo enumarableMethodInfo = FindEnumerableMethod();
+
+            // Add the list as first parameter of the call
+            var invokeParameter = new List<object> { result };
+            // Convert everty other parameter expression to real values
+            IEnumerable<object> enumerableParameters = queryableMethodArguments.Skip(1).Select(GetArgumentValueFromExpression);
+            // Add the other parameter to the complete list
+            invokeParameter.AddRange(enumerableParameters);
+
+            Type[] requestedGenericParameters = enumarableMethodInfo.GetGenericMethodDefinition().GetGenericArguments();
+            Type[] genericParameters = queryableMethodInfo.GetGenericArguments();
+            Type[] usableParameters = genericParameters.Take(requestedGenericParameters.Length).ToArray();
+            MethodInfo enumarableGenericMethod = enumarableMethodInfo.MakeGenericMethod(usableParameters);
+            var filtered = enumarableGenericMethod.Invoke(null, invokeParameter.ToArray());
             return filtered;
         }
+
+        private object GetArgumentValueFromExpression(Expression e)
+        {
+            if (e is ConstantExpression c)
+            {
+                return c.Value;
+            }
+            if (e is UnaryExpression u && u.Operand is LambdaExpression l)
+            {
+                return l.Compile();
+            }
+            throw new NotImplementedException($"Expression of type {e.NodeType} not supported.");
+        }
+
+        private static MethodInfo FindEnumerableMinMax(MethodInfo queryableMethodInfo)
+        {
+            Type[] genericParams = queryableMethodInfo.GetGenericArguments();
+            MethodInfo finalMethodInfo = typeof(Enumerable).GetMethods().Single(enumerableMethodInfo =>
+            {
+                Type[] enumerableArguments = enumerableMethodInfo.GetGenericArguments();
+                return
+                    enumerableMethodInfo.Name == queryableMethodInfo.Name &&
+                    enumerableArguments.Length == genericParams.Length - 1 &&
+                    enumerableMethodInfo.ReturnType == genericParams[1];
+            });
+            return finalMethodInfo;
+        }           
     }
 }

src/CouchDB.Driver/ExpressionVisitors/WhereExpressionVisitor.cs → src/CouchDB.Driver/ExpressionVisitors/BoolMemberToConstantEvaluator.cs

@@ -2,15 +2,15 @@
 using System.Linq.Expressions;
 using System.Reflection;
 
-namespace CouchDB.Driver.ExpressionVisitors
+namespace CouchDB.Driver.CompositeExpressionsEvaluator
 {
-    internal  class WhereExpressionVisitor : ExpressionVisitor
+    internal  class BoolMemberToConstantEvaluator : ExpressionVisitor
     {
         private bool _visitingWhereMethod;
 
         protected override Expression VisitMethodCall(MethodCallExpression m)
         {
-            _visitingWhereMethod = m.Method.Name == "Where" && m.Method.DeclaringType == typeof(Queryable);
+            _visitingWhereMethod = m.Method.Name == nameof(Queryable.Where) && m.Method.DeclaringType == typeof(Queryable);
             if (_visitingWhereMethod)
             {
                 Expression result = base.VisitMethodCall(m);
@@ -22,7 +22,7 @@ protected override Expression VisitMethodCall(MethodCallExpression m)
 
         protected override Expression VisitBinary(BinaryExpression expression)
         {
-            if (expression.Right is ConstantExpression c && c.Type == typeof(bool) && 
+            if (_visitingWhereMethod && expression.Right is ConstantExpression c && c.Type == typeof(bool) && 
                 (expression.NodeType == ExpressionType.Equal || expression.NodeType == ExpressionType.NotEqual))
             {
                 return expression;