Query: Navigation Expansion take 2

- Add TransparentIdentifier to uniquify them across the pipeline
- Convert all Enumerable based method which we can translate to Queryable
- Add QueryableMethodProvider to get methodInfos on Queryable robustly
- Split out subquery member pushdown out of navigation expansion
- Lastly expand navigations

src/EFCore.Cosmos/Query/Internal/CosmosProjectionBindingExpressionVisitor.cs

@@ -3,9 +3,11 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq.Expressions;
 using System.Reflection;
 using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Internal;
+using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Metadata;
 using Microsoft.EntityFrameworkCore.Query;
 using Microsoft.EntityFrameworkCore.Storage;
@@ -237,6 +239,92 @@ protected override Expression VisitMember(MemberExpression memberExpression)
             }
         }
 
+        protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression)
+        {
+            if (methodCallExpression.TryGetEFPropertyArguments(out var source, out var memberName))
+            {
+                if (!_clientEval)
+                {
+                    return null;
+                }
+
+                var visitedSource = Visit(source);
+
+                EntityShaperExpression shaperExpression;
+                switch (visitedSource)
+                {
+                    case EntityShaperExpression shaper:
+                        shaperExpression = shaper;
+                        break;
+
+                    case UnaryExpression unaryExpression:
+                        shaperExpression = unaryExpression.Operand as EntityShaperExpression;
+                        if (shaperExpression == null)
+                        {
+                            return null;
+                        }
+                        break;
+
+                    default:
+                        return null;
+                }
+
+                EntityProjectionExpression innerEntityProjection;
+                switch (shaperExpression.ValueBufferExpression)
+                {
+                    case ProjectionBindingExpression innerProjectionBindingExpression:
+                        innerEntityProjection = (EntityProjectionExpression)_selectExpression.Projection[
+                            innerProjectionBindingExpression.Index.Value].Expression;
+                        break;
+
+                    case UnaryExpression unaryExpression:
+                        innerEntityProjection = (EntityProjectionExpression)((UnaryExpression)unaryExpression.Operand).Operand;
+                        break;
+
+                    default:
+                        throw new InvalidOperationException();
+                }
+
+                var navigationProjection = innerEntityProjection.BindMember(
+                    memberName, visitedSource.Type, clientEval: true, out var propertyBase);
+
+                if (!(propertyBase is INavigation navigation)
+                    || !navigation.IsEmbedded())
+                {
+                    return null;
+                }
+
+                switch (navigationProjection)
+                {
+                    case EntityProjectionExpression entityProjection:
+                        return new EntityShaperExpression(
+                            navigation.GetTargetType(),
+                            Expression.Convert(Expression.Convert(entityProjection, typeof(object)), typeof(ValueBuffer)),
+                            nullable: true);
+
+                    case ObjectArrayProjectionExpression objectArrayProjectionExpression:
+                        {
+                            var innerShaperExpression = new EntityShaperExpression(
+                                navigation.GetTargetType(),
+                                Expression.Convert(
+                                    Expression.Convert(objectArrayProjectionExpression.InnerProjection, typeof(object)), typeof(ValueBuffer)),
+                                nullable: true);
+
+                            return new CollectionShaperExpression(
+                                objectArrayProjectionExpression,
+                                innerShaperExpression,
+                                navigation,
+                                innerShaperExpression.EntityType.ClrType);
+                        }
+
+                    default:
+                        throw new InvalidOperationException();
+                }
+            }
+
+            return base.VisitMethodCall(methodCallExpression);
+        }
+
         /// <summary>
         ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
         ///     the same compatibility standards as public APIs. It may be changed or removed without notice in

src/EFCore.Cosmos/Query/Internal/CosmosSqlTranslatingExpressionVisitor.cs

@@ -102,17 +102,33 @@ protected override Expression VisitMember(MemberExpression memberExpression)
 
         private bool TryBindMember(Expression source, MemberIdentity member, out Expression expression)
         {
+            Type convertedType = null;
+            if (source is UnaryExpression unaryExpression
+                && unaryExpression.NodeType == ExpressionType.Convert)
+            {
+                if (unaryExpression.Type != typeof(object))
+                {
+                    convertedType = unaryExpression.Type;
+                }
+                source = unaryExpression.Operand;
+            }
+
             if (source is EntityProjectionExpression entityProjectionExpression)
             {
                 expression = member.MemberInfo != null
-                    ? entityProjectionExpression.BindMember(member.MemberInfo, null, clientEval: false, out _)
-                    : entityProjectionExpression.BindMember(member.Name, null, clientEval: false, out _);
+                    ? entityProjectionExpression.BindMember(member.MemberInfo, convertedType, clientEval: false, out _)
+                    : entityProjectionExpression.BindMember(member.Name, convertedType, clientEval: false, out _);
                 return expression != null;
             }
 
             if (source is MemberExpression innerMemberExpression
                 && TryBindMember(innerMemberExpression, MemberIdentity.Create(innerMemberExpression.Member), out var innerResult))
             {
+                if (convertedType != null)
+                {
+                    innerResult = Expression.Convert(innerResult, convertedType);
+                }
+
                 return TryBindMember(innerResult, member, out expression);
             }
 

src/EFCore.InMemory/Query/Internal/InMemoryExpressionTranslatingExpressionVisitor.cs

@@ -9,7 +9,6 @@
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Metadata;
 using Microsoft.EntityFrameworkCore.Query;
-using Microsoft.EntityFrameworkCore.Query.NavigationExpansion.Internal;
 using Microsoft.EntityFrameworkCore.Storage;
 
 namespace Microsoft.EntityFrameworkCore.InMemory.Query.Internal
@@ -19,16 +18,51 @@ public class InMemoryExpressionTranslatingExpressionVisitor : ExpressionVisitor
         private const string CompiledQueryParameterPrefix = "__";
 
         private readonly QueryableMethodTranslatingExpressionVisitor _queryableMethodTranslatingExpressionVisitor;
+        private readonly EntityProjectionFindingExpressionVisitor _entityProjectionFindingExpressionVisitor;
 
         public InMemoryExpressionTranslatingExpressionVisitor(
             QueryableMethodTranslatingExpressionVisitor queryableMethodTranslatingExpressionVisitor)
         {
             _queryableMethodTranslatingExpressionVisitor = queryableMethodTranslatingExpressionVisitor;
+            _entityProjectionFindingExpressionVisitor = new EntityProjectionFindingExpressionVisitor();
+        }
+
+        private class EntityProjectionFindingExpressionVisitor : ExpressionVisitor
+        {
+            private bool _found;
+            public bool Find(Expression expression)
+            {
+                _found = false;
+
+                Visit(expression);
+
+                return _found;
+            }
+
+            public override Expression Visit(Expression expression)
+            {
+                if (_found)
+                {
+                    return expression;
+                }
+
+                if (expression is EntityProjectionExpression)
+                {
+                    _found = true;
+                    return expression;
+                }
+
+                return base.Visit(expression);
+            }
         }
 
         public virtual Expression Translate(Expression expression)
         {
-            return Visit(expression);
+            var result = Visit(expression);
+
+            return _entityProjectionFindingExpressionVisitor.Find(result)
+                ? null
+                : result;
         }
 
         protected override Expression VisitMember(MemberExpression memberExpression)
@@ -244,9 +278,6 @@ protected override Expression VisitExtension(Expression extensionExpression)
                             : Expression.Convert(translation, nullConditionalExpression.Type);
                     }
 
-                case CorrelationPredicateExpression correlationPredicateExpression:
-                    return Visit(correlationPredicateExpression.EqualExpression);
-
                 default:
                     throw new InvalidOperationException();
             }

src/EFCore.InMemory/Query/Internal/InMemoryProjectionBindingExpressionVisitor.cs

@@ -6,7 +6,6 @@
 using System.Linq;
 using System.Linq.Expressions;
 using Microsoft.EntityFrameworkCore.Query;
-using Microsoft.EntityFrameworkCore.Query.NavigationExpansion.Internal;
 using Microsoft.EntityFrameworkCore.Storage;
 
 namespace Microsoft.EntityFrameworkCore.InMemory.Query.Internal

src/EFCore.InMemory/Query/Internal/InMemoryQueryExpression.cs

@@ -384,33 +384,15 @@ public virtual void AddInnerJoin(
             _projectionMapping = projectionMapping;
         }
 
-        public readonly struct TransparentIdentifier<TOuter, TInner>
-        {
-            [UsedImplicitly]
-#pragma warning disable IDE0051 // Remove unused private members
-            private TransparentIdentifier(TOuter outer, TInner inner)
-#pragma warning restore IDE0051 // Remove unused private members
-            {
-                Outer = outer;
-                Inner = inner;
-            }
-
-            [UsedImplicitly]
-            public readonly TOuter Outer;
-
-            [UsedImplicitly]
-            public readonly TInner Inner;
-        }
-
         public virtual void AddLeftJoin(
             InMemoryQueryExpression innerQueryExpression,
             LambdaExpression outerKeySelector,
             LambdaExpression innerKeySelector,
             Type transparentIdentifierType)
         {
             // GroupJoin phase
-            var groupTransparentIdentifierType = typeof(TransparentIdentifier<,>)
-                .MakeGenericType(typeof(ValueBuffer), typeof(IEnumerable<ValueBuffer>));
+            var groupTransparentIdentifierType = TransparentIdentifierFactory.Create(
+                typeof(ValueBuffer), typeof(IEnumerable<ValueBuffer>));
             var outerParameter = Parameter(typeof(ValueBuffer), "outer");
             var innerParameter = Parameter(typeof(IEnumerable<ValueBuffer>), "inner");
             var outerMemberInfo = groupTransparentIdentifierType.GetTypeInfo().GetDeclaredField("Outer");

src/EFCore.InMemory/Query/Internal/InMemoryQueryableMethodTranslatingExpressionVisitor.cs

@@ -203,7 +203,7 @@ protected override ShapedQueryExpression TranslateJoin(ShapedQueryExpression out
             outerKeySelector = TranslateLambdaExpression(outer, outerKeySelector);
             innerKeySelector = TranslateLambdaExpression(inner, innerKeySelector);
 
-            var transparentIdentifierType = CreateTransparentIdentifierType(
+            var transparentIdentifierType = TransparentIdentifierFactory.Create(
                 resultSelector.Parameters[0].Type,
                 resultSelector.Parameters[1].Type);
 
@@ -237,7 +237,7 @@ protected override ShapedQueryExpression TranslateLeftJoin(ShapedQueryExpression
             outerKeySelector = TranslateLambdaExpression(outer, outerKeySelector);
             innerKeySelector = TranslateLambdaExpression(inner, innerKeySelector);
 
-            var transparentIdentifierType = CreateTransparentIdentifierType(
+            var transparentIdentifierType = TransparentIdentifierFactory.Create(
                 resultSelector.Parameters[0].Type,
                 resultSelector.Parameters[1].Type);
 
@@ -349,7 +349,7 @@ protected override ShapedQueryExpression TranslateSelectMany(
             {
                 if (Visit(collectionSelectorBody) is ShapedQueryExpression inner)
                 {
-                    var transparentIdentifierType = CreateTransparentIdentifierType(
+                    var transparentIdentifierType = TransparentIdentifierFactory.Create(
                         resultSelector.Parameters[0].Type,
                         resultSelector.Parameters[1].Type);
 

src/EFCore.Relational/Query/Internal/RelationalProjectionBindingExpressionVisitor.cs

@@ -7,7 +7,6 @@
 using System.Linq.Expressions;
 using System.Reflection;
 using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
-using Microsoft.EntityFrameworkCore.Query.NavigationExpansion.Internal;
 using Microsoft.EntityFrameworkCore.Storage;
 
 namespace Microsoft.EntityFrameworkCore.Query.Internal

src/EFCore.Relational/Query/Internal/ShaperExpressionDedupingExpressionVisitor.cs

@@ -5,7 +5,6 @@
 using System.Linq;
 using System.Linq.Expressions;
 using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
-using Microsoft.EntityFrameworkCore.Query.NavigationExpansion.Internal;
 
 namespace Microsoft.EntityFrameworkCore.Query.Internal
 {

src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs

@@ -7,7 +7,9 @@
 using System.Linq;
 using System.Linq.Expressions;
 using System.Reflection;
+using System.Runtime.InteropServices.ComTypes;
 using Microsoft.EntityFrameworkCore.Diagnostics;
+using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Internal;
 using Microsoft.EntityFrameworkCore.Metadata;
 using Microsoft.EntityFrameworkCore.Metadata.Internal;
@@ -59,9 +61,6 @@ private RelationalQueryableMethodTranslatingExpressionVisitor(
             _sqlExpressionFactory = sqlExpressionFactory;
         }
 
-        private static Type CreateTransparentIdentifierType(Type outerType, Type innerType)
-            => typeof(TransparentIdentifier<,>).MakeGenericType(outerType, innerType);
-
         protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression)
         {
             if (methodCallExpression.Method.DeclaringType == typeof(RelationalQueryableExtensions)
@@ -447,7 +446,7 @@ protected override ShapedQueryExpression TranslateJoin(
             var joinPredicate = CreateJoinPredicate(outer, outerKeySelector, inner, innerKeySelector);
             if (joinPredicate != null)
             {
-                var transparentIdentifierType = CreateTransparentIdentifierType(
+                var transparentIdentifierType = TransparentIdentifierFactory.Create(
                     resultSelector.Parameters[0].Type,
                     resultSelector.Parameters[1].Type);
 
@@ -470,7 +469,7 @@ protected override ShapedQueryExpression TranslateLeftJoin(ShapedQueryExpression
             var joinPredicate = CreateJoinPredicate(outer, outerKeySelector, inner, innerKeySelector);
             if (joinPredicate != null)
             {
-                var transparentIdentifierType = CreateTransparentIdentifierType(
+                var transparentIdentifierType = TransparentIdentifierFactory.Create(
                     resultSelector.Parameters[0].Type,
                     resultSelector.Parameters[1].Type);
 
@@ -735,7 +734,7 @@ protected override ShapedQueryExpression TranslateSelectMany(
             {
                 if (Visit(collectionSelectorBody) is ShapedQueryExpression inner)
                 {
-                    var transparentIdentifierType = CreateTransparentIdentifierType(
+                    var transparentIdentifierType = TransparentIdentifierFactory.Create(
                         resultSelector.Parameters[0].Type,
                         resultSelector.Parameters[1].Type);
 
@@ -947,6 +946,28 @@ protected override Expression VisitMember(MemberExpression memberExpression)
                 return memberExpression.Update(innerExpression);
             }
 
+            protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression)
+            {
+                if (methodCallExpression.TryGetEFPropertyArguments(out var source, out var navigationName))
+                {
+                    source = Visit(source);
+                    if (source is EntityShaperExpression
+                        || (source is UnaryExpression innerUnaryExpression
+                        && innerUnaryExpression.NodeType == ExpressionType.Convert
+                        && innerUnaryExpression.Operand is EntityShaperExpression))
+                    {
+                        var collectionNavigation = Expand(source, MemberIdentity.Create(navigationName));
+                        if (collectionNavigation != null)
+                        {
+                            return collectionNavigation;
+                        }
+                    }
+
+                    return methodCallExpression.Update(null, new[] { source, methodCallExpression.Arguments[1] });
+                }
+                return base.VisitMethodCall(methodCallExpression);
+            }
+
             protected override Expression VisitExtension(Expression extensionExpression)
                 => extensionExpression is EntityShaperExpression
                     ? extensionExpression
@@ -983,7 +1004,8 @@ private Expression Expand(Expression source, MemberIdentity member)
                         ? entityType.FindNavigation(member.MemberInfo)
                         : entityType.FindNavigation(member.Name);
 
-                    if (navigation != null)
+                    if (navigation != null
+                        && navigation.ForeignKey.IsOwnership)
                     {
                         if (navigation.IsCollection())
                         {