Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Query: Implement collection materialization in async #16212

Merged
merged 1 commit into from
Jun 26, 2019
Merged

Query: Implement collection materialization in async #16212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 25 additions & 9 deletions src/EFCore.Relational/Query/Pipeline/AsyncQueryingEnumerable.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ private class AsyncQueryingEnumerable<T> : IAsyncEnumerable<T>
{
private readonly RelationalQueryContext _relationalQueryContext;
private readonly SelectExpression _selectExpression;
private readonly Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> _shaper;
private readonly Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> _shaper;
private readonly IQuerySqlGeneratorFactory _querySqlGeneratorFactory;
private readonly Type _contextType;
private readonly IDiagnosticsLogger<DbLoggerCategory.Query> _logger;
Expand All @@ -33,7 +33,7 @@ public AsyncQueryingEnumerable(
ISqlExpressionFactory sqlExpressionFactory,
IParameterNameGeneratorFactory parameterNameGeneratorFactory,
SelectExpression selectExpression,
Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> shaper,
Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> shaper,
Type contextType,
IDiagnosticsLogger<DbLoggerCategory.Query> logger)
{
Expand All @@ -57,7 +57,7 @@ private sealed class AsyncEnumerator : IAsyncEnumerator<T>
private ResultCoordinator _resultCoordinator;
private readonly RelationalQueryContext _relationalQueryContext;
private readonly SelectExpression _selectExpression;
private readonly Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> _shaper;
private readonly Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> _shaper;
private readonly IQuerySqlGeneratorFactory _querySqlGeneratorFactory;
private readonly Type _contextType;
private readonly IDiagnosticsLogger<DbLoggerCategory.Query> _logger;
Expand Down Expand Up @@ -136,13 +136,29 @@ public async ValueTask<bool> MoveNextAsync()
_resultCoordinator = new ResultCoordinator();
}

var hasNext = _resultCoordinator.HasNext ?? await _dataReader.ReadAsync(_cancellationToken);
_resultCoordinator.HasNext = null;
var hasNext = _resultCoordinator.HasNext ?? await _dataReader.ReadAsync();
Current = default;

Current
= hasNext
? _shaper(_relationalQueryContext, _dataReader.DbDataReader, _indexMap, _resultCoordinator)
: default;
if (hasNext)
{
while (true)
{
_resultCoordinator.ResultReady = true;
_resultCoordinator.HasNext = null;
Current = _shaper(_relationalQueryContext, _dataReader.DbDataReader, Current, _indexMap, _resultCoordinator);
if (_resultCoordinator.ResultReady)
{
break;
}

if (!await _dataReader.ReadAsync())
{
_resultCoordinator.HasNext = false;

break;
}
}
}

return hasNext;
}
Expand Down
296 changes: 171 additions & 125 deletions src/EFCore.Relational/Query/Pipeline/IncludeCompilingExpressionVisitor.cs
View file
Open in desktop

Large diffs are not rendered by default.

32 changes: 24 additions & 8 deletions src/EFCore.Relational/Query/Pipeline/QueryingEnumerable.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ private class QueryingEnumerable<T> : IEnumerable<T>
{
private readonly RelationalQueryContext _relationalQueryContext;
private readonly SelectExpression _selectExpression;
private readonly Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> _shaper;
private readonly Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> _shaper;
private readonly IQuerySqlGeneratorFactory _querySqlGeneratorFactory;
private readonly Type _contextType;
private readonly IDiagnosticsLogger<DbLoggerCategory.Query> _logger;
Expand All @@ -31,7 +31,7 @@ public QueryingEnumerable(RelationalQueryContext relationalQueryContext,
ISqlExpressionFactory sqlExpressionFactory,
IParameterNameGeneratorFactory parameterNameGeneratorFactory,
SelectExpression selectExpression,
Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> shaper,
Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> shaper,
Type contextType,
IDiagnosticsLogger<DbLoggerCategory.Query> logger)
{
Expand All @@ -55,7 +55,7 @@ private sealed class Enumerator : IEnumerator<T>
private ResultCoordinator _resultCoordinator;
private readonly RelationalQueryContext _relationalQueryContext;
private readonly SelectExpression _selectExpression;
private readonly Func<QueryContext, DbDataReader, int[], ResultCoordinator, T> _shaper;
private readonly Func<QueryContext, DbDataReader, T, int[], ResultCoordinator, T> _shaper;
private readonly IQuerySqlGeneratorFactory _querySqlGeneratorFactory;
private readonly Type _contextType;
private readonly IDiagnosticsLogger<DbLoggerCategory.Query> _logger;
Expand Down Expand Up @@ -132,12 +132,28 @@ public bool MoveNext()
}

var hasNext = _resultCoordinator.HasNext ?? _dataReader.Read();
_resultCoordinator.HasNext = null;
Current = default;

Current
= hasNext
? _shaper(_relationalQueryContext, _dataReader.DbDataReader, _indexMap, _resultCoordinator)
: default;
if (hasNext)
{
while (true)
{
_resultCoordinator.ResultReady = true;
_resultCoordinator.HasNext = null;
Current = _shaper(_relationalQueryContext, _dataReader.DbDataReader, Current, _indexMap, _resultCoordinator);
if (_resultCoordinator.ResultReady)
{
break;
}

if (!_dataReader.Read())
{
_resultCoordinator.HasNext = false;

break;
}
}
}

return hasNext;
}
Expand Down
75 changes: 58 additions & 17 deletions src/EFCore.Relational/Query/Pipeline/RelationalCollectionShaperExpression.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,47 +1,88 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Linq.Expressions;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Query.Expressions.Internal;
using Microsoft.EntityFrameworkCore.Query.Internal;
using Microsoft.EntityFrameworkCore.Query.Pipeline;

namespace Microsoft.EntityFrameworkCore.Relational.Query.Pipeline
{
public class RelationalCollectionShaperExpression : CollectionShaperExpression
public class RelationalCollectionShaperExpression : Expression, IPrintable
{
public RelationalCollectionShaperExpression(
int collectionIndex,
Expression outerKeySelector,
Expression innerKeySelector,
int collectionId,
Expression parentIdentifier,
Expression outerIdentifier,
Expression selfIdentifier,
Expression innerShaper,
INavigation navigation)
: base(null, innerShaper, navigation)
{
CollectionIndex = collectionIndex;
OuterKeySelector = outerKeySelector;
InnerKeySelector = innerKeySelector;
CollectionId = collectionId;
ParentIdentifier = parentIdentifier;
OuterIdentifier = outerIdentifier;
SelfIdentifier = selfIdentifier;
InnerShaper = innerShaper;
Navigation = navigation;
}

public int CollectionIndex { get; }
public Expression OuterKeySelector { get; }
public Expression InnerKeySelector { get; }
public int CollectionId { get; }
public Expression ParentIdentifier { get; }
public Expression OuterIdentifier { get; }
public Expression SelfIdentifier { get; }
public Expression InnerShaper { get; }
public INavigation Navigation { get; }

public override Type Type => typeof(IEnumerable<>).MakeGenericType(InnerShaper.Type);
public override ExpressionType NodeType => ExpressionType.Extension;

protected override Expression VisitChildren(ExpressionVisitor visitor)
{
// Projection is always null so we do not need to visit it.
var outerKeySelector = visitor.Visit(OuterKeySelector);
var innerKeySelector = visitor.Visit(InnerKeySelector);
var parentIdentifier = visitor.Visit(ParentIdentifier);
var outerIdentifier = visitor.Visit(OuterIdentifier);
var selfIdentifier = visitor.Visit(SelfIdentifier);
var innerShaper = visitor.Visit(InnerShaper);

return Update(outerKeySelector, innerKeySelector, innerShaper);
return Update(parentIdentifier, outerIdentifier, selfIdentifier, innerShaper);
}

public RelationalCollectionShaperExpression Update(
Expression outerKeySelector, Expression innerKeySelector, Expression innerShaper)
Expression parentIdentifier, Expression outerIdentifier, Expression selfIdentifier, Expression innerShaper)
{
return outerKeySelector != OuterKeySelector || innerKeySelector != InnerKeySelector || innerShaper != InnerShaper
? new RelationalCollectionShaperExpression(CollectionIndex, outerKeySelector, innerKeySelector, innerShaper, Navigation)
return parentIdentifier != ParentIdentifier
|| outerIdentifier != OuterIdentifier
|| selfIdentifier != SelfIdentifier
|| innerShaper != InnerShaper
? new RelationalCollectionShaperExpression(
CollectionId, parentIdentifier, outerIdentifier, selfIdentifier, innerShaper, Navigation)
: this;
}

public void Print(ExpressionPrinter expressionPrinter)
{
expressionPrinter.StringBuilder.AppendLine("RelationalCollectionShaper:");
using (expressionPrinter.StringBuilder.Indent())
{
expressionPrinter.StringBuilder.AppendLine($"CollectionId: {CollectionId}");
expressionPrinter.StringBuilder.Append("ParentIdentifier:");
expressionPrinter.Visit(ParentIdentifier);
expressionPrinter.StringBuilder.AppendLine();
expressionPrinter.StringBuilder.Append("OuterIdentifier:");
expressionPrinter.Visit(OuterIdentifier);
expressionPrinter.StringBuilder.AppendLine();
expressionPrinter.StringBuilder.Append("SelfIdentifier:");
expressionPrinter.Visit(SelfIdentifier);
expressionPrinter.StringBuilder.AppendLine();
expressionPrinter.StringBuilder.Append("InnerShaper:");
expressionPrinter.Visit(InnerShaper);
expressionPrinter.StringBuilder.AppendLine();
expressionPrinter.StringBuilder.AppendLine($"Navigation: {Navigation.Name}");

}
}
}
}
10 changes: 0 additions & 10 deletions src/EFCore.Relational/Query/Pipeline/RelationalProjectionBindingRemovingExpressionVisitor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,16 +106,6 @@ protected override Expression VisitExtension(Expression extensionExpression)
projectionBindingExpression.Type);
}

if (extensionExpression is RelationalCollectionShaperExpression relationalCollectionShaperExpression)
{
return new RelationalCollectionShaperExpression(
relationalCollectionShaperExpression.CollectionIndex,
Visit(relationalCollectionShaperExpression.OuterKeySelector),
Visit(relationalCollectionShaperExpression.InnerKeySelector),
Visit(relationalCollectionShaperExpression.InnerShaper),
relationalCollectionShaperExpression.Navigation);
}

return base.VisitExtension(extensionExpression);
}

Expand Down
58 changes: 43 additions & 15 deletions src/EFCore.Relational/Query/Pipeline/RelationalShapedQueryCompilingExpressionVisitor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Collections.Generic;
using System.Data.Common;
using System.Linq;
using System.Linq.Expressions;
Expand Down Expand Up @@ -42,30 +43,33 @@ public RelationalShapedQueryCompilingExpressionVisitor(

protected override Expression VisitShapedQueryExpression(ShapedQueryExpression shapedQueryExpression)
{
var shaperBody = InjectEntityMaterializer(shapedQueryExpression.ShaperExpression);

var selectExpression = (SelectExpression)shapedQueryExpression.QueryExpression;

var dataReaderParameter = Expression.Parameter(typeof(DbDataReader), "dataReader");
var indexMapParameter = Expression.Parameter(typeof(int[]), "indexMap");
var resultCoordinatorParameter = Expression.Parameter(typeof(ResultCoordinator), "resultCoordinator");
var indexMapParameter = Expression.Parameter(typeof(int[]), "indexMap");

var shaper = new ShaperExpressionProcessingExpressionVisitor(
selectExpression,
dataReaderParameter,
resultCoordinatorParameter,
indexMapParameter)
.Inject(shapedQueryExpression.ShaperExpression);

shaper = InjectEntityMaterializer(shaper);

shaperBody = new RelationalProjectionBindingRemovingExpressionVisitor(selectExpression, dataReaderParameter)
.Visit(shaperBody);
shaperBody = new IncludeCompilingExpressionVisitor(dataReaderParameter, resultCoordinatorParameter, TrackQueryResults)
.Visit(shaperBody);
shaper = new RelationalProjectionBindingRemovingExpressionVisitor(selectExpression, dataReaderParameter)
.Visit(shaper);
shaper = new CustomShaperCompilingExpressionVisitor(
dataReaderParameter, resultCoordinatorParameter, TrackQueryResults)
.Visit(shaper);

if (selectExpression.IsNonComposedFromSql())
{
shaperBody = new IndexMapInjectingExpressionVisitor(indexMapParameter).Visit(shaperBody);
shaper = new IndexMapInjectingExpressionVisitor(indexMapParameter).Visit(shaper);
}

var shaperLambda = Expression.Lambda(
shaperBody,
QueryCompilationContext.QueryContextParameter,
dataReaderParameter,
indexMapParameter,
resultCoordinatorParameter);
var shaperLambda = (LambdaExpression)shaper;

return Expression.New(
(Async
Expand Down Expand Up @@ -110,8 +114,32 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp

private class ResultCoordinator
{
public bool ResultReady { get; set; }
public bool? HasNext { get; set; }
public object[] KeyValues { get; set; }
public IList<CollectionMaterializationContext> Collections { get; } = new List<CollectionMaterializationContext>();
public object[] OuterKeyValues { get; set; }
public object[] InnerKeyValues { get; set; }
}

private class CollectionMaterializationContext
{
public CollectionMaterializationContext(object parent, object collection, object[] outerIdentifier)
{
Parent = parent;
Collection = collection;
OuterIdentifier = outerIdentifier;
}

public object Parent { get; }
public object Collection { get; }
public object Current { get; private set; }
public object[] OuterIdentifier { get; }
public object[] SelfIdentifier { get; set; }

public void UpdateCurrent(object current)
{
Current = current;
}
}
}
}
12 changes: 5 additions & 7 deletions src/EFCore.Relational/Query/Pipeline/RelationalShapedQueryOptimizingExpressionVisitors.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,28 +36,26 @@ public override Expression Visit(Expression query)
query = new SqlExpressionOptimizingVisitor(SqlExpressionFactory, UseRelationalNulls).Visit(query);
query = new NullComparisonTransformingExpressionVisitor().Visit(query);

if (query is ShapedQueryExpression shapedQueryExpression)
{
shapedQueryExpression.ShaperExpression
= new ShaperExpressionProcessingExpressionVisitor((SelectExpression)shapedQueryExpression.QueryExpression)
.Inject(shapedQueryExpression.ShaperExpression);
}

return query;
}
}

public class CollectionJoinApplyingExpressionVisitor : ExpressionVisitor
{
private int _collectionId;

protected override Expression VisitExtension(Expression extensionExpression)
{
if (extensionExpression is CollectionShaperExpression collectionShaperExpression)
{
var collectionId = _collectionId++;

var innerShaper = Visit(collectionShaperExpression.InnerShaper);

var selectExpression = (SelectExpression)collectionShaperExpression.Projection.QueryExpression;
return selectExpression.ApplyCollectionJoin(
collectionShaperExpression.Projection.Index.Value,
collectionId,
innerShaper,
collectionShaperExpression.Navigation);
}
Expand Down
Loading