diff --git a/docs/release-notes/.FSharp.Core/11.0.200.md b/docs/release-notes/.FSharp.Core/11.0.200.md index 49e81a0efdf..fa81ab045fc 100644 --- a/docs/release-notes/.FSharp.Core/11.0.200.md +++ b/docs/release-notes/.FSharp.Core/11.0.200.md @@ -6,5 +6,7 @@ * Correct `FSharpValue.PreComputeTuplePropertyInfo` to report that struct tuples store elements in fields, not properties, and recommend supported tuple readers instead of reporting a valid index as out of range. ([Issue #20801](https://github.com/dotnet/fsharp/issues/20801), [PR #20815](https://github.com/dotnet/fsharp/pull/20815)) * Quotation debug text from `Expr.ToString()`, `Expr.ToString(full)`, and `%A` now prints struct tuple construction as `NewStructTuple`; reference tuples remain `NewTuple`. ([Issue #20800](https://github.com/dotnet/fsharp/issues/20800), [PR #20814](https://github.com/dotnet/fsharp/pull/20814)) +* Fix struct tuples in query expressions and in lambdas converted to LINQ expression trees: selecting one no longer fails with "unreachable", and taking one apart no longer fails with a tuple index out of range. ([Issue #15522](https://github.com/dotnet/fsharp/issues/15522), [PR #20894](https://github.com/dotnet/fsharp/pull/20894)) +* Fix LINQ providers failing on, or evaluating on the client, queries with a `for`, `join`, `groupJoin` or `leftOuterJoin` after two `for`s: the query now continues one flat chain of `SelectMany`, `Join` and `GroupJoin`, as C# builds it, instead of nesting the rest of the query inside the first `SelectMany`, which EF Core could translate only as a lateral join, if at all. ([Issue #8459](https://github.com/dotnet/fsharp/issues/8459), [PR #20902](https://github.com/dotnet/fsharp/pull/20902)) ### Changed diff --git a/docs/release-notes/PENDING_BREAKING_CHANGES.md b/docs/release-notes/PENDING_BREAKING_CHANGES.md index df80c994bea..ccb0acdc5ac 100644 --- a/docs/release-notes/PENDING_BREAKING_CHANGES.md +++ b/docs/release-notes/PENDING_BREAKING_CHANGES.md @@ -52,6 +52,20 @@ a.Equals(b) // Was: false, Now: true // New: if (node.NodeType == ExpressionType.ArrayIndex) ``` +### A `for` or join after two `for`s continues the chain + +Over an `IQueryable`, a query that goes on after two `for`s no longer nests the rest inside the collection selector of the +first `SelectMany`; like C#, it pairs the elements of the first two sources and continues from the pairs. + +```fsharp +query { for c in db.Customers do for o in c.Orders do for l in o.Lines do select (c.Name, l.Id) } +// Was: Customers.SelectMany(c => c.Orders.SelectMany(o => o.Lines, ...), (c, x) => x) +// Now: Customers.SelectMany(c => c.Orders, (c, o) => new AnonymousObject(c, o)).SelectMany(pair => pair.Item2.Lines, ...) +``` + +**Impact:** Code that walks the expression tree of such a query sees a longer chain and no nested `SelectMany`, `Join` +or `GroupJoin`. The rows are the same. + --- ## IQueryable Type Preservation 🟡 diff --git a/src/FSharp.Core/Linq.fs b/src/FSharp.Core/Linq.fs index b8025b39bb4..431f6fd9514 100644 --- a/src/FSharp.Core/Linq.fs +++ b/src/FSharp.Core/Linq.fs @@ -475,7 +475,14 @@ module LeafExpressionConverter = | TupleGet(arg, n) -> let argP = ConvExprToLinqInContext env arg - let rec build ty argP n = + let rec build (ty: Type) argP n = + // A struct tuple holds its elements in fields, which PreComputeTuplePropertyInfo cannot return + if ty.IsValueType then + if n < 7 then + Expression.Field(argP, ty.GetField("Item" + string (n + 1))) |> asExpr + else + build (ty.GetGenericArguments().[7]) (Expression.Field(argP, ty.GetField "Rest") |> asExpr) (n - 7) + else match FSharpValue.PreComputeTuplePropertyInfo(ty, n) with | propInfo, None -> Expression.Property(argP, propInfo) |> asExpr diff --git a/src/FSharp.Core/Query.fs b/src/FSharp.Core/Query.fs index 2f84efd0eee..1964214142c 100644 --- a/src/FSharp.Core/Query.fs +++ b/src/FSharp.Core/Query.fs @@ -295,6 +295,7 @@ open Microsoft.FSharp.Linq.RuntimeHelpers.LeafExpressionConverter open Microsoft.FSharp.Quotations open Microsoft.FSharp.Quotations.Patterns open Microsoft.FSharp.Quotations.DerivedPatterns +open Microsoft.FSharp.Reflection open Microsoft.FSharp.Linq.QueryRunExtensions @@ -1127,6 +1128,66 @@ module Query = let (|CallLeftOuterJoin|_|) = (|SpecificCallToMethod|_|) (methodhandleof (fun (query:QueryBuilder, arg1, arg2, arg3, arg4, arg5) -> query.LeftOuterJoin(arg1, arg2, arg3, arg4, arg5))) + /// A 'join', 'groupJoin' or 'leftOuterJoin'. The inner group of a 'leftOuterJoin' is already replaced by 'group.DefaultIfEmpty()'. + [] + type JoinLike = + { Make: bool * Type * Type * Type * Type * Expr * Expr * Var * Expr * Var * Expr * Var * Var * Expr -> Expr + InnerResultConv: ConversionDescription -> ConversionDescription + QTy: Type + OuterSource: Expr + InnerSource: Expr + OuterKeyVar: Var + OuterKeySelector: Expr + InnerKeyVar: Var + InnerKeySelector: Expr + OuterResultVar: Var + InnerResultVar: Var + ElementSelector: Expr } + + [] + let (|CallJoinLike|_|) e = + let joinLike make innerResultConv qTy (outerSource, innerSource, outerKeyVar, outerKeySelector, innerKeyVar, innerKeySelector, outerResultVar, innerResultVar, elementSelector) = + ValueSome + { Make = make + InnerResultConv = innerResultConv + QTy = qTy + OuterSource = outerSource + InnerSource = innerSource + OuterKeyVar = outerKeyVar + OuterKeySelector = outerKeySelector + InnerKeyVar = innerKeyVar + InnerKeySelector = innerKeySelector + OuterResultVar = outerResultVar + InnerResultVar = innerResultVar + ElementSelector = elementSelector } + match e with + | CallJoin(_, GenericArgs [_; qTy; _; _; _], + [ outerSource + innerSource + Lambda(outerKeyVar, outerKeySelector) + Lambda(innerKeyVar, innerKeySelector) + LambdasNoDetupling([outerResultVar; innerResultVar], elementSelector)]) -> + joinLike MakeJoin id qTy (outerSource, innerSource, outerKeyVar, outerKeySelector, innerKeyVar, innerKeySelector, outerResultVar, innerResultVar, elementSelector) + + | CallGroupJoin(_, GenericArgs [_; qTy; _; _; _], + [ outerSource + innerSource + Lambda(outerKeyVar, outerKeySelector) + Lambda(innerKeyVar, innerKeySelector) + LambdasNoDetupling([outerResultVar; innerResultGroupVar], elementSelector)]) -> + joinLike MakeGroupJoin MakeSeqConv qTy (outerSource, innerSource, outerKeyVar, outerKeySelector, innerKeyVar, innerKeySelector, outerResultVar, innerResultGroupVar, elementSelector) + + | CallLeftOuterJoin(_, GenericArgs [ _; qTy; innerSourceTy; _; _], + [ outerSource + innerSource + Lambda(outerKeyVar, outerKeySelector) + Lambda(innerKeyVar, innerKeySelector) + LambdasNoDetupling([outerResultVar; innerResultGroupVar], elementSelector)]) -> + let elementSelector = elementSelector.Substitute (fun v -> if v = innerResultGroupVar then Some (MakeDefaultIfEmpty ([innerSourceTy], [Expr.Var innerResultGroupVar])) else None) + joinLike MakeGroupJoin id qTy (outerSource, innerSource, outerKeyVar, outerKeySelector, innerKeyVar, innerKeySelector, outerResultVar, innerResultGroupVar, elementSelector) + + | _ -> ValueNone + let (|CallAverageBy|_|) = (|SpecificCall2|_|) (methodhandleof (fun (query:QueryBuilder, arg1: QuerySource, arg2: double->double) -> query.AverageBy(arg1, arg2))) @@ -1156,8 +1217,9 @@ module Query = /// immutable-to-mutable conversion information, convert it back to an expression involving immutable tuples or records. let rec ConvMutableToImmutable conv mutExpr = match conv with - | TupleConv convs -> - Expr.NewTuple (convs |> List.mapi (fun i conv -> ConvMutableToImmutable conv (AnonymousObjectGet (mutExpr, i)))) + | TupleConv (isStruct, convs) -> + let elements = convs |> List.mapi (fun i conv -> ConvMutableToImmutable conv (AnonymousObjectGet (mutExpr, i))) + if isStruct then Expr.NewStructTuple elements else Expr.NewTuple elements | RecordConv (typ, convs) -> Expr.NewRecord (typ, convs |> List.mapi (fun i conv -> ConvMutableToImmutable conv (AnonymousObjectGet (mutExpr, i)))) @@ -1265,6 +1327,50 @@ module Query = | TransInnerResult.Select(canElim, isQTy, mutSource, mutSelectorVar, mutSelectorBody) -> MakeSelect(canElim, isQTy, CommitTransInnerResult mutSource, mutSelectorVar, mutSelectorBody) + /// IQueryable.SelectMany expects an IEnumerable return + let CoerceToIEnumerable (mutElemTy: Type) (mutSource: Expr) = + let mutSourceTy = mutSource.Type + if mutSourceTy.IsGenericType && mutSourceTy.GetGenericTypeDefinition() = typedefof> then + mutSource + else + Expr.Coerce (mutSource, MakeIEnumerableTy mutElemTy) + + /// Whether a 'for' with this body translates into nothing but 'Where' and 'Select' + let rec ForBodyIsProjection immutBody = + match immutBody with + | CallQueryBuilderYield _ -> true + | LetExprReduction reduced + | MacroReduction reduced -> ForBodyIsProjection reduced + | CallWhere (_, _, immutBody, _) + | IfThenElse (_, immutBody, ZeroOnElseBranch) -> ForBodyIsProjection immutBody + | _ -> false + + let IsFreeIn (v: Var) (e: Expr) = e.GetFreeVars() |> Seq.contains v + + let rec ExistsInExpr (predicate: Expr -> bool) (e: Expr) = + predicate e + || (match e with + | ExprShape.ShapeCombination (_, args) -> List.exists (ExistsInExpr predicate) args + | ExprShape.ShapeLambda (_, body) -> ExistsInExpr predicate body + | ExprShape.ShapeVar _ -> false) + + /// Whether the elements of the source hold a group already put through 'DefaultIfEmpty', as 'leftOuterJoin' builds them. + /// EF Core translates what is nested under such a source with a lateral join where the database has one, but cannot + /// carry that sequence along the main chain, so continuing the chain there would lose a query that works. + let HoldsDefaultedGroup (mutSource: Expr) = + let isLinqCall (name: string) (e: Expr) = + match e with + | Call (None, meth, _) -> + String.Equals (meth.Name, name, StringComparison.Ordinal) + && (Type.op_Equality (meth.DeclaringType, typeof) || Type.op_Equality (meth.DeclaringType, typeof)) + | _ -> false + + mutSource + |> ExistsInExpr (fun e -> + match e with + | Call (None, _, args) when isLinqCall "GroupJoin" e -> ExistsInExpr (isLinqCall "DefaultIfEmpty") (List.last args) + | _ -> false) + /// Given the inner of query expression in terms of query.For, query.Select, query.Yield, query.Where etc., /// and including immutable tuples and immutable records, build an equivalent query expression /// in terms of LINQ operators, operating over mutable tuples. Return the conversion @@ -1297,7 +1403,7 @@ module Query = let mutSource, sourceConv = TransInner CanEliminate.Yes check immutSource // If the body of a "For" is nothing but Yield/IfThenElse/Where, then it can be fully rewritten away. - let rec TransFor mutSource immutSelector = + let rec TransFor (mutSource, sourceConv, immutSelectorVar) immutSelector = match immutSelector with // query.For (source, (fun selectorVar -> yield res)) @> @@ -1311,10 +1417,10 @@ module Query = TransInnerResult.MakeSelect (canElim, qTyIsIQueryable qTy, mutSource, mutSelectorVar, mutSelectorBody), selectorConv | LetExprReduction reduced -> - TransFor mutSource reduced + TransFor (mutSource, sourceConv, immutSelectorVar) reduced | MacroReduction reduced -> - TransFor mutSource reduced + TransFor (mutSource, sourceConv, immutSelectorVar) reduced // query.For (source, (fun selectorVar -> if g then selectorBody else query.Zero())) @> // ~~> TRANS(query.For (source.Where(fun selectorVar -> g), (fun selectorVar -> selectorBody)) @@ -1323,7 +1429,41 @@ module Query = let mutSelectorVar, mutPredicateBody = ConvertImmutableConsumerToMutableConsumer sourceConv (immutSelectorVar, MacroExpand immutPredicateBody) let mutSource = MakeWhere(qTyIsIQueryable qTy, CommitTransInnerResult mutSource, mutSelectorVar, mutPredicateBody) - TransFor (TransInnerResult.Other mutSource) immutSelectorBody + TransFor (TransInnerResult.Other mutSource, sourceConv, immutSelectorVar) immutSelectorBody + + // A provider translates a join only in the main chain, so a join that follows a 'for' continues it. + // + // query.For (source, (fun selectorVar -> query.For (join (outer, inner, ...), (fun joinedVar -> body)))) @> + // ~~> TRANS(query.For (join (source.SelectMany(fun selectorVar -> outer, (fun selectorVar outerVar -> (selectorVar, outerVar))), inner, ...), (fun joinedVar -> body))) + | CallQueryBuilderFor (_, _, [ CallJoinLike join; Lambda (immutJoinedVar, immutBody) ]) + when qTyIsIQueryable qTy + && not (IsFreeIn immutSelectorVar join.InnerSource) + && not (IsFreeIn immutSelectorVar join.InnerKeySelector) + && not (IsFreeIn immutSelectorVar immutBody) + && not (HoldsDefaultedGroup (CommitTransInnerResult mutSource)) -> + + let mutPairs, pairConv, immutPairVar, readFromPair = TransPairs check (mutSource, sourceConv, immutSelectorVar) (join.OuterSource, join.OuterKeyVar) + // Expanded first: the lets that take a tuple of the outer element apart are only removed while they read 'tupledArg' + let join = + { join with + OuterKeyVar = immutPairVar + OuterKeySelector = readFromPair join.OuterKeyVar (MacroExpand join.OuterKeySelector) + OuterResultVar = immutPairVar + ElementSelector = readFromPair join.OuterResultVar (MacroExpand join.ElementSelector) } + let mutJoined, joinedConv = TransJoin check true (mutPairs, pairConv) join + TransFor (mutJoined, joinedConv, immutJoinedVar) immutBody + + // A provider translates a 'SelectMany' nested in the collection selector of another only as a lateral join, if at all. + // + // query.For (source, (fun selectorVar -> query.For (inner, (fun innerVar -> body)))) @>, where body is more than a projection + // ~~> TRANS(query.For (source.SelectMany(fun selectorVar -> inner, (fun selectorVar innerVar -> (selectorVar, innerVar))), (fun (selectorVar, innerVar) -> body))) + | CallQueryBuilderFor (_, _, [ immutInnerSource; Lambda (immutInnerVar, immutBody) ]) + when qTyIsIQueryable qTy + && not (ForBodyIsProjection immutBody) + && not (HoldsDefaultedGroup (CommitTransInnerResult mutSource)) -> + + let mutPairs, pairConv, immutPairVar, readFromPair = TransPairs check (mutSource, sourceConv, immutSelectorVar) (immutInnerSource, immutInnerVar) + TransFor (TransInnerResult.Other mutPairs, pairConv, immutPairVar) (readFromPair immutInnerVar immutBody) // query.For (source, (fun selectorVar -> immutSelectorBody)) @> // ~~> source.SelectMany(fun selectorVar -> immutSelectorBody) @@ -1344,18 +1484,11 @@ module Query = let mutTargetSelector = Expr.Var mutInterimVar mutInterimSelectorBody, mutInterimVar, mutTargetSelector - // IQueryable.SelectMany expects an IEnumerable return - let mutInterimSelectorBody = - let mutSelectorBodyTy = mutInterimSelectorBodyPreCoerce.Type - if mutSelectorBodyTy.IsGenericType && mutSelectorBodyTy.GetGenericTypeDefinition() = typedefof> then - mutInterimSelectorBodyPreCoerce - else - let mutSeqTy = MakeIEnumerableTy mutInterimVar.Type - Expr.Coerce (mutInterimSelectorBodyPreCoerce, mutSeqTy) + let mutInterimSelectorBody = CoerceToIEnumerable mutInterimVar.Type mutInterimSelectorBodyPreCoerce TransInnerResult.Other(MakeSelectMany(qTyIsIQueryable qTy, mutElemTy, CommitTransInnerResult mutSource, mutSelectorVar, mutInterimSelectorBody, mutInterimVar, mutTargetSelector)), selectorConv - TransFor mutSource immutSelector + TransFor (mutSource, sourceConv, immutSelectorVar) immutSelector // These occur in the F# quotation form of F# sequence expressions | CallWhere (_, [_; qTy], immutSource, Lambda(immutSelectorVar, immutPredicateBody)) -> @@ -1480,81 +1613,9 @@ module Query = | _ -> GroupingConv (immutKeySelector.Type, immutElementSelector.Type, selectorConv) TransInnerResult.Other(MakeGroupValBy(qTyIsIQueryable qTy, mutVar1.Type, mutKeySelector.Type, mutElementSelector.Type, mutSource, mutVar2, mutKeySelector, mutVar1, mutElementSelector)), conv - | CallJoin(_, GenericArgs [_; qTy; _; _; _], - [ immutOuterSource - immutInnerSource - Lambda(immutOuterKeyVar, immutOuterKeySelector) - Lambda(immutInnerKeyVar, immutInnerKeySelector) - LambdasNoDetupling([immutOuterResultGroupVar; immutInnerResultKeyVar], immutElementSelector)]) -> - - let (mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar: Var, mutOuterKeySelector, mutInnerKeyVar: Var, mutInnerKeySelector: Expr) = - TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) - - let mutOuterResultVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (immutOuterResultGroupVar, MacroExpand immutElementSelector) - let mutInnerResultKeyVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer innerSourceConv (immutInnerResultKeyVar, mutElementSelector) - let mutElementSelector, elementSelectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector - let mutElementSelector = CleanupLeaf mutElementSelector - - let joinExpr = - MakeJoin - (qTyIsIQueryable qTy, mutOuterKeyVar.Type, mutInnerKeyVar.Type, mutInnerKeySelector.Type, - mutElementSelector.Type, mutOuterSource, mutInnerSource, mutOuterKeyVar, mutOuterKeySelector, - mutInnerKeyVar, mutInnerKeySelector, mutOuterResultVar, mutInnerResultKeyVar, mutElementSelector) - - TransInnerResult.Other joinExpr, elementSelectorConv - - | CallGroupJoin - (_, GenericArgs [_; qTy; _; _; _], - [ immutOuterSource - immutInnerSource - Lambda(immutOuterKeyVar, immutOuterKeySelector) - Lambda(immutInnerKeyVar, immutInnerKeySelector) - LambdasNoDetupling([immutOuterResultGroupVar; immutInnerResultGroupVar], immutElementSelector)]) -> - - let (mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar: Var, mutOuterKeySelector, mutInnerKeyVar: Var, mutInnerKeySelector: Expr) = - TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) - - let mutOuterResultGroupVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (immutOuterResultGroupVar, MacroExpand immutElementSelector) - let innerGroupConv = MakeSeqConv innerSourceConv - let mutInnerResultKeyVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer innerGroupConv (immutInnerResultGroupVar, mutElementSelector) - let mutElementSelector, elementSelectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector - let mutElementSelector = CleanupLeaf mutElementSelector - - let joinExpr = - MakeGroupJoin - (qTyIsIQueryable qTy, mutOuterKeyVar.Type, mutInnerKeyVar.Type, - mutInnerKeySelector.Type, mutElementSelector.Type, mutOuterSource, - mutInnerSource, mutOuterKeyVar, mutOuterKeySelector, mutInnerKeyVar, - mutInnerKeySelector, mutOuterResultGroupVar, mutInnerResultKeyVar, mutElementSelector) - - TransInnerResult.Other joinExpr, elementSelectorConv - - | CallLeftOuterJoin - (_, GenericArgs [ _; qTy; immutInnerSourceTy; _; _], - [ immutOuterSource - immutInnerSource - Lambda(immutOuterKeyVar, immutOuterKeySelector) - Lambda(immutInnerKeyVar, immutInnerKeySelector) - LambdasNoDetupling([immutOuterResultGroupVar; immutInnerResultGroupVar], immutElementSelector)]) -> - - // Replace uses of 'innerResultGroupVar' with 'innerResultGroupVar.DefaultIfEmpty()' and call MakeGroupJoin - let immutElementSelector = immutElementSelector.Substitute (fun v -> if v = immutInnerResultGroupVar then Some (MakeDefaultIfEmpty ([immutInnerSourceTy], [Expr.Var immutInnerResultGroupVar])) else None) - - let (mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar: Var, mutOuterKeySelector, mutInnerKeyVar: Var, mutInnerKeySelector: Expr) = - TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) - - let mutOuterResultGroupVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (immutOuterResultGroupVar, MacroExpand immutElementSelector) - let mutInnerResultKeyVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer innerSourceConv (immutInnerResultGroupVar, mutElementSelector) - let mutElementSelector, elementSelectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector - let mutElementSelector = CleanupLeaf mutElementSelector - - let joinExpr = - MakeGroupJoin - (qTyIsIQueryable qTy, mutOuterKeyVar.Type, mutInnerKeyVar.Type, mutInnerKeySelector.Type, - mutElementSelector.Type, mutOuterSource, mutInnerSource, mutOuterKeyVar, mutOuterKeySelector, - mutInnerKeyVar, mutInnerKeySelector, mutOuterResultGroupVar, mutInnerResultKeyVar, mutElementSelector) - - TransInnerResult.Other joinExpr, elementSelectorConv + | CallJoinLike join -> + let mutOuterSource, outerSourceConv = TransInnerAndCommit CanEliminate.Yes check join.OuterSource + TransJoin check (qTyIsIQueryable join.QTy) (mutOuterSource, outerSourceConv) join | LetExprReduction reduced -> TransInner canElim check reduced @@ -1637,19 +1698,66 @@ module Query = | _ -> e, NoConv - and TransJoinInputs check (immutOuterSource, immutInnerSource, immutOuterKeyVar, immutOuterKeySelector, immutInnerKeyVar, immutInnerKeySelector) = + and TransJoin check isIQ (mutOuterSource, outerSourceConv) (join: JoinLike) = - let mutOuterSource, outerSourceConv = TransInnerAndCommit CanEliminate.Yes check immutOuterSource - let mutInnerSource, innerSourceConv = TransInnerAndCommit CanEliminate.Yes check immutInnerSource - let mutOuterKeyVar, mutOuterKeySelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (immutOuterKeyVar, MacroExpand immutOuterKeySelector) - let mutInnerKeyVar, mutInnerKeySelector = ConvertImmutableConsumerToMutableConsumer innerSourceConv (immutInnerKeyVar, MacroExpand immutInnerKeySelector) + let mutInnerSource, innerSourceConv = TransInnerAndCommit CanEliminate.Yes check join.InnerSource + let mutOuterKeyVar, mutOuterKeySelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (join.OuterKeyVar, MacroExpand join.OuterKeySelector) + let mutInnerKeyVar, mutInnerKeySelector = ConvertImmutableConsumerToMutableConsumer innerSourceConv (join.InnerKeyVar, MacroExpand join.InnerKeySelector) // Keys may be composite tuples - convert them to be mutables. Note, if there is a tuple on one side, there must be a tuple on the other side. let mutOuterKeySelector, _ = ProduceMoreMutables TransNone mutOuterKeySelector let mutOuterKeySelector = CleanupLeaf mutOuterKeySelector let mutInnerKeySelector, _ = ProduceMoreMutables TransNone mutInnerKeySelector let mutInnerKeySelector = CleanupLeaf mutInnerKeySelector - mutOuterSource, outerSourceConv, mutInnerSource, innerSourceConv, mutOuterKeyVar, mutOuterKeySelector, mutInnerKeyVar, mutInnerKeySelector + + let mutOuterResultVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer outerSourceConv (join.OuterResultVar, MacroExpand join.ElementSelector) + let mutInnerResultVar, mutElementSelector = ConvertImmutableConsumerToMutableConsumer (join.InnerResultConv innerSourceConv) (join.InnerResultVar, mutElementSelector) + let mutElementSelector, elementSelectorConv = ProduceMoreMutables TransInnerNoCheck mutElementSelector + let mutElementSelector = CleanupLeaf mutElementSelector + + let joinExpr = + join.Make + (isIQ, mutOuterKeyVar.Type, mutInnerKeyVar.Type, mutInnerKeySelector.Type, + mutElementSelector.Type, mutOuterSource, mutInnerSource, mutOuterKeyVar, mutOuterKeySelector, + mutInnerKeyVar, mutInnerKeySelector, mutOuterResultVar, mutInnerResultVar, mutElementSelector) + + TransInnerResult.Other joinExpr, elementSelectorConv + + /// source.SelectMany(fun sourceVar -> inner, (fun sourceVar innerVar -> (sourceVar, innerVar))), where 'inner' may use 'sourceVar'. + /// The pairs are consumed through a variable of an immutable tuple type, and the returned function rewrites a consumer + /// of 'sourceVar' and of a variable standing for 'innerVar' into a consumer of that pair. + and TransPairs check (mutSource, sourceConv, immutSourceVar: Var) (immutInnerSource, immutInnerVar: Var) = + let mutSourceVar, immutInnerSource = ConvertImmutableConsumerToMutableConsumer sourceConv (immutSourceVar, MacroExpand immutInnerSource) + let mutInnerSource, innerConv = TransInnerAndCommit CanEliminate.Yes check immutInnerSource + let mutInnerVar = Var (immutInnerVar.Name, ConvImmutableTypeToMutableType innerConv immutInnerVar.Type) + + // The elements of a tuple go into the pair one by one, as they do after two 'for's, so a provider can drop the ones + // nothing reads; EF Core cannot keep the group of a 'leftOuterJoin' that a whole tuple would carry along. + let sourceIsStruct, sourceElementConvs, immutSourceElementTys, mutSourceElements = + match sourceConv with + | TupleConv (isStruct, convs) -> + ValueSome isStruct, convs, FSharpType.GetTupleElements immutSourceVar.Type |> List.ofArray, + convs |> List.mapi (fun i _ -> AnonymousObjectGet (Expr.Var mutSourceVar, i)) + | _ -> ValueNone, [ sourceConv ], [ immutSourceVar.Type ], [ Expr.Var mutSourceVar ] + + let mutPair = CleanupLeaf (NewAnonymousObject (mutSourceElements @ [ Expr.Var mutInnerVar ])) + let mutPairs = MakeSelectMany (true, mutPair.Type, CommitTransInnerResult mutSource, mutSourceVar, CoerceToIEnumerable mutInnerVar.Type mutInnerSource, mutInnerVar, mutPair) + let immutPairVar = Var ("pair", FSharpType.MakeTupleType (Array.ofList (immutSourceElementTys @ [ immutInnerVar.Type ]))) + + let sourceElementCount = immutSourceElementTys.Length + let immutSourceFromPair = + let elements = List.init sourceElementCount (fun i -> Expr.TupleGet (Expr.Var immutPairVar, i)) + match sourceIsStruct with + | ValueSome true -> Expr.NewStructTuple elements + | ValueSome false -> Expr.NewTuple elements + | ValueNone -> elements.Head + let readFromPair (immutSecondVar: Var) (e: Expr) = + e.Substitute (fun v -> + if v = immutSourceVar then Some immutSourceFromPair + elif v = immutSecondVar then Some (Expr.TupleGet (Expr.Var immutPairVar, sourceElementCount)) + else None) + + mutPairs, TupleConv (false, sourceElementConvs @ [ innerConv ]), immutPairVar, readFromPair /// Given a query expression in terms of query.For, query.Select, query.Yield, query.Where etc., /// and including immutable tuples and immutable records, build an equivalent query expression diff --git a/src/FSharp.Core/QueryExtensions.fs b/src/FSharp.Core/QueryExtensions.fs index eafdae2d187..18edd4b9b5b 100644 --- a/src/FSharp.Core/QueryExtensions.fs +++ b/src/FSharp.Core/QueryExtensions.fs @@ -108,14 +108,6 @@ module internal Adapters = let anonObjectTypes = tupleTypes |> Array.map snd - let tupleToAnonTypeMap = - let t = Dictionary() - - for k, v in tupleTypes do - t.[k] <- v - - t - let anonToTupleTypeMap = let t = Dictionary() @@ -173,18 +165,13 @@ module internal Adapters = walk i e e.Type + /// Reference and struct tuples of the same arity share one anonymous object type. let RewriteTupleType (ty: Type) conv = - // Tuples are generic, so lookup only for generic types - assert ty.IsGenericType - let generic = ty.GetGenericTypeDefinition() - - match tupleToAnonTypeMap.TryGetValue generic with - | true, mutableTupleType -> - // Recursively transform type arguments - mutableTupleType.MakeGenericType(ty.GetGenericArguments() |> Array.toList |> conv |> Array.ofList) - | _ -> - assert false - failwith "unreachable" + assert (FSharpType.IsTuple ty) + let elementTypes = ty.GetGenericArguments() + let mutableTupleType = anonObjectTypes.[elementTypes.Length - 1] + // Recursively transform type arguments + mutableTupleType.MakeGenericType(elementTypes |> Array.toList |> conv |> Array.ofList) let (|RecordFieldGetSimplification|_|) (expr: Expr) = match expr with @@ -206,7 +193,7 @@ module internal Adapters = /// that we can invert the process in final query results. [] type ConversionDescription = - | TupleConv of ConversionDescription list + | TupleConv of isStruct: bool * ConversionDescription list | RecordConv of Type * ConversionDescription list | GroupingConv (* origKeyType: *) of Type (* origElemType: *) * Type * ConversionDescription | SeqConv of ConversionDescription @@ -216,18 +203,18 @@ module internal Adapters = /// "yield" or "select", convert it to a type involving anonymous objects according to the conversion data. let rec ConvImmutableTypeToMutableType conv ty = match conv with - | TupleConv convs -> + | TupleConv(isStruct, convs) -> assert (FSharpType.IsTuple ty) match convs with | x1 :: x2 :: x3 :: x4 :: x5 :: x6 :: x7 :: x8 :: tail -> - let els = [ x1; x2; x3; x4; x5; x6; x7; TupleConv(x8 :: tail) ] + let els = [ x1; x2; x3; x4; x5; x6; x7; TupleConv(isStruct, x8 :: tail) ] RewriteTupleType ty (List.map2 ConvImmutableTypeToMutableType els) | _ -> RewriteTupleType ty (List.map2 ConvImmutableTypeToMutableType convs) | RecordConv(_, convs) -> assert (isPartiallyImmutableRecord ty) let types = [| for f in FSharpType.GetRecordFields ty -> f.PropertyType |] - ConvImmutableTypeToMutableType (TupleConv convs) (FSharpType.MakeTupleType types) + ConvImmutableTypeToMutableType (TupleConv(false, convs)) (FSharpType.MakeTupleType types) | GroupingConv(_keyTy, _elemTy, conv) -> assert ty.IsGenericType assert (ty.GetGenericTypeDefinition() = typedefof>) @@ -324,7 +311,7 @@ module internal Adapters = let argExprsNow, argScripts = exprs |> List.map (ProduceMoreMutables tipf) |> List.unzip - NewAnonymousObject argExprsNow, TupleConv argScripts + NewAnonymousObject argExprsNow, TupleConv(expr.Type.IsValueType, argScripts) // Replace immutable records by anonymous objects | Patterns.NewRecord(typ, args) when isPartiallyImmutableRecord typ -> diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj b/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj index 494509d0906..3a20b60f7e4 100644 --- a/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj +++ b/tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj @@ -92,6 +92,15 @@ + + + + + + + + + diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs new file mode 100644 index 00000000000..03011a4d99f --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryKeyScenarios.fs @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.QueryKeyScenarios + +open System.Linq +open System.Linq.Expressions +open Microsoft.FSharp.Linq.RuntimeHelpers + +let private block = [ Violation.Statement ExpressionType.Block ] + +let private customers (data: Data) = Seq.ofList data.Customers + +/// Orders paired with the lines that share both the order number and, as quantity, the customer number. +let private matchingLines (data: Data) = + seq { + for o in data.Orders do + for l in data.Lines do + if l.OrderId = o.Id && l.Quantity = o.CustomerId then + yield o.Id, l.Id + } + +let private matchingLinesOrNone (data: Data) = + seq { + for o in data.Orders do + match matchingLines data |> Seq.filter (fun (order, _) -> order = o.Id) |> List.ofSeq with + | [] -> yield o.Id, -1 + | matches -> yield! matches + } + +let private joins () = + [ Scenario.rows + "join on a scalar" + (fun db -> query { for c in db.Customers do join o in db.Orders on (c.Id = o.CustomerId); select (c.Name, o.Id) }) + (fun data -> seq { for c in data.Customers do for o in c.Orders -> c.Name, o.Id }) + + Scenario.rows + "join on a tuple" + (fun db -> + query { for o in db.Orders do join l in db.Lines on ((o.Id, o.CustomerId) = (l.OrderId, l.Quantity)); select (o.Id, l.Id) }) + matchingLines + + Scenario.rows + "join on a struct tuple" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on (struct (o.Id, o.CustomerId) = struct (l.OrderId, l.Quantity)) + select (o.Id, l.Id) + }) + matchingLines + + Scenario.rows + "join on a record, fields in declaration order" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on ({ OrderNumber = o.Id; Units = o.CustomerId } = { OrderNumber = l.OrderId; Units = l.Quantity }) + select (o.Id, l.Id) + }) + matchingLines + + Scenario.rows + "join on a record, fields out of declaration order" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on ({ Units = o.CustomerId; OrderNumber = o.Id } = { Units = l.Quantity; OrderNumber = l.OrderId }) + select (o.Id, l.Id) + }) + matchingLines + |> Scenario.violating block + + Scenario.rows + "join on an anonymous record, fields in alphabetical order" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on ({| Order = o.Id; Units = o.CustomerId |} = {| Order = l.OrderId; Units = l.Quantity |}) + select (o.Id, l.Id) + }) + matchingLines + + Scenario.rows + "join on an anonymous record, fields out of alphabetical order" + (fun db -> + query { + for o in db.Orders do + join l in db.Lines on ({| Units = o.CustomerId; Order = o.Id |} = {| Units = l.Quantity; Order = l.OrderId |}) + select (o.Id, l.Id) + }) + matchingLines + |> Scenario.violating block + + Scenario.rows + "groupJoin on a tuple" + (fun db -> + query { + for o in db.Orders do + groupJoin l in db.Lines on ((o.Id, o.CustomerId) = (l.OrderId, l.Quantity)) into lines + select (o.Id, lines.Count()) + }) + (fun data -> seq { for o in data.Orders -> o.Id, matchingLines data |> Seq.filter (fun (order, _) -> order = o.Id) |> Seq.length }) + + Scenario.rows + "leftOuterJoin on a tuple" + (fun db -> + query { + for o in db.Orders do + leftOuterJoin l in db.Lines on ((o.Id, o.CustomerId) = (l.OrderId, l.Quantity)) into lines + for l in lines do + select (o.Id, (if obj.ReferenceEquals(l, null) then -1 else l.Id)) + }) + matchingLinesOrNone ] + +let private groupings () = + [ Scenario.rows + "groupBy a scalar" + (fun db -> query { for c in db.Customers do groupBy c.City into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy _.City) + + Scenario.rows + "groupBy a scalar, groups themselves" + (fun db -> query { for c in db.Customers do groupBy c.City }) + (customers >> Seq.groupBy _.City >> Seq.map (fun (city, group) -> Grouping(city, group) :> IGrouping<_, _>)) + + Scenario.rows + "groupBy a tuple" + (fun db -> query { for c in db.Customers do groupBy (c.City, c.Age > 28) into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> c.City, c.Age > 28)) + + Scenario.rows + "groupBy a tuple, element of the key" + (fun db -> query { for c in db.Customers do groupBy (c.City, c.Age > 28) into g; select (fst g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> c.City, c.Age > 28) >> Seq.map (fun ((city, _), count) -> city, count)) + |> Scenario.violating [ Violation.RuntimeHelperCall "Operators.Fst" ] + + Scenario.rows + "groupBy a struct tuple" + (fun db -> query { for c in db.Customers do groupBy (struct (c.City, c.Age > 28)) into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> struct (c.City, c.Age > 28))) + + Scenario.rows + "groupBy a record, fields in declaration order" + (fun db -> query { for c in db.Customers do groupBy { KeyCity = c.City; KeyAge = c.Age / 10 } into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> { KeyCity = c.City; KeyAge = c.Age / 10 })) + + Scenario.rows + "groupBy a record, fields out of declaration order" + (fun db -> query { for c in db.Customers do groupBy { KeyAge = c.Age / 10; KeyCity = c.City } into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> { KeyCity = c.City; KeyAge = c.Age / 10 })) + |> Scenario.violating block + + Scenario.rows + "groupBy an anonymous record, fields in alphabetical order" + (fun db -> query { for c in db.Customers do groupBy {| City = c.City; Decade = c.Age / 10 |} into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> {| City = c.City; Decade = c.Age / 10 |})) + + Scenario.rows + "groupBy an anonymous record, fields out of alphabetical order" + (fun db -> query { for c in db.Customers do groupBy {| Decade = c.Age / 10; City = c.City |} into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy (fun c -> {| City = c.City; Decade = c.Age / 10 |})) + |> Scenario.violating block + + Scenario.rows + "groupValBy a tuple of values" + (fun db -> query { for c in db.Customers do groupValBy (c.Name, c.Age) c.City into g; select (g.Key, g.Count()) }) + (customers >> Seq.countBy _.City) + |> Scenario.violating [ Violation.RuntimeHelperCall "new Grouping`2" ] ] + +let cases () = joins () @ groupings () diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs new file mode 100644 index 00000000000..5325268456b --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryMethodSyntaxScenarios.fs @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.QueryMethodSyntaxScenarios + +open System.Linq +open System.Linq.Expressions + +let private block = [ Violation.Statement ExpressionType.Block ] + +let private reading typeName = + [ Violation.ReadsUnnamedMember typeName ] + +let private customers (data: Data) = Seq.ofList data.Customers + +let cases () = + [ Scenario.rows "Select a tuple" (fun db -> db.Customers.Select(fun c -> (c.Id, c.Name))) (customers >> Seq.map (fun c -> c.Id, c.Name)) + + Scenario.rows + "Select a struct tuple" + (fun db -> db.Customers.Select(fun c -> struct (c.Id, c.Name))) + (customers >> Seq.map (fun c -> struct (c.Id, c.Name))) + + Scenario.rows + "Where over a query built earlier" + (fun db -> + let withLargeOrders = db.Orders.Where(fun o -> o.Total > 4.0).Select(fun o -> o.CustomerId) + db.Customers.Where(fun c -> withLargeOrders.Contains c.Id).Select(fun c -> c.Name)) + (customers >> Seq.filter (fun c -> c.Orders |> Seq.exists (fun o -> o.Total > 4.0)) >> Seq.map _.Name) + + Scenario.rows + "Where over a query built earlier with a let" + (fun db -> + let withLargeOrders = db.Orders.Where(fun o -> let total = o.Total in total > 4.0).Select(fun o -> o.CustomerId) + db.Customers.Where(fun c -> withLargeOrders.Contains c.Id).Select(fun c -> c.Name)) + (customers >> Seq.filter (fun c -> c.Orders |> Seq.exists (fun o -> o.Total > 4.0)) >> Seq.map _.Name) + |> Scenario.violating block + + Scenario.rows + "Select a record, fields in declaration order" + (fun db -> db.Customers.Select(fun c -> { PersonName = c.Name; PersonAge = c.Age })) + (customers >> Seq.map (fun c -> { PersonName = c.Name; PersonAge = c.Age })) + + Scenario.rows + "Select a record, fields out of declaration order" + (fun db -> db.Customers.Select(fun c -> { PersonAge = c.Age; PersonName = c.Name })) + (customers >> Seq.map (fun c -> { PersonName = c.Name; PersonAge = c.Age })) + |> Scenario.violating block + + Scenario.rows + "Select an anonymous record, fields in alphabetical order" + (fun db -> db.Customers.Select(fun c -> {| Age = c.Age; Name = c.Name |})) + (customers >> Seq.map (fun c -> {| Age = c.Age; Name = c.Name |})) + + Scenario.rows + "Select an anonymous record, fields out of alphabetical order" + (fun db -> db.Customers.Select(fun c -> {| Name = c.Name; Age = c.Age |})) + (customers >> Seq.map (fun c -> {| Age = c.Age; Name = c.Name |})) + |> Scenario.violating block + + Scenario.rows + "Select with a let used twice" + (fun db -> db.Customers.Select(fun c -> let upper = c.Name.ToUpper() in upper + upper)) + (customers >> Seq.map (fun c -> c.Name.ToUpper() + c.Name.ToUpper())) + |> Scenario.violating block + + Scenario.rows + "Where taking a tuple apart" + (fun db -> db.Customers.Select(fun c -> (c.Id, c.Name)).Where(fun (id, _) -> id > 1)) + (customers >> Seq.map (fun c -> c.Id, c.Name) >> Seq.filter (fun (id, _) -> id > 1)) + |> Scenario.violating (reading "Tuple`2" @ block) + + Scenario.rows + "Where taking a struct tuple apart" + (fun db -> db.Customers.Select(fun c -> struct (c.Id, c.Name)).Where(fun (struct (id, _)) -> id > 1)) + (customers >> Seq.map (fun c -> struct (c.Id, c.Name)) >> Seq.filter (fun (struct (id, _)) -> id > 1)) + |> Scenario.violating (reading "ValueTuple`2" @ block) + + Scenario.rows + "Where taking the last of nine apart" + (fun db -> + db.Customers + .Select(fun c -> (c.Id + 100, c.Name, c.City, c.Age, c.Id * 7, c.Name.Length, c.City.Length, c.Age * 2, c.Id)) + .Where(fun (_, _, _, _, _, _, _, _, last) -> last > 1) + .Select(fun (_, name, _, _, _, _, _, _, _) -> name)) + (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) + |> Scenario.violating (reading "Tuple`8" @ block) + + Scenario.rows + "Where taking the last of a struct nine apart" + (fun db -> + db.Customers + .Select(fun c -> struct (c.Id + 100, c.Name, c.City, c.Age, c.Id * 7, c.Name.Length, c.City.Length, c.Age * 2, c.Id)) + .Where(fun (struct (_, _, _, _, _, _, _, _, last)) -> last > 1) + .Select(fun (struct (_, name, _, _, _, _, _, _, _)) -> name)) + (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) + |> Scenario.violating (reading "ValueTuple`8" @ block) + + Scenario.rows + "Where over a query that selects a tuple" + (fun db -> (query { for c in db.Customers do select (c.Id, c.Name) }).Where(fun (id, _) -> id > 1)) + (customers >> Seq.map (fun c -> c.Id, c.Name) >> Seq.filter (fun (id, _) -> id > 1)) + |> Scenario.violating (reading "Tuple`2" @ block) + + Scenario.rows + "Where on a nullable property" + (fun db -> db.Customers.Where(fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020)) + (customers >> Seq.filter (fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020)) + |> Scenario.violating block + + Scenario.rows + "Where with a lambda over another source" + (fun db -> db.Customers.Where(fun c -> db.Orders.Any(fun o -> o.CustomerId = c.Id))) + (customers >> Seq.filter (fun c -> c.Orders.Count > 0)) + |> Scenario.violating + [ Violation.RuntimeHelperCall "LeafExpressionConverter.QuotationToLambdaExpression" + Violation.RuntimeHelperCall "LeafExpressionConverter.SubstHelper" ] + + Scenario.rows + "OrderBy, ThenByDescending" + (fun db -> db.Customers.OrderBy(fun c -> c.City).ThenByDescending(fun c -> c.Age).Select(fun c -> c.Name)) + (customers >> Seq.sortBy (fun c -> c.City, -c.Age) >> Seq.map _.Name) + |> Scenario.ordered + + Scenario.rows + "GroupBy a tuple" + (fun db -> db.Customers.GroupBy(fun c -> (c.City, c.Age > 28)).Select(fun g -> (g.Key, g.Count()))) + (customers >> Seq.countBy (fun c -> c.City, c.Age > 28)) + + Scenario.rows + "SelectMany over a navigation collection" + (fun db -> db.Customers.SelectMany(fun c -> c.Orders :> seq).Select(fun o -> o.Id)) + (fun data -> data.Orders |> Seq.map _.Id) ] diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs new file mode 100644 index 00000000000..6b2339a996b --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryOperatorScenarios.fs @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.QueryOperatorScenarios + +open System +open System.Linq +open Microsoft.FSharp.Linq +open Microsoft.FSharp.Linq.NullableOperators + +let private customers (data: Data) = Seq.ofList data.Customers + +let private lines (data: Data) = Seq.ofList data.Lines + +let private discountOrLowest (line: OrderLine) = + if line.Discount.HasValue then line.Discount.Value else Int32.MinValue + +let private discounts (data: Data) = + data.Lines |> List.filter _.Discount.HasValue |> List.map _.Discount.Value + +let private sequences () = + [ Scenario.rows + "where" + (fun db -> query { for c in db.Customers do where (c.Age > 26); select c.Name }) + (customers >> Seq.filter (fun c -> c.Age > 26) >> Seq.map _.Name) + + Scenario.rows + "where with string methods" + (fun db -> query { for c in db.Customers do where (c.Name.StartsWith "A" || c.City = "Paris"); select c.Name }) + (customers >> Seq.filter (fun c -> c.Name.StartsWith "A" || c.City = "Paris") >> Seq.map _.Name) + + Scenario.rows + "where on a nullable property" + (fun db -> query { for c in db.Customers do where (c.Registered.HasValue && c.Registered.Value.Year > 2020); select c.Name }) + (customers >> Seq.filter (fun c -> c.Registered.HasValue && c.Registered.Value.Year > 2020) >> Seq.map _.Name) + + Scenario.rows + "where with a nullable operator" + (fun db -> query { for l in db.Lines do where (l.Discount ?> 1); select l.Id }) + (lines >> Seq.filter (fun l -> l.Discount ?> 1) >> Seq.map _.Id) + + Scenario.rows + "if then" + (fun db -> query { for c in db.Customers do if c.Age > 26 then yield c.Name }) + (customers >> Seq.filter (fun c -> c.Age > 26) >> Seq.map _.Name) + + Scenario.rows + "sortBy, thenByDescending" + (fun db -> query { for c in db.Customers do sortBy c.City; thenByDescending c.Age; select c.Name }) + (customers >> Seq.sortBy (fun c -> c.City, -c.Age) >> Seq.map _.Name) + |> Scenario.ordered + |> Scenario.flat [ "OrderBy"; "ThenByDescending"; "Select" ] + + Scenario.rows + "sortByDescending, thenBy" + (fun db -> query { for c in db.Customers do sortByDescending c.City; thenBy c.Age; select c.Name }) + (customers >> Seq.sortBy _.Age >> Seq.sortByDescending _.City >> Seq.map _.Name) + |> Scenario.ordered + |> Scenario.flat [ "OrderByDescending"; "ThenBy"; "Select" ] + + Scenario.rows + "sortByNullable, thenByNullableDescending" + (fun db -> query { for l in db.Lines do sortByNullable l.Discount; thenByNullableDescending (Nullable l.Id); select l.Id }) + (lines >> Seq.sortBy (fun l -> discountOrLowest l, -l.Id) >> Seq.map _.Id) + |> Scenario.ordered + |> Scenario.flat [ "OrderBy"; "ThenByDescending"; "Select" ] + + Scenario.rows + "sortByNullableDescending, thenByNullable" + (fun db -> query { for l in db.Lines do sortByNullableDescending l.Discount; thenByNullable (Nullable l.Id); select l.Id }) + (lines >> Seq.sortBy (fun l -> -(int64 (discountOrLowest l)), l.Id) >> Seq.map _.Id) + |> Scenario.ordered + |> Scenario.flat [ "OrderByDescending"; "ThenBy"; "Select" ] + + Scenario.rows "distinct" (fun db -> query { for c in db.Customers do select c.City; distinct }) (customers >> Seq.map _.City >> Seq.distinct) + + Scenario.rows + "skip, take" + (fun db -> query { for c in db.Customers do sortBy c.Id; skip 1; take 2; select c.Name }) + (customers >> Seq.skip 1 >> Seq.take 2 >> Seq.map _.Name) + |> Scenario.ordered + + Scenario.rows + "skipWhile, takeWhile" + (fun db -> query { for c in db.Customers do sortBy c.Id; skipWhile (c.Id < 2); takeWhile (c.Id < 4); select c.Name }) + (customers >> Seq.skipWhile (fun c -> c.Id < 2) >> Seq.takeWhile (fun c -> c.Id < 4) >> Seq.map _.Name) + |> Scenario.ordered + + Scenario.rows + "join" + (fun db -> query { for o in db.Orders do join l in db.Lines on (o.Id = l.OrderId); select (o.Id, l.Id) }) + (fun data -> seq { for o in data.Orders do for l in o.Lines -> o.Id, l.Id }) + + Scenario.rows + "groupJoin" + (fun db -> query { for o in db.Orders do groupJoin l in db.Lines on (o.Id = l.OrderId) into lines; select (o.Id, lines.Count()) }) + (fun data -> seq { for o in data.Orders -> o.Id, o.Lines.Count }) + + Scenario.rows + "leftOuterJoin" + (fun db -> + query { + for o in db.Orders do + leftOuterJoin l in db.Lines on (o.Id = l.OrderId) into lines + for l in lines do + select (o.Id, (if obj.ReferenceEquals(l, null) then -1 else l.Id)) + }) + (fun data -> + seq { + for o in data.Orders do + if o.Lines.Count = 0 then yield o.Id, -1 else for l in o.Lines -> o.Id, l.Id + }) + + Scenario.rows + "leftOuterJoin tested with isNull" + (fun db -> + query { + for o in db.Orders do + leftOuterJoin l in db.Lines on (o.Id = l.OrderId) into lines + for l in lines do + select (o.Id, isNull l) + }) + (fun data -> + seq { + for o in data.Orders do + if o.Lines.Count = 0 then yield o.Id, true else for _ in o.Lines -> o.Id, false + }) + |> Scenario.violating [ Violation.RuntimeHelperCall "Operators.IsNull" ] ] + +let private values () = + [ Scenario.value "count" (fun db -> query { for c in db.Customers do where (c.Age > 26); count }) (fun _ -> 3) + + Scenario.value "exists" (fun db -> query { for c in db.Customers do exists (c.City = "Paris") }) (fun _ -> true) + + Scenario.value "all" (fun db -> query { for c in db.Customers do all (c.Age > 26) }) (fun _ -> false) + + Scenario.value "contains" (fun db -> query { for c in db.Customers do select c.Id; contains 2 }) (fun _ -> true) + + Scenario.value "find" (fun db -> query { for c in db.Customers do find (c.Age < 30) }) (fun data -> data.Customers.[1]) + + Scenario.value "head" (fun db -> query { for c in db.Customers do sortBy c.Id; head }) (fun data -> data.Customers.[0]) + + Scenario.value + "headOrDefault" + (fun db -> query { for c in db.Customers do where (c.Age > 100); headOrDefault }) + (fun _ -> null) + + Scenario.value "last" (fun db -> query { for c in db.Customers do sortBy c.Id; last }) (fun data -> data.Customers.[3]) + + Scenario.value + "lastOrDefault" + (fun db -> query { for c in db.Customers do where (c.Age > 100); lastOrDefault }) + (fun _ -> null) + + Scenario.value + "exactlyOne" + (fun db -> query { for c in db.Customers do where (c.City = "Paris"); exactlyOne }) + (fun data -> data.Customers.[1]) + + Scenario.value + "exactlyOneOrDefault" + (fun db -> query { for c in db.Customers do where (c.City = "Rome"); exactlyOneOrDefault }) + (fun _ -> null) + + Scenario.value "nth" (fun db -> query { for c in db.Customers do sortBy c.Id; nth 2 }) (fun data -> data.Customers.[2]) + + Scenario.value "minBy" (fun db -> query { for c in db.Customers do minBy c.Age }) (customers >> Seq.map _.Age >> Seq.min) + + Scenario.value "maxBy" (fun db -> query { for c in db.Customers do maxBy c.Age }) (customers >> Seq.map _.Age >> Seq.max) + + Scenario.value "sumBy" (fun db -> query { for c in db.Customers do sumBy c.Age }) (customers >> Seq.sumBy _.Age) + + Scenario.value + "averageBy" + (fun db -> query { for c in db.Customers do averageBy (float c.Age) }) + (customers >> Seq.averageBy (fun c -> float c.Age)) + + Scenario.value + "minByNullable" + (fun db -> query { for l in db.Lines do minByNullable l.Discount }) + (discounts >> List.min >> Nullable) + + Scenario.value + "maxByNullable" + (fun db -> query { for l in db.Lines do maxByNullable l.Discount }) + (discounts >> List.max >> Nullable) + + Scenario.value + "sumByNullable" + (fun db -> query { for l in db.Lines do sumByNullable l.Discount }) + (discounts >> List.sum >> Nullable) + + Scenario.value + "averageByNullable" + (fun db -> query { for l in db.Lines do averageByNullable (Nullable.float l.Discount) }) + (discounts >> List.averageBy float >> Nullable) ] + +let cases () = sequences () @ values () diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs new file mode 100644 index 00000000000..8c1e684614e --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProjectionScenarios.fs @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.QueryProjectionScenarios + +open System.Linq.Expressions + +let private block = [ Violation.Statement ExpressionType.Block ] + +let private customers (data: Data) = Seq.ofList data.Customers + +let cases () = + [ Scenario.rows "scalar" (fun db -> query { for c in db.Customers do select c.Name }) (customers >> Seq.map _.Name) + + Scenario.rows "entity" (fun db -> query { for c in db.Customers do select c }) customers + + Scenario.rows + "computed scalars" + (fun db -> query { for c in db.Customers do select (c.Name + "!", c.Age + 1, (if c.Age > 28 then "senior" else "junior")) }) + (customers >> Seq.map (fun c -> c.Name + "!", c.Age + 1, (if c.Age > 28 then "senior" else "junior"))) + + Scenario.rows + "tuple of two" + (fun db -> query { for c in db.Customers do select (c.Id, c.Name) }) + (customers >> Seq.map (fun c -> c.Id, c.Name)) + + Scenario.rows + "tuple of seven" + (fun db -> query { for c in db.Customers do select (c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2) }) + (customers >> Seq.map (fun c -> c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2)) + + Scenario.rows + "tuple of eight" + (fun db -> query { for c in db.Customers do select (c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7) }) + (customers >> Seq.map (fun c -> c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7)) + + Scenario.rows + "tuple of nine" + (fun db -> query { for c in db.Customers do select (c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7, c.Age - c.Id) }) + (customers >> Seq.map (fun c -> c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7, c.Age - c.Id)) + + Scenario.rows + "nested tuple" + (fun db -> query { for c in db.Customers do select ((c.Id, c.Name), c.Age) }) + (customers >> Seq.map (fun c -> (c.Id, c.Name), c.Age)) + + Scenario.rows + "struct tuple of two" + (fun db -> query { for c in db.Customers do select (struct (c.Id, c.Name)) }) + (customers >> Seq.map (fun c -> struct (c.Id, c.Name))) + + Scenario.rows + "struct tuple of nine" + (fun db -> + query { for c in db.Customers do select (struct (c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7, c.Age - c.Id)) }) + (customers >> Seq.map (fun c -> struct (c.Id, c.Name, c.City, c.Age, c.Id + 100, c.Name.Length, c.Age * 2, c.Id * 7, c.Age - c.Id))) + + Scenario.rows + "struct tuple inside a tuple" + (fun db -> query { for c in db.Customers do select (struct (c.Id, c.Name), c.Age) }) + (customers >> Seq.map (fun c -> struct (c.Id, c.Name), c.Age)) + + Scenario.rows + "tuple inside a struct tuple" + (fun db -> query { for c in db.Customers do select (struct ((c.Id, c.Name), c.Age)) }) + (customers >> Seq.map (fun c -> struct ((c.Id, c.Name), c.Age))) + + Scenario.rows + "tuple taken apart by the next for" + (fun db -> + query { + for (id, name) in (query { for c in db.Customers do select (c.Id, c.Name) }) do + where (id > 1) + select name + }) + (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) + |> Scenario.violating [ Violation.ReadsUnnamedMember "Tuple`2" ] + + Scenario.rows + "struct tuple taken apart by the next for" + (fun db -> + query { + for struct (id, name) in (query { for c in db.Customers do select (struct (c.Id, c.Name)) }) do + where (id > 1) + select name + }) + (customers >> Seq.filter (fun c -> c.Id > 1) >> Seq.map _.Name) + |> Scenario.violating [ Violation.ReadsUnnamedMember "ValueTuple`2" ] + + Scenario.rows + "record, fields in declaration order" + (fun db -> query { for c in db.Customers do select { PersonName = c.Name; PersonAge = c.Age } }) + (customers >> Seq.map (fun c -> { PersonName = c.Name; PersonAge = c.Age })) + + Scenario.rows + "record, fields out of declaration order" + (fun db -> query { for c in db.Customers do select { PersonAge = c.Age; PersonName = c.Name } }) + (customers >> Seq.map (fun c -> { PersonName = c.Name; PersonAge = c.Age })) + |> Scenario.violating block + + Scenario.rows + "record of three, fields rotated" + (fun db -> query { for c in db.Customers do select { Town = c.City; FullName = c.Name; Years = c.Age } }) + (customers >> Seq.map (fun c -> { FullName = c.Name; Years = c.Age; Town = c.City })) + |> Scenario.violating block + + Scenario.rows + "record holding a tuple" + (fun db -> query { for c in db.Customers do select {| Age = c.Age; Pair = (c.Id, c.Name) |} }) + (customers >> Seq.map (fun c -> {| Age = c.Age; Pair = (c.Id, c.Name) |})) + + Scenario.rows + "anonymous record, fields in alphabetical order" + (fun db -> query { for c in db.Customers do select {| Age = c.Age; Name = c.Name |} }) + (customers >> Seq.map (fun c -> {| Age = c.Age; Name = c.Name |})) + + Scenario.rows + "anonymous record, fields out of alphabetical order" + (fun db -> query { for c in db.Customers do select {| Name = c.Name; Age = c.Age |} }) + (customers >> Seq.map (fun c -> {| Age = c.Age; Name = c.Name |})) + |> Scenario.violating block + + Scenario.rows + "anonymous record, method calls in fields out of order" + (fun db -> query { for c in db.Customers do select {| Upper = c.Name.ToUpper(); City = c.City.Trim() |} }) + (customers >> Seq.map (fun c -> {| City = c.City.Trim(); Upper = c.Name.ToUpper() |})) + |> Scenario.violating block + + Scenario.rows + "anonymous record inside an anonymous record" + (fun db -> query { for c in db.Customers do select {| Person = {| Name = c.Name; Age = c.Age |} |} }) + (customers >> Seq.map (fun c -> {| Person = {| Age = c.Age; Name = c.Name |} |})) + |> Scenario.violating block + + Scenario.rows + "struct anonymous record, fields in alphabetical order" + (fun db -> query { for c in db.Customers do select (struct {| Age = c.Age; Name = c.Name |}) }) + (customers >> Seq.map (fun c -> struct {| Age = c.Age; Name = c.Name |})) + + Scenario.rows + "struct anonymous record, fields out of alphabetical order" + (fun db -> query { for c in db.Customers do select (struct {| Name = c.Name; Age = c.Age |}) }) + (customers >> Seq.map (fun c -> struct {| Age = c.Age; Name = c.Name |})) + |> Scenario.violating block + + Scenario.rows + "CLIMutable record, fields in declaration order" + (fun db -> query { for c in db.Customers do select { DtoName = c.Name; DtoAge = c.Age } }) + (customers >> Seq.map (fun c -> { DtoName = c.Name; DtoAge = c.Age })) + + Scenario.rows + "CLIMutable record, fields out of declaration order" + (fun db -> query { for c in db.Customers do select { DtoAge = c.Age; DtoName = c.Name } }) + (customers >> Seq.map (fun c -> { DtoName = c.Name; DtoAge = c.Age })) + |> Scenario.violating block + + Scenario.rows + "class with property initializers" + (fun db -> query { for c in db.Customers do select (CustomerView(ViewName = c.Name, ViewAge = c.Age)) }) + (customers >> Seq.map (fun c -> CustomerView(ViewName = c.Name, ViewAge = c.Age))) + + Scenario.rows "option" (fun db -> query { for c in db.Customers do select (Some c.Age) }) (customers >> Seq.map (fun c -> Some c.Age)) + |> Scenario.violating [ Violation.RuntimeHelperCall "FSharpOption`1.Some" ] ] diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs new file mode 100644 index 00000000000..bad748d75ab --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryProviderShape.fs @@ -0,0 +1,355 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Core.UnitTests.Linq + +open System +open System.Collections +open System.Collections.Generic +open System.Linq +open System.Linq.Expressions +open System.Reflection +open Microsoft.FSharp.Reflection + +type ICapturedQuery = + abstract InMemory: IQueryable + +// EnumerableQuery turns Queryable calls into Enumerable ones only over its own constants; +// over any other IQueryable it calls Queryable again, which here would never end. +type private InMemoryRewriter() = + inherit ExpressionVisitor() + + override _.VisitConstant node = + match node.Value with + | :? ICapturedQuery as captured -> Expression.Constant captured.InMemory + | _ -> node + +/// An in-memory query that stays on the provider that created it and prints its root by name. +[] +type CapturedQuery<'T>(provider: IQueryProvider, name: string, items: seq<'T>, derived: Expression voption) = + + let enumerate () = + match derived with + | ValueSome expression -> provider.Execute>(expression).GetEnumerator() + | ValueNone -> items.GetEnumerator() + + override _.ToString() = + match derived with + | ValueSome expression -> expression.ToString() + | ValueNone -> name + + interface ICapturedQuery with + member _.InMemory = + match derived with + | ValueSome expression -> EnumerableQuery<'T>(InMemoryRewriter().Visit expression) + | ValueNone -> EnumerableQuery<'T> items + + interface IOrderedQueryable<'T> with + member this.Expression = + match derived with + | ValueSome expression -> expression + | ValueNone -> Expression.Constant this + + member _.ElementType = typeof<'T> + member _.Provider = provider + member _.GetEnumerator() : IEnumerator<'T> = enumerate () + member _.GetEnumerator() : IEnumerator = enumerate () + +/// Executes in memory, and records what it was asked to execute. +[] +type CapturingQueryProvider() = + let executed = ResizeArray() + let mutable depth = 0 + + let inMemory () = + EnumerableQuery Seq.empty :> IQueryProvider + + let track (expression: Expression) (run: Expression -> 'TResult) = + if depth = 0 then + executed.Add expression + + depth <- depth + 1 + + try + run (InMemoryRewriter().Visit expression) + finally + depth <- depth - 1 + + /// The expressions executed directly, not from inside another execution. + member _.Executed = executed :> IReadOnlyList + + member this.Source(name: string, items: seq<'T>) : IQueryable<'T> = + CapturedQuery<'T>(this, name, items, ValueNone) + + member this.Derived<'T>(expression: Expression) : IQueryable<'T> = + CapturedQuery<'T>(this, "", Seq.empty, ValueSome expression) + + interface IQueryProvider with + member this.CreateQuery<'T>(expression: Expression) : IQueryable<'T> = this.Derived<'T> expression + + member this.CreateQuery(expression: Expression) : IQueryable = + let elementType = + expression.Type.GetInterfaces() + |> Array.append [| expression.Type |] + |> Array.find (fun ty -> ty.IsGenericType && ty.GetGenericTypeDefinition() = typedefof>) + |> fun ty -> ty.GetGenericArguments().[0] + + typeof + .GetMethod(nameof Unchecked.defaultof.Derived) + .MakeGenericMethod(elementType) + .Invoke(this, [| expression |]) + :?> IQueryable + + member _.Execute<'TResult>(expression: Expression) : 'TResult = + track expression (inMemory ()).Execute<'TResult> + + member _.Execute(expression: Expression) : obj = + track expression (inMemory ()).Execute + +/// Something in an expression tree that a C# expression lambda never contains, so no LINQ provider translates it. +[] +type Violation = + | Statement of ExpressionType + | LambdaInvocation + | UnboundParameter of name: string + | RuntimeHelperCall of methodName: string + | NestedInCollectionSelector of methodName: string + | ReadsUnnamedMember of typeName: string + | LeftProvider + | Throws of exceptionType: string + +module ProviderShape = + + let private fsharpCore = typeof.Assembly + + let private allMembers = BindingFlags.Public ||| BindingFlags.NonPublic + + let private isLinqOperator (call: MethodCallExpression) = + call.Arguments.Count > 0 + && (call.Method.DeclaringType = typeof + || call.Method.DeclaringType = typeof) + + let private isStatement nodeType = + match nodeType with + | ExpressionType.Block + | ExpressionType.Assign + | ExpressionType.Loop + | ExpressionType.Goto + | ExpressionType.Label + | ExpressionType.Try + | ExpressionType.Switch + | ExpressionType.RuntimeVariables + | ExpressionType.DebugInfo -> true + | _ -> false + + let private flattensOrJoins (call: MethodCallExpression) = + isLinqOperator call + && (match call.Method.Name with + | "SelectMany" + | "Join" + | "GroupJoin" -> true + | _ -> false) + + type private ShapeVisitor() = + inherit ExpressionVisitor() + + let found = ResizeArray() + let bound = HashSet(HashIdentity.Reference) + let mutable collectionSelectorDepth = 0 + + let bind (parameters: seq) = + parameters |> Seq.filter bound.Add |> Seq.toArray + + let unbind (parameters: ParameterExpression array) = + for parameter in parameters do + bound.Remove parameter |> ignore + + [] + let (|AssignsBound|_|) (expression: Expression) = + match expression with + | :? BinaryExpression as assign when assign.NodeType = ExpressionType.Assign -> + match assign.Left with + | :? ParameterExpression as variable when bound.Contains variable -> ValueSome assign.Right + | _ -> ValueNone + | _ -> ValueNone + + member _.Violations = found |> Seq.distinct |> List.ofSeq + + override _.VisitLambda<'T>(node: Expression<'T>) = + let added = bind node.Parameters + + try + base.VisitLambda node + finally + unbind added + + // A block is reported once: its variables and their initial assignments are part of it. + override this.VisitBlock node = + found.Add(Violation.Statement ExpressionType.Block) + let added = bind node.Variables + + try + for expression in node.Expressions do + match expression with + | AssignsBound value -> this.Visit value |> ignore + | expression -> this.Visit expression |> ignore + + node + finally + unbind added + + override _.VisitParameter node = + if not (bound.Contains node) then + found.Add(Violation.UnboundParameter node.Name) + + node + + override _.VisitInvocation node = + found.Add Violation.LambdaInvocation + base.VisitInvocation node + + // A query built earlier and used in this one is part of what the provider translates. + override this.VisitConstant node = + match node.Value with + | :? IQueryable as query -> + match query.Expression with + | :? ConstantExpression as root when obj.ReferenceEquals(root.Value, query) -> () + | built -> this.Visit built |> ignore + | _ -> () + + node + + override _.VisitNew node = + if node.Type.Assembly = fsharpCore && not (node.Type.Name.StartsWith("AnonymousObject", StringComparison.Ordinal)) then + found.Add(Violation.RuntimeHelperCall $"new {node.Type.Name}") + + base.VisitNew node + + override this.VisitMethodCall node = + if node.Method.DeclaringType.Assembly = fsharpCore then + found.Add(Violation.RuntimeHelperCall $"{node.Method.DeclaringType.Name}.{node.Method.Name}") + + if collectionSelectorDepth > 0 && flattensOrJoins node then + found.Add(Violation.NestedInCollectionSelector node.Method.Name) + + if isLinqOperator node && node.Method.Name = "SelectMany" then + this.Visit node.Object |> ignore + this.Visit node.Arguments.[0] |> ignore + collectionSelectorDepth <- collectionSelectorDepth + 1 + + try + this.Visit node.Arguments.[1] |> ignore + finally + collectionSelectorDepth <- collectionSelectorDepth - 1 + + for argument in Seq.skip 2 node.Arguments do + this.Visit argument |> ignore + + node + else + base.VisitMethodCall node + + override this.Visit node = + match node with + | null -> null + | node -> + if node.NodeType <> ExpressionType.Block && isStatement node.NodeType then + found.Add(Violation.Statement node.NodeType) + + base.Visit node + + [] + let private (|Lambda|_|) (argument: Expression) = + match argument with + | :? UnaryExpression as quote when quote.NodeType = ExpressionType.Quote -> ValueSome(quote.Operand :?> LambdaExpression) + | :? LambdaExpression as lambda -> ValueSome lambda + | _ -> ValueNone + + /// A tuple or record built without saying which member each argument becomes. + let private buildsUnnamed (body: Expression) = + match body with + | :? NewExpression as construction when construction.Arguments.Count > 0 -> + match construction.Members with + | null -> + FSharpType.IsTuple construction.Type + || FSharpType.IsRecord(construction.Type, allMembers) + | _ -> false + | _ -> false + + type private MemberReadFinder(owner: ParameterExpression) = + inherit ExpressionVisitor() + + member val Found = false with get, set + + override this.VisitMember node = + if obj.ReferenceEquals(node.Expression, owner) then + this.Found <- true + + base.VisitMember node + + let private readsMemberOf (owner: ParameterExpression) (body: Expression) = + let finder = MemberReadFinder owner + finder.Visit body |> ignore + finder.Found + + let private yields (elementType: Type) (source: Expression) = + typedefof>.MakeGenericType(elementType).IsAssignableFrom source.Type + + /// Members read from elements that an earlier operator built without naming them: + /// the provider has to take apart an object it was not told how to. + let private unnamedMemberReads (expression: Expression) = + let found = ResizeArray() + + let rec producesUnnamed (expression: Expression) = + match expression with + | :? MethodCallExpression as call when isLinqOperator call -> + let source = call.Arguments.[0] + let sourceUnnamed = producesUnnamed source + + let lambdas = + [ for argument in Seq.skip 1 call.Arguments do + match argument with + | Lambda lambda -> lambda + | _ -> () ] + + if sourceUnnamed then + for lambda in lambdas do + let element = lambda.Parameters.[0] + + if yields element.Type source && readsMemberOf element lambda.Body then + found.Add(Violation.ReadsUnnamedMember element.Type.Name) + + match call.Method.Name with + | "Select" + | "SelectMany" + | "Join" + | "GroupJoin" -> lambdas |> List.last |> _.Body |> buildsUnnamed + | "Where" + | "OrderBy" + | "OrderByDescending" + | "ThenBy" + | "ThenByDescending" + | "Skip" + | "SkipWhile" + | "Take" + | "TakeWhile" + | "Distinct" + | "DefaultIfEmpty" -> sourceUnnamed + | _ -> false + | _ -> false + + producesUnnamed expression |> ignore + List.ofSeq found + + let violations (expression: Expression) = + let visitor = ShapeVisitor() + visitor.Visit expression |> ignore + visitor.Violations @ unnamedMemberReads expression |> List.distinct + + /// The LINQ operators along the source chain, outermost last. + let spine (expression: Expression) = + let rec walk (expression: Expression) operators = + match expression with + | :? MethodCallExpression as call when isLinqOperator call -> walk call.Arguments.[0] (call.Method.Name :: operators) + | _ -> operators + + walk expression [] diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs new file mode 100644 index 00000000000..b2a270042e9 --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenario.fs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Core.UnitTests.Linq + +open System +open System.Collections +open System.Globalization +open System.Linq +open System.Linq.Expressions +open System.Reflection +open Microsoft.FSharp.Reflection + +[] +type Outcome = + | Rows of IQueryable + | Value of obj + +/// One query, what it must return, and what is known to be wrong with its expression tree today. +[] +type Scenario = + { Name: string + Run: Db -> Outcome + Expected: Data -> obj list + Ordered: bool + Spine: string list voption + Violations: Violation list } + + override this.ToString() = this.Name + +module Scenario = + + let private allMembers = BindingFlags.Public ||| BindingFlags.NonPublic + + [] + let private (|Grouping|_|) (value: obj) = + match value.GetType().GetInterface(typedefof>.Name) with + | null -> ValueNone + | grouping -> ValueSome(grouping.GetProperty("Key").GetValue value) + + [] + let private (|Tuple|_|) (value: obj) = + if FSharpType.IsTuple(value.GetType()) then + ValueSome(FSharpValue.GetTupleFields value) + else + ValueNone + + [] + let private (|Record|_|) (value: obj) = + if FSharpType.IsRecord(value.GetType(), allMembers) then + let names = FSharpType.GetRecordFields(value.GetType(), allMembers) |> Seq.map _.Name + ValueSome(Seq.zip names (FSharpValue.GetRecordFields(value, allMembers))) + else + ValueNone + + [] + let private (|Union|_|) (value: obj) = + if FSharpType.IsUnion(value.GetType(), allMembers) then + let case, fields = FSharpValue.GetUnionFields(value, value.GetType(), allMembers) + ValueSome struct (case.Name, fields) + else + ValueNone + + /// Text that is equal for equal results, whichever provider materialized them. + let rec render (value: obj) = + let renderAll (values: seq) = + values |> Seq.map render |> String.concat "; " + + match value with + | null -> "null" + | :? string as text -> $"\"{text}\"" + | :? IFormattable as number -> number.ToString(null, CultureInfo.InvariantCulture) + | Grouping key -> $"{render key} => [{renderAll (Seq.cast (value :?> IEnumerable))}]" + | :? IEnumerable as items -> $"[{renderAll (Seq.cast items)}]" + | Tuple elements -> $"({renderAll elements})" + | Record fields -> + let rendered = fields |> Seq.map (fun (name, field) -> $"{name} = {render field}") + $"""{{ {String.concat "; " rendered} }}""" + | Union(case, fields) -> $"{case}({renderAll fields})" + | value -> string value + + let rows name (run: Db -> IQueryable<'T>) (expected: Data -> seq<'T>) = + { Name = name + Run = fun db -> Outcome.Rows(run db) + Expected = fun data -> [ for row in expected data -> box row ] + Ordered = false + Spine = ValueNone + Violations = [] } + + let value name (run: Db -> 'T) (expected: Data -> 'T) = + { Name = name + Run = fun db -> Outcome.Value(box (run db)) + Expected = fun data -> [ box (expected data) ] + Ordered = true + Spine = ValueNone + Violations = [] } + + let ordered scenario = { scenario with Ordered = true } + + /// The operators the source chain must consist of, outermost last. + let flat spine scenario = + { scenario with + Spine = ValueSome spine } + + /// What is wrong with the expression tree today; the scenario fails once that stops being true. + let violating violations scenario = + { scenario with + Violations = violations } + + [] + type private Observed = + { Expression: Expression + Violations: Violation list + Rows: obj list } + + let rec private rootCause (error: exn) = + match error, error.InnerException with + | :? TargetInvocationException, null -> error + | :? TargetInvocationException, inner -> rootCause inner + | _ -> error + + let private observe (scenario: Scenario) = + let provider = CapturingQueryProvider() + let data = Data() + + let observed = + try + match scenario.Run(Db.inMemory provider data) with + | Outcome.Rows rows -> + let stayed = obj.ReferenceEquals(rows.Provider, provider) + + { Expression = rows.Expression + Violations = (if stayed then [] else [ Violation.LeftProvider ]) + Rows = [ for row in rows -> row ] } + // The value comes from the last execution; whatever ran before it went to the provider too. + | Outcome.Value value when provider.Executed.Count > 0 -> + let earlier = provider.Executed |> Seq.take (provider.Executed.Count - 1) + + { Expression = provider.Executed.[provider.Executed.Count - 1] + Violations = [ for expression in earlier do yield! ProviderShape.violations expression ] + Rows = [ value ] } + | Outcome.Value value -> + { Expression = Expression.Empty() + Violations = [ Violation.LeftProvider ] + Rows = [ value ] } + with error -> + { Expression = Expression.Empty() + Violations = [ Violation.Throws((rootCause error).GetType().Name) ] + Rows = [] } + + data, observed + + let private inOrder ordered (rows: obj list) = + let rendered = List.map render rows + if ordered then rendered else List.sort rendered + + /// Everything the scenario gets wrong, empty when it behaves as described. + let failures (scenario: Scenario) = + let data, observed = observe scenario + + let violations = + observed.Violations @ ProviderShape.violations observed.Expression + |> List.distinct + + let threw = + violations + |> List.exists (function + | Violation.Throws _ -> true + | _ -> false) + + [ if set violations <> set scenario.Violations then + $"violations: expected %A{scenario.Violations}, got %A{violations}" + + if not threw then + let expected = inOrder scenario.Ordered (scenario.Expected data) + let actual = inOrder scenario.Ordered observed.Rows + + if expected <> actual then + $"rows: expected %A{expected}, got %A{actual}" + + match scenario.Spine with + | ValueSome spine when spine <> ProviderShape.spine observed.Expression -> + $"operators: expected %A{spine}, got %A{ProviderShape.spine observed.Expression}" + | _ -> () ] + |> List.map (fun failure -> $"{scenario.Name}: {failure}\n in {observed.Expression}") + + let check scenario = + match failures scenario with + | [] -> () + | failures -> failwith (String.concat "\n" failures) diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenarioDomain.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenarioDomain.fs new file mode 100644 index 00000000000..ada7f2ba70f --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryScenarioDomain.fs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Core.UnitTests.Linq + +open System +open System.Collections.Generic +open System.Linq + +[] +type Product() = + member val Id = 0 with get, set + member val Name = "" with get, set + member val Category = "" with get, set + override this.ToString() = $"Product#{this.Id}" + +[] +type OrderLine() = + member val Id = 0 with get, set + member val OrderId = 0 with get, set + member val ProductId = 0 with get, set + member val Quantity = 0 with get, set + member val Price = 0.0 with get, set + member val Discount = Nullable() with get, set + override this.ToString() = $"OrderLine#{this.Id}" + +[] +type Order() = + member val Id = 0 with get, set + member val CustomerId = 0 with get, set + member val Total = 0.0 with get, set + member val Lines: ICollection = List() with get, set + override this.ToString() = $"Order#{this.Id}" + +[] +type Customer() = + member val Id = 0 with get, set + member val Name = "" with get, set + member val City = "" with get, set + member val Age = 0 with get, set + member val Registered = Nullable() with get, set + member val Orders: ICollection = List() with get, set + override this.ToString() = $"Customer#{this.Id}" + +type CustomerView() = + member val ViewName = "" with get, set + member val ViewAge = 0 with get, set + override this.ToString() = $"CustomerView({this.ViewName}, {this.ViewAge})" + +type NameAndAge = { PersonName: string; PersonAge: int } + +type NameAgeCity = + { FullName: string + Years: int + Town: string } + +[] +type NameAndAgeDto = { DtoName: string; DtoAge: int } + +type CityAndAge = { KeyCity: string; KeyAge: int } + +type OrderAndQuantity = { OrderNumber: int; Units: int } + +type Data() = + let line id orderId productId quantity price discount = + OrderLine(Id = id, OrderId = orderId, ProductId = productId, Quantity = quantity, Price = price, Discount = discount) + + let order id customerId total (lines: OrderLine list) = + Order(Id = id, CustomerId = customerId, Total = total, Lines = List lines) + + let customer id name city age registered (orders: Order list) = + Customer(Id = id, Name = name, City = city, Age = age, Registered = registered, Orders = List orders) + + let orders = + [ order 10 1 12.5 [ line 100 10 1 2 2.5 (Nullable 1); line 101 10 2 1 7.5 (Nullable()) ] + order 11 1 3.0 [ line 102 11 3 1 3.0 (Nullable 2) ] + order 12 2 5.0 [ line 103 12 1 2 2.5 (Nullable()) ] + order 13 3 0.0 [] ] + + let ordersOf customerId = + orders |> List.filter (fun order -> order.CustomerId = customerId) + + member val Products = + [ Product(Id = 1, Name = "Pen", Category = "Office") + Product(Id = 2, Name = "Ink", Category = "Office") + Product(Id = 3, Name = "Tea", Category = "Food") ] + + member val Customers = + [ customer 1 "Alice" "London" 30 (Nullable(DateTime(2020, 1, 1))) (ordersOf 1) + customer 2 "Bob" "Paris" 25 (Nullable()) (ordersOf 2) + customer 3 "Carol" "London" 35 (Nullable(DateTime(2021, 6, 15))) (ordersOf 3) + customer 4 "Dave" "Berlin" 28 (Nullable()) [] ] + + member _.Orders = orders + member _.Lines = orders |> List.collect (fun order -> List.ofSeq order.Lines) + +[] +type Db = + { Customers: IQueryable + Orders: IQueryable + Lines: IQueryable + Products: IQueryable } + +module Db = + + let inMemory (provider: CapturingQueryProvider) (data: Data) = + { Customers = provider.Source("Customers", data.Customers) + Orders = provider.Source("Orders", data.Orders) + Lines = provider.Source("Lines", data.Lines) + Products = provider.Source("Products", data.Products) } diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs new file mode 100644 index 00000000000..6e284ed7d4d --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryShapeTests.fs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +namespace FSharp.Core.UnitTests.Linq + +open Xunit + +type QueryShapeTests() = + + static let rows (cases: Scenario list) = + cases |> Seq.map (fun scenario -> [| box scenario |]) + + static member Projections = rows (QueryProjectionScenarios.cases ()) + static member Operators = rows (QueryOperatorScenarios.cases ()) + static member Keys = rows (QueryKeyScenarios.cases ()) + static member Structures = rows (QueryStructureScenarios.cases ()) + static member MethodSyntax = rows (QueryMethodSyntaxScenarios.cases ()) + + [] + member _.Projection(scenario: Scenario) = Scenario.check scenario + + [] + member _.Operator(scenario: Scenario) = Scenario.check scenario + + [] + member _.Key(scenario: Scenario) = Scenario.check scenario + + [] + member _.Structure(scenario: Scenario) = Scenario.check scenario + + [] + member _.``Method syntax``(scenario: Scenario) = Scenario.check scenario diff --git a/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs new file mode 100644 index 00000000000..8393af71208 --- /dev/null +++ b/tests/FSharp.Core.UnitTests/FSharp.Core/Microsoft.FSharp.Linq/QueryStructureScenarios.fs @@ -0,0 +1,404 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module FSharp.Core.UnitTests.Linq.QueryStructureScenarios + +open System.Linq + +let private quotedLambda = + [ Violation.RuntimeHelperCall "LeafExpressionConverter.QuotationToLambdaExpression" + Violation.RuntimeHelperCall "LeafExpressionConverter.SubstHelper" ] + +let private nested operators = + operators |> List.map Violation.NestedInCollectionSelector + +let private orDefault (items: seq<'T>) = + if Seq.isEmpty items then Seq.singleton Unchecked.defaultof<'T> else items + +let private idOrNone (order: Order) = + match order with + | null -> -1 + | order -> order.Id + +let private nameOrNone (product: Product) = + match product with + | null -> "" + | product -> product.Name + +let private ordersWithLines (data: Data) = + seq { for c in data.Customers do for o in c.Orders do for l in o.Lines -> c.Name, o.Id, l.Id } + +let private customersWithOrders (data: Data) = + seq { for c in data.Customers do for o in c.Orders -> c.Name, o.Id } + +let private leftJoinedOrders (data: Data) = + seq { for c in data.Customers do for o in orDefault c.Orders -> c.Name, idOrNone o } + +let private leftJoinedOrdersAndProducts (data: Data) = + seq { + for c in data.Customers do + for o in orDefault c.Orders do + for p in orDefault (data.Products |> List.filter (fun p -> p.Id = c.Id)) -> c.Name, idOrNone o, nameOrNone p + } + +let private loops () = + [ Scenario.rows + "two for" + (fun db -> query { for c in db.Customers do for p in db.Products do select (c.Name, p.Name) }) + (fun data -> seq { for c in data.Customers do for p in data.Products -> c.Name, p.Name }) + |> Scenario.flat [ "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "two for, where over both" + (fun db -> query { for c in db.Customers do for o in db.Orders do where (o.CustomerId = c.Id); select (c.Name, o.Id) }) + customersWithOrders + |> Scenario.flat [ "SelectMany"; "Where"; "Select"; "Select" ] + + Scenario.rows + "for, where, for, where" + (fun db -> + query { + for c in db.Customers do + where (c.Age > 26) + for o in db.Orders do + where (o.CustomerId = c.Id) + select (c.Name, o.Id) + }) + (fun data -> seq { for c in data.Customers do if c.Age > 26 then for o in c.Orders -> c.Name, o.Id }) + |> Scenario.flat [ "Where"; "SelectMany"; "Where"; "Select"; "Select" ] + + Scenario.rows + "for over a navigation collection" + (fun db -> query { for c in db.Customers do for o in c.Orders do select (c.Name, o.Id) }) + customersWithOrders + |> Scenario.flat [ "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "for over two levels of navigation collections" + (fun db -> query { for c in db.Customers do for o in c.Orders do for l in o.Lines do select (c.Name, o.Id, l.Id) }) + ordersWithLines + |> Scenario.flat [ "SelectMany"; "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "three for" + (fun db -> + query { + for c in db.Customers do + for o in db.Orders do + for l in db.Lines do + where (o.CustomerId = c.Id && l.OrderId = o.Id) + select (c.Name, o.Id, l.Id) + }) + ordersWithLines + |> Scenario.flat [ "SelectMany"; "SelectMany"; "Where"; "Select"; "Select" ] + + Scenario.rows + "four for" + (fun db -> + query { + for c in db.Customers do + for o in c.Orders do + for l in o.Lines do + for p in db.Products do + where (p.Id = l.ProductId) + select (c.Name, l.Id, p.Name) + }) + (fun data -> + seq { + for c in data.Customers do + for o in c.Orders do + for l in o.Lines do + for p in data.Products do + if p.Id = l.ProductId then yield c.Name, l.Id, p.Name + }) + |> Scenario.flat [ "SelectMany"; "SelectMany"; "SelectMany"; "Where"; "Select"; "Select" ] + + Scenario.rows + "if then between three for" + (fun db -> + query { + for c in db.Customers do + where (c.Age > 26) + for o in c.Orders do + if o.Total > 4.0 then + for l in o.Lines do + yield (c.Name, l.Id) + }) + (fun data -> + seq { + for c in data.Customers do + if c.Age > 26 then + for o in c.Orders do + if o.Total > 4.0 then + for l in o.Lines -> c.Name, l.Id + }) + |> Scenario.flat [ "Where"; "SelectMany"; "Where"; "SelectMany"; "Select" ] + + Scenario.rows + "yield! after two for" + (fun db -> query { for c in db.Customers do for o in c.Orders do yield! o.Lines }) + (fun data -> Seq.ofList data.Lines) + |> Scenario.flat [ "SelectMany"; "SelectMany" ] + + Scenario.rows + "for after sortBy" + (fun db -> query { for c in db.Customers do sortBy c.Age; for p in db.Products do select (c.Name, p.Name) }) + (fun data -> seq { for c in data.Customers |> List.sortBy _.Age do for p in data.Products -> c.Name, p.Name }) + |> Scenario.ordered + |> Scenario.flat [ "OrderBy"; "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "for after join" + (fun db -> + query { + for c in db.Customers do + join o in db.Orders on (c.Id = o.CustomerId) + for l in db.Lines do + where (l.OrderId = o.Id) + select (c.Name, o.Id, l.Id) + }) + ordersWithLines + |> Scenario.flat [ "Join"; "SelectMany"; "Where"; "Select"; "Select" ] + + Scenario.rows + "let between for" + (fun db -> + query { for c in db.Customers do let next = c.Id + 9 in for o in db.Orders do where (o.Id = next); select (c.Name, o.Id) }) + (fun data -> seq { for c in data.Customers do for o in data.Orders do if o.Id = c.Id + 9 then yield c.Name, o.Id }) + |> Scenario.flat [ "SelectMany"; "Where"; "Select"; "Select" ] + + Scenario.rows + "if then around for" + (fun db -> query { for c in db.Customers do if c.Age > 26 then for o in c.Orders do yield (c.Name, o.Id) }) + (fun data -> seq { for c in data.Customers do if c.Age > 26 then for o in c.Orders -> c.Name, o.Id }) + |> Scenario.flat [ "Where"; "SelectMany"; "Select" ] + + Scenario.rows + "for around if then" + (fun db -> query { for c in db.Customers do for o in db.Orders do if o.CustomerId = c.Id then yield (c.Name, o.Id) }) + customersWithOrders + |> Scenario.flat [ "SelectMany"; "Select" ] + + Scenario.rows + "yield! of a navigation collection" + (fun db -> query { for c in db.Customers do yield! c.Orders }) + (fun data -> Seq.ofList data.Orders) + |> Scenario.flat [ "SelectMany" ] + + Scenario.rows + "distinct and take after two for" + (fun db -> query { for c in db.Customers do for o in c.Orders do select c.City; distinct; take 1 }) + (fun _ -> seq { "London" }) + |> Scenario.ordered + |> Scenario.flat [ "SelectMany"; "Select"; "Distinct"; "Take" ] + + Scenario.rows + "groupBy after two for" + (fun db -> query { for c in db.Customers do for o in c.Orders do groupBy c.City into g; select (g.Key, g.Count()) }) + (fun data -> seq { for c in data.Customers do for _ in c.Orders -> c.City } |> Seq.countBy id) + |> Scenario.violating [ Violation.RuntimeHelperCall "new Grouping`2" ] + + Scenario.value "count over two for" (fun db -> query { for c in db.Customers do for o in c.Orders do count }) (fun _ -> 4) + + Scenario.value + "exists over two for" + (fun db -> query { for c in db.Customers do for o in c.Orders do exists (o.Total > 10.0 && c.City = "London") }) + (fun _ -> true) + |> Scenario.violating [ Violation.ReadsUnnamedMember "Tuple`2" ] ] + +let private subqueries () = + [ Scenario.rows + "nested query as the source" + (fun db -> query { for o in (query { for o in db.Orders do where (o.Total > 4.0); select o }) do select o.Id }) + (fun data -> seq { for o in data.Orders do if o.Total > 4.0 then yield o.Id }) + |> Scenario.flat [ "Where"; "Select" ] + + Scenario.rows + "nested query as the source of the second for" + (fun db -> + query { + for c in db.Customers do + for o in (query { for o in db.Orders do where (o.CustomerId = c.Id); select o }) do + select (c.Name, o.Id) + }) + customersWithOrders + |> Scenario.flat [ "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "nested query in select" + (fun db -> + query { for c in db.Customers do select (c.Name, query { for o in db.Orders do where (o.CustomerId = c.Id); select o.Id }) }) + (fun data -> seq { for c in data.Customers -> c.Name, (seq { for o in c.Orders -> o.Id }).AsQueryable() }) + + Scenario.rows + "second for over a source filtered by a lambda" + (fun db -> query { for c in db.Customers do for o in db.Orders.Where(fun o -> o.CustomerId = c.Id) do select (c.Name, o.Id) }) + customersWithOrders + |> Scenario.violating quotedLambda + + Scenario.rows + "where with a lambda over another source" + (fun db -> query { for c in db.Customers do where (db.Orders.Any(fun o -> o.CustomerId = c.Id)); select c.Name }) + (fun data -> seq { for c in data.Customers do if c.Orders.Count > 0 then yield c.Name }) + |> Scenario.violating quotedLambda ] + +let private leftJoins () = + [ Scenario.rows + "left join spelled leftOuterJoin" + (fun db -> + query { + for c in db.Customers do + leftOuterJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders do + select (c.Name, (if obj.ReferenceEquals(o, null) then -1 else o.Id)) + }) + leftJoinedOrders + |> Scenario.flat [ "GroupJoin"; "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "left join spelled groupJoin and DefaultIfEmpty" + (fun db -> + query { + for c in db.Customers do + groupJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders.DefaultIfEmpty() do + select (c.Name, (if obj.ReferenceEquals(o, null) then -1 else o.Id)) + }) + leftJoinedOrders + |> Scenario.flat [ "GroupJoin"; "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "two left joins spelled leftOuterJoin" + (fun db -> + query { + for c in db.Customers do + leftOuterJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders do + leftOuterJoin p in db.Products on (c.Id = p.Id) into products + for p in products do + select + (c.Name, + (if obj.ReferenceEquals(o, null) then -1 else o.Id), + (if obj.ReferenceEquals(p, null) then "" else p.Name)) + }) + leftJoinedOrdersAndProducts + |> Scenario.violating (nested [ "GroupJoin"; "SelectMany" ]) + + Scenario.rows + "two left joins spelled groupJoin and DefaultIfEmpty" + (fun db -> + query { + for c in db.Customers do + groupJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders.DefaultIfEmpty() do + groupJoin p in db.Products on (c.Id = p.Id) into products + for p in products.DefaultIfEmpty() do + select + (c.Name, + (if obj.ReferenceEquals(o, null) then -1 else o.Id), + (if obj.ReferenceEquals(p, null) then "" else p.Name)) + }) + leftJoinedOrdersAndProducts + |> Scenario.flat [ "GroupJoin"; "SelectMany"; "GroupJoin"; "SelectMany"; "Select"; "Select" ] + + Scenario.rows + "join after a left join" + (fun db -> + query { + for c in db.Customers do + leftOuterJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders do + join p in db.Products on (c.Id = p.Id) + select (c.Name, (if obj.ReferenceEquals(o, null) then -1 else o.Id), p.Name) + }) + (leftJoinedOrdersAndProducts >> Seq.filter (fun (_, _, product) -> product <> "")) + |> Scenario.violating (nested [ "Join" ]) + + Scenario.rows + "two for after a left join" + (fun db -> + query { + for c in db.Customers do + leftOuterJoin o in db.Orders on (c.Id = o.CustomerId) into orders + for o in orders do + for l in db.Lines do + where (not (obj.ReferenceEquals(o, null)) && l.OrderId = o.Id) + select (c.Name, o.Id, l.Id) + }) + ordersWithLines + |> Scenario.violating (nested [ "SelectMany" ]) + + Scenario.rows + "join after two for" + (fun db -> + query { + for c in db.Customers do + for o in c.Orders do + join l in db.Lines on (o.Id = l.OrderId) + select (c.Name, o.Id, l.Id) + }) + ordersWithLines + |> Scenario.flat [ "SelectMany"; "Join"; "Select"; "Select"; "Select" ] + + Scenario.rows + "join after a for over tuples" + (fun db -> + query { + for c in db.Customers do + for (o, id) in (query { for o in db.Orders do select (o, o.Id) }) do + join l in db.Lines on (id = l.OrderId) + where (o.CustomerId = c.Id) + select (c.Name, id, l.Id) + }) + ordersWithLines + |> Scenario.flat [ "SelectMany"; "Join"; "Select"; "Where"; "Select"; "Select" ] + + Scenario.rows + "join on a tuple key after a for over tuples" + (fun db -> + query { + for c in db.Customers do + for (o, id) in (query { for o in db.Orders do select (o, o.Id) }) do + join l in db.Lines on ((id, 1) = (l.OrderId, 1)) + where (o.CustomerId = c.Id) + select (c.Name, id, l.Id) + }) + ordersWithLines + |> Scenario.flat [ "SelectMany"; "Join"; "Select"; "Where"; "Select"; "Select" ] + + Scenario.rows + "join whose inner key uses an earlier variable" + (fun db -> + query { + for c in db.Customers do + for (o, id) in (query { for o in db.Orders do select (o, o.Id) }) do + join l in db.Lines on ((id, o.CustomerId) = (l.OrderId, c.Id)) + select (c.Name, id, l.Id) + }) + ordersWithLines + |> Scenario.violating (nested [ "Join" ]) + + Scenario.rows + "groupJoin after two for" + (fun db -> + query { + for c in db.Customers do + for o in c.Orders do + groupJoin l in db.Lines on (o.Id = l.OrderId) into lines + select (c.Name, o.Id, lines.Count()) + }) + (fun data -> seq { for c in data.Customers do for o in c.Orders -> c.Name, o.Id, o.Lines.Count }) + |> Scenario.flat [ "SelectMany"; "GroupJoin"; "Select"; "Select"; "Select" ] + + Scenario.rows + "join over a source that uses an earlier variable" + (fun db -> + query { + for c in db.Customers do + for o in c.Orders do + join same in c.Orders on (o.Id = same.Id) + select (c.Name, same.Id) + }) + customersWithOrders + |> Scenario.violating (nested [ "Join" ]) ] + +let cases () = loops () @ subqueries () @ leftJoins ()