1 //---------------------------------------------------------------------
2 // <copyright file="PropertyPushdownHelper.cs" company="Microsoft">
3 // Copyright (c) Microsoft Corporation. All rights reserved.
7 // @backupOwner Microsoft
8 //---------------------------------------------------------------------
11 using System.Collections.Generic;
12 //using System.Diagnostics; // Please use PlanCompiler.Assert instead of Debug.Assert in this class...
14 // It is fine to use Debug.Assert in cases where you assert an obvious thing that is supposed
15 // to prevent from simple mistakes during development (e.g. method argument validation
16 // in cases where it was you who created the variables or the variables had already been validated or
17 // in "else" clauses where due to code changes (e.g. adding a new value to an enum type) the default
18 // "else" block is chosen why the new condition should be treated separately). This kind of asserts are
19 // (can be) helpful when developing new code to avoid simple mistakes but have no or little value in
20 // the shipped product.
21 // PlanCompiler.Assert *MUST* be used to verify conditions in the trees. These would be assumptions
22 // about how the tree was built etc. - in these cases we probably want to throw an exception (this is
23 // what PlanCompiler.Assert does when the condition is not met) if either the assumption is not correct
24 // or the tree was built/rewritten not the way we thought it was.
25 // Use your judgment - if you rather remove an assert than ship it use Debug.Assert otherwise use
26 // PlanCompiler.Assert.
28 using System.Globalization;
30 using System.Data.Common;
31 using md = System.Data.Metadata.Edm;
32 using System.Data.Query.InternalTrees;
34 namespace System.Data.Query.PlanCompiler
38 /// The PropertyPushdownHelper module is a submodule of the StructuredTypeEliminator
39 /// module. It serves as a useful optimization sidekick for NominalTypeEliminator which
40 /// is the real guts of eliminating structured types.
42 /// The goal of this module is to identify a list of desired properties for each node
43 /// (and Var) in the tree that is of a structured type. This list of desired properties
44 /// is identified in a top-down push fashion.
46 /// While it is desirable to get as accurate information as possible, it is unnecessary
47 /// for this module to be super-efficient (i.e.) it is ok for it to get a superset
48 /// of the appropriate information for each node, but it is absolutely not ok for it
49 /// to get a subset. Later phases (projection pruning) can help eliminate unnecessary
50 /// information, but the query cannot be made incorrect.
52 /// This module is implemented as a visitor - it leverages information about
53 /// types in the query - made possible by the TypeFlattener module - and walks
54 /// down the tree pushing properties to each child of a node. It builds two maps:
56 /// (*) a node-property map
57 /// (*) a var-property map
59 /// Each of these keeps trackof the properties needed from each node/var.
61 /// These maps are returned to the caller and will be used by the NominalTypeEliminator
62 /// module to eliminate all structured types.
64 internal class PropertyPushdownHelper : BasicOpVisitor
69 private readonly Dictionary<Node, PropertyRefList> m_nodePropertyRefMap;
70 private readonly Dictionary<Var, PropertyRefList> m_varPropertyRefMap;
71 private readonly StructuredTypeInfo m_structuredTypeInfo;
77 private PropertyPushdownHelper(StructuredTypeInfo structuredTypeInfo)
79 m_structuredTypeInfo = structuredTypeInfo;
80 m_varPropertyRefMap = new Dictionary<Var, PropertyRefList>();
81 m_nodePropertyRefMap = new Dictionary<Node, PropertyRefList>();
86 #region Process Driver
90 /// Walks the tree, and "pushes" down information about required properties
91 /// to every node and Var in the tree.
93 /// <param name="itree">The query tree</param>
94 /// <param name="structuredTypeInfo">Type info for structured types appearing in query.</param>
95 /// <param name="varPropertyRefs">List of desired properties from each Var</param>
96 /// <param name="nodePropertyRefs">List of desired properties from each node</param>
97 internal static void Process(Command itree, StructuredTypeInfo structuredTypeInfo, out Dictionary<Var, PropertyRefList> varPropertyRefs, out Dictionary<Node, PropertyRefList> nodePropertyRefs)
99 PropertyPushdownHelper pph = new PropertyPushdownHelper(structuredTypeInfo);
100 pph.Process(itree.Root);
102 varPropertyRefs = pph.m_varPropertyRefMap;
103 nodePropertyRefs = pph.m_nodePropertyRefMap;
107 /// the driver routine. Invokes the visitor, and then returns the collected
110 /// <param name="rootNode">node in the tree to begin processing at</param>
111 private void Process(Node rootNode)
113 // simply invoke the visitor
114 rootNode.Op.Accept(this, rootNode);
119 #region private methods
121 #region state maintenance
124 /// Get the list of propertyrefs for a node. If none exists, create an
125 /// empty structure and store it in the map
127 /// <param name="node">Specific node</param>
128 /// <returns>List of properties expected from this node</returns>
129 private PropertyRefList GetPropertyRefList(Node node)
131 PropertyRefList propRefs;
132 if (!m_nodePropertyRefMap.TryGetValue(node, out propRefs))
134 propRefs = new PropertyRefList();
135 m_nodePropertyRefMap[node] = propRefs;
141 /// Add a list of property references for this node
143 /// <param name="node">the node</param>
144 /// <param name="propertyRefs">list of property references</param>
145 private void AddPropertyRefs(Node node, PropertyRefList propertyRefs)
147 PropertyRefList refs = GetPropertyRefList(node);
148 refs.Append(propertyRefs);
152 /// Get the list of desired properties for a Var
154 /// <param name="v">the var</param>
155 /// <returns>List of desired properties</returns>
156 private PropertyRefList GetPropertyRefList(Var v)
158 PropertyRefList propRefs;
159 if (!m_varPropertyRefMap.TryGetValue(v, out propRefs))
161 propRefs = new PropertyRefList();
162 m_varPropertyRefMap[v] = propRefs;
168 /// Add a new set of properties to a Var
170 /// <param name="v">the var</param>
171 /// <param name="propertyRefs">desired properties</param>
172 private void AddPropertyRefs(Var v, PropertyRefList propertyRefs)
174 PropertyRefList currentRefs = GetPropertyRefList(v);
175 currentRefs.Append(propertyRefs);
180 #region Visitor Helpers
183 /// Gets the list of "identity" properties for an entity. Gets the
184 /// "entitysetid" property in addition to the "key" properties
186 /// <param name="type"></param>
187 /// <returns></returns>
188 private static PropertyRefList GetIdentityProperties(md.EntityType type)
190 PropertyRefList desiredProperties = GetKeyProperties(type);
191 desiredProperties.Add(EntitySetIdPropertyRef.Instance);
192 return desiredProperties;
196 /// Gets the list of key properties for an entity
198 /// <param name="entityType"></param>
199 /// <returns></returns>
200 private static PropertyRefList GetKeyProperties(md.EntityType entityType)
202 PropertyRefList desiredProperties = new PropertyRefList();
203 foreach (md.EdmMember p in entityType.KeyMembers)
205 md.EdmProperty edmP = p as md.EdmProperty;
206 PlanCompiler.Assert(edmP != null, "EntityType had non-EdmProperty key member?");
207 SimplePropertyRef pRef = new SimplePropertyRef(edmP);
208 desiredProperties.Add(pRef);
210 return desiredProperties;
216 /// Default visitor for an Op.
218 /// Simply walks through all children looking for Ops of structured
219 /// types, and asks for all their properties.
222 /// Several of the ScalarOps take the default handling, to simply ask
223 /// for all the children's properties:
241 /// They do not exist here to eliminate noise.
243 /// Note that the NewRecordOp and the NewInstanceOp could be optimized to only
244 /// push down the appropriate references, but it isn't clear to Murali that the
245 /// complexity is worth it.
247 /// <param name="n"></param>
248 protected override void VisitDefault(Node n)
250 // for each child that is a complex type, simply ask for all properties
251 foreach (Node chi in n.Children)
253 ScalarOp chiOp = chi.Op as ScalarOp;
254 if (chiOp != null && TypeUtils.IsStructuredType(chiOp.Type))
256 AddPropertyRefs(chi, PropertyRefList.All);
267 /// Ref - ask for all properties
268 /// Entity, ComplexType - ask for the same properties I've been asked for
269 /// Record - ask for all properties (Note: This should be more optimized in the future
270 /// since we can actually "remap" the properties)
272 /// <param name="op"></param>
273 /// <param name="n"></param>
274 public override void Visit(SoftCastOp op, Node n)
276 PropertyRefList childProps = null;
278 if (md.TypeSemantics.IsReferenceType(op.Type))
280 childProps = PropertyRefList.All;
282 else if (md.TypeSemantics.IsNominalType(op.Type))
284 PropertyRefList myProps = m_nodePropertyRefMap[n];
285 childProps = myProps.Clone();
287 else if (md.TypeSemantics.IsRowType(op.Type))
290 // Note: We should do a better job here (by translating
291 // our PropertyRefs to the equivalent property refs on the child
293 childProps = PropertyRefList.All;
296 if (childProps != null)
298 AddPropertyRefs(n.Child0, childProps);
306 /// Pushes its desired properties to each of the WHEN/ELSE clauses
308 /// <param name="op"></param>
309 /// <param name="n"></param>
310 public override void Visit(CaseOp op, Node n)
312 // First find the properties that my parent expects from me
313 PropertyRefList pdProps = GetPropertyRefList(n);
315 // push down the same properties to my then/else clauses.
316 // the "when" clauses are irrelevant
317 for (int i = 1; i < n.Children.Count - 1; i += 2)
319 PropertyRefList cdProps = pdProps.Clone();
320 AddPropertyRefs(n.Children[i], cdProps);
322 AddPropertyRefs(n.Children[n.Children.Count - 1], pdProps.Clone());
324 // Now visit the children
329 /// CollectOp handling.
331 /// <param name="op"></param>
332 /// <param name="n"></param>
333 public override void Visit(CollectOp op, Node n)
335 // Simply visit the children without pushing down any references to them.
340 /// ComparisonOp handling
342 /// <param name="op"></param>
343 /// <param name="n"></param>
344 public override void Visit(ComparisonOp op, Node n)
346 // Check to see if the children are structured types. Furthermore,
347 // if the children are of entity types, then all we really need are
348 // the key properties (and the entityset property)
349 // For record and ref types, simply keep going
350 md.TypeUsage childOpType = (n.Child0.Op as ScalarOp).Type;
352 if (!TypeUtils.IsStructuredType(childOpType))
356 else if (md.TypeSemantics.IsRowType(childOpType) || md.TypeSemantics.IsReferenceType(childOpType))
360 PlanCompiler.Assert(md.TypeSemantics.IsEntityType(childOpType), "unexpected childOpType?");
361 PropertyRefList desiredProperties = GetIdentityProperties(TypeHelpers.GetEdmType<md.EntityType>(childOpType));
363 // Now push these set of properties to each child
364 foreach (Node chi in n.Children)
365 AddPropertyRefs(chi, desiredProperties);
367 // Visit the children
373 /// ElementOp handling
375 /// <param name="op"></param>
376 /// <param name="n"></param>
377 public override void Visit(ElementOp op, Node n)
379 // Cannot occur at this stage of processing
380 throw EntityUtil.NotSupported();
384 /// GetEntityRefOp handling
386 /// Ask for the "identity" properties from the input entity, and push that
389 /// <param name="op"></param>
390 /// <param name="n"></param>
391 public override void Visit(GetEntityRefOp op, Node n)
393 ScalarOp childOp = n.Child0.Op as ScalarOp;
394 PlanCompiler.Assert(childOp != null, "input to GetEntityRefOp is not a ScalarOp?");
396 // bug 428542 - the child is of the entity type; not this op
397 md.EntityType entityType = TypeHelpers.GetEdmType<md.EntityType>(childOp.Type);
399 PropertyRefList desiredProperties = GetIdentityProperties(entityType);
400 AddPropertyRefs(n.Child0, desiredProperties);
408 /// Simply requests the "typeid" property from
409 /// the input. No other property is required
411 /// <param name="op">IsOf op</param>
412 /// <param name="n">Node to visit</param>
413 public override void Visit(IsOfOp op, Node n)
416 // The only property I need from my child is the typeid property;
417 PropertyRefList childProps = new PropertyRefList();
418 childProps.Add(TypeIdPropertyRef.Instance);
419 AddPropertyRefs(n.Child0, childProps);
425 /// Common handler for RelPropertyOp and PropertyOp.
426 /// Simply pushes down the desired set of properties to the child
428 /// <param name="op">the *propertyOp</param>
429 /// <param name="n">node tree corresponding to the Op</param>
430 /// <param name="propertyRef">the property reference</param>
431 private void VisitPropertyOp(Op op, Node n, PropertyRef propertyRef)
433 PropertyRefList cdProps = new PropertyRefList();
434 if (!TypeUtils.IsStructuredType(op.Type))
436 cdProps.Add(propertyRef);
440 // Get the list of properties my parent expects from me.
441 PropertyRefList pdProps = GetPropertyRefList(n);
443 // Ask my child (which is really my container type) for each of these
446 // If I've been asked for all my properties, then get the
447 // corresponding flat list of properties from my children.
448 // For now, however, simply ask for all properties in this case
449 // What we really need to do is to get the "flattened" list of
450 // properties from the input, and prepend each of these with
451 // our property name. We don't have that info available, so
452 // I'm taking the easier route.
453 if (pdProps.AllProperties)
459 foreach (PropertyRef p in pdProps.Properties)
461 cdProps.Add(p.CreateNestedPropertyRef(propertyRef));
466 // push down my expectations
467 AddPropertyRefs(n.Child0, cdProps);
472 /// RelPropertyOp handling.
473 /// Delegates to VisitPropertyOp. Marks the rel-property as required from the
476 /// <param name="op">the RelPropertyOp</param>
477 /// <param name="n">node tree corresponding to the op</param>
478 public override void Visit(RelPropertyOp op, Node n)
480 VisitPropertyOp(op, n, new RelPropertyRef(op.PropertyInfo));
484 /// PropertyOp handling
486 /// Pushes down the requested properties along with the current
487 /// property to the child
489 /// <param name="op"></param>
490 /// <param name="n"></param>
491 public override void Visit(PropertyOp op, Node n)
493 VisitPropertyOp(op, n, new SimplePropertyRef(op.PropertyInfo));
499 /// Simply passes down "my" desired properties, and additionally
500 /// asks for the TypeID property
502 /// <param name="op"></param>
503 /// <param name="n"></param>
504 public override void Visit(TreatOp op, Node n)
506 // First find the properties that my parent expects from me
507 PropertyRefList pdProps = GetPropertyRefList(n);
509 // Push down each of these, and in addition, push down the typeid property
511 PropertyRefList childProps = pdProps.Clone();
512 childProps.Add(TypeIdPropertyRef.Instance);
513 AddPropertyRefs(n.Child0, childProps);
518 /// VarRefOp handling
520 /// Simply passes along the current "desired" properties
521 /// to the corresponding Var
523 /// <param name="op"></param>
524 /// <param name="n"></param>
525 public override void Visit(VarRefOp op, Node n)
527 if (TypeUtils.IsStructuredType(op.Var.Type))
529 // Get the properties that my parent expects from me.
530 PropertyRefList myProps = GetPropertyRefList(n);
531 // Add this onto the list of properties expected from the var itself
532 AddPropertyRefs(op.Var, myProps);
541 /// VarDefOp handling
543 /// Pushes the "desired" properties to the
544 /// defining expression
546 /// <param name="op"></param>
547 /// <param name="n"></param>
548 public override void Visit(VarDefOp op, Node n)
550 if (TypeUtils.IsStructuredType(op.Var.Type))
552 PropertyRefList myProps = GetPropertyRefList(op.Var);
553 // Push this down to the expression defining the var
554 AddPropertyRefs(n.Child0, myProps);
560 /// VarDefListOp handling
562 /// <param name="op"></param>
563 /// <param name="n"></param>
564 public override void Visit(VarDefListOp op, Node n)
566 // Simply visit the children without pushing down any references to them.
576 /// CrossApplyOp handling
577 /// OuterApplyOp handling
579 /// Handling for all ApplyOps: Process the right child, and then
580 /// the left child - since the right child may have references to the
583 /// <param name="op">apply op</param>
584 /// <param name="n"></param>
585 protected override void VisitApplyOp(ApplyBaseOp op, Node n)
587 VisitNode(n.Child1); // the right input
588 VisitNode(n.Child0); // the left input
593 /// DistinctOp handling
595 /// Require all properties out of all structured vars
597 /// <param name="op"></param>
598 /// <param name="n"></param>
599 public override void Visit(DistinctOp op, Node n)
601 foreach (Var v in op.Keys)
602 if (TypeUtils.IsStructuredType(v.Type))
604 AddPropertyRefs(v, PropertyRefList.All);
610 /// FilterOp handling
612 /// Process the predicate child, and then the input child - since the
613 /// predicate child will probably have references to the input.
615 /// <param name="op"></param>
616 /// <param name="n"></param>
617 public override void Visit(FilterOp op, Node n)
619 VisitNode(n.Child1); // visit predicate first
620 VisitNode(n.Child0); // then visit the relop input
624 /// GroupByOp handling
626 /// <param name="op"></param>
627 /// <param name="n"></param>
628 protected override void VisitGroupByOp(GroupByBaseOp op, Node n)
630 // First "request" all properties for every key (that is a structured type)
631 foreach (Var v in op.Keys)
633 if (TypeUtils.IsStructuredType(v.Type))
635 AddPropertyRefs(v, PropertyRefList.All);
639 // Now visit the aggregate definitions, the key definitions, and
640 // the relop input in that order
641 VisitChildrenReverse(n);
647 /// CrossJoinOp handling
648 /// InnerJoinOp handling
649 /// LeftOuterJoinOp handling
650 /// FullOuterJoinOp handling
652 /// Handler for all JoinOps. For all joins except cross joins, process
653 /// the predicate first, and then the inputs - the inputs can be processed
656 /// For cross joins, simply process all the (relop) inputs
658 /// <param name="op">join op</param>
659 /// <param name="n"></param>
660 protected override void VisitJoinOp(JoinBaseOp op, Node n)
662 if (n.Op.OpType == OpType.CrossJoin)
666 VisitNode(n.Child2); // the predicate first
667 VisitNode(n.Child0); // then, the left input
668 VisitNode(n.Child1); // the right input
673 /// ProjectOp handling
675 /// <param name="op"></param>
676 /// <param name="n"></param>
677 public override void Visit(ProjectOp op, Node n)
679 VisitNode(n.Child1); // visit projections first
680 VisitNode(n.Child0); // then visit the relop input
685 /// ScanTableOp handler
687 /// <param name="op"></param>
688 /// <param name="n"></param>
689 public override void Visit(ScanTableOp op, Node n)
691 PlanCompiler.Assert(!n.HasChild0, "scanTableOp with an input?");
697 /// ask for all properties from the view definition
698 /// that have currently been requested from the view itself
700 /// <param name="op">current ScanViewOp</param>
701 /// <param name="n">current node</param>
702 public override void Visit(ScanViewOp op, Node n)
704 PlanCompiler.Assert(op.Table.Columns.Count == 1, "ScanViewOp with multiple columns?");
705 Var columnVar = op.Table.Columns[0];
706 PropertyRefList columnProps = GetPropertyRefList(columnVar);
708 Var inputVar = NominalTypeEliminator.GetSingletonVar(n.Child0);
709 PlanCompiler.Assert(inputVar != null, "cannot determine single Var from ScanViewOp's input");
711 AddPropertyRefs(inputVar, columnProps.Clone());
718 /// UnionAllOp handling
719 /// IntersectOp handling
720 /// ExceptOp handling
722 /// Visitor for a SetOp. Pushes desired properties to the corresponding
723 /// Vars of the input
725 /// <param name="op">the setop</param>
726 /// <param name="n"></param>
727 protected override void VisitSetOp(SetOp op, Node n)
729 foreach (VarMap varMap in op.VarMap)
730 foreach (KeyValuePair<Var, Var> kv in varMap)
732 if (TypeUtils.IsStructuredType(kv.Key.Type))
734 // Get the set of expected properties for the unionVar, and
735 // push it down to the inputvars
736 // For Intersect and ExceptOps, we need all properties
738 // We call GetPropertyRefList() always to initialize
739 // the map, even though we may not use it
741 PropertyRefList myProps = GetPropertyRefList(kv.Key);
742 if (op.OpType == OpType.Intersect || op.OpType == OpType.Except)
744 myProps = PropertyRefList.All;
745 // We "want" all properties even on the output of the setop
746 AddPropertyRefs(kv.Key, myProps);
750 myProps = myProps.Clone();
752 AddPropertyRefs(kv.Value, myProps);
761 /// First, "request" that for any sort key that is a structured type, we
762 /// need all its properties. Then process any local definitions, and
763 /// finally the relop input
765 /// <param name="op"></param>
766 /// <param name="n"></param>
767 protected override void VisitSortOp(SortBaseOp op, Node n)
769 // foreach sort key, every single bit of the Var is needed
770 foreach (InternalTrees.SortKey sk in op.Keys)
771 if (TypeUtils.IsStructuredType(sk.Var.Type))
772 AddPropertyRefs(sk.Var, PropertyRefList.All);
774 // if the sort has any local definitions, process those first
777 // then process the relop input
782 /// UnnestOp handling
784 /// <param name="op"></param>
785 /// <param name="n"></param>
786 public override void Visit(UnnestOp op, Node n)
796 /// PhysicalProjectOp handling
798 /// <param name="op"></param>
799 /// <param name="n"></param>
800 public override void Visit(PhysicalProjectOp op, Node n)
802 // Insist that we need all properties from all the outputs
803 foreach (Var v in op.Outputs)
805 if (TypeUtils.IsStructuredType(v.Type))
807 AddPropertyRefs(v, PropertyRefList.All);
811 // simply visit the children
816 /// MultiStreamNestOp handling
818 /// <param name="op"></param>
819 /// <param name="n"></param>
820 public override void Visit(MultiStreamNestOp op, Node n)
822 // Cannot occur at this stage of processing
823 throw EntityUtil.NotSupported();
827 /// SingleStreamNestOp handling
829 /// <param name="op"></param>
830 /// <param name="n"></param>
831 public override void Visit(SingleStreamNestOp op, Node n)
833 // Cannot occur at this stage of processing
834 throw EntityUtil.NotSupported();