4659 lines
240 KiB
C#
4659 lines
240 KiB
C#
/*
|
||
Copyright (c) Microsoft Corporation
|
||
|
||
All rights reserved.
|
||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||
compliance with the License. You may obtain a copy of the License
|
||
at http://www.apache.org/licenses/LICENSE-2.0
|
||
|
||
|
||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||
|
||
|
||
See the Apache Version 2.0 License for specific language governing permissions and
|
||
limitations under the License.
|
||
|
||
*/
|
||
|
||
//
|
||
// <20> Microsoft Corporation. All rights reserved.
|
||
//
|
||
using System;
|
||
using System.Collections;
|
||
using System.Collections.Generic;
|
||
using System.Collections.ObjectModel;
|
||
using System.Text;
|
||
using System.IO;
|
||
using System.Reflection;
|
||
using System.Threading;
|
||
using System.Linq;
|
||
using System.Linq.Expressions;
|
||
using System.CodeDom;
|
||
using System.Xml;
|
||
using System.Diagnostics;
|
||
|
||
using Microsoft.Research.DryadLinq.Internal;
|
||
using Microsoft.Hpc.Dryad;
|
||
using System.Globalization;
|
||
|
||
namespace Microsoft.Research.DryadLinq
|
||
{
|
||
/// <summary>
|
||
/// This class handles code generation for multiple queries and execution invocation.
|
||
/// </summary>
|
||
internal class HpcLinqQueryGen
|
||
{
|
||
internal const int StartPhaseId = -4;
|
||
private const string DryadLinqProgram = "DryadLinqProgram__.xml";
|
||
private const string QueryGraph = "QueryGraph__.txt";
|
||
private const string VertexHostExe = "VertexHost.exe";
|
||
|
||
private static int s_uniqueProgId = -1;
|
||
private static object s_queryGenLock = new Object();
|
||
|
||
private int m_currentPhaseId = StartPhaseId;
|
||
private int m_nextVertexId = 0;
|
||
private bool m_codeGenDone = false;
|
||
private HpcLinqCodeGen m_codeGen;
|
||
private Expression[] m_queryExprs;
|
||
private DryadQueryNode[] m_queryPlan1;
|
||
private DryadQueryNode[] m_queryPlan2;
|
||
private DryadQueryNode[] m_queryPlan3;
|
||
private string[] m_outputTableUris;
|
||
private bool[] m_isTempOutput;
|
||
private string[] m_outputDatapaths;
|
||
private Type[] m_outputTypes;
|
||
private QueryNodeInfo[] m_queryNodeInfos;
|
||
private DryadLinqQuery[] m_outputTables;
|
||
private string m_DryadLinqProgram;
|
||
private string m_queryGraph;
|
||
private Dictionary<Expression, QueryNodeInfo> m_exprNodeInfoMap;
|
||
private Dictionary<Expression, QueryNodeInfo> m_referencedQueryMap;
|
||
private Dictionary<string, DryadInputNode> m_inputUriMap;
|
||
private Dictionary<string, DryadOutputNode> m_outputUriMap;
|
||
private JobExecutor queryExecutor;
|
||
private HpcLinqContext m_context;
|
||
|
||
internal HpcLinqQueryGen(HpcLinqContext context,
|
||
VertexCodeGen vertexCodeGen,
|
||
Expression queryExpr,
|
||
string tableUri,
|
||
bool isTempOutput)
|
||
{
|
||
this.m_queryExprs = new Expression[] { queryExpr };
|
||
string fullTableUri = tableUri;
|
||
this.m_outputTableUris = new string[] { fullTableUri };
|
||
this.m_isTempOutput = new bool[] { isTempOutput };
|
||
this.m_context = context;
|
||
this.Initialize(vertexCodeGen);
|
||
}
|
||
|
||
// This constructor is specifically to support Materialize() calls.
|
||
// it assumes that the Expressions all terminate with a ToDsc node.
|
||
internal HpcLinqQueryGen(HpcLinqContext context,
|
||
VertexCodeGen vertexCodeGen,
|
||
Expression[] qlist)
|
||
{
|
||
this.m_queryExprs = new Expression[qlist.Length];
|
||
this.m_outputTableUris = new string[qlist.Length];
|
||
this.m_isTempOutput = new bool[qlist.Length];
|
||
this.m_context = context;
|
||
for (int i = 0; i < this.m_queryExprs.Length; i++)
|
||
{
|
||
MethodCallExpression mcExpr = (MethodCallExpression)qlist[i];
|
||
string tableUri;
|
||
this.m_queryExprs[i] = mcExpr.Arguments[0];
|
||
|
||
|
||
//this block supports the scenario: q-nonToDsc
|
||
if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_AnonymousDscPlaceholder)
|
||
{
|
||
ExpressionSimplifier<string> e1 = new ExpressionSimplifier<string>();
|
||
tableUri = e1.Eval(mcExpr.Arguments[1]);
|
||
this.m_isTempOutput[i] = true;
|
||
}
|
||
|
||
//this block supports the scenario: q.ToDsc()
|
||
else if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_ToDscWorker)
|
||
{
|
||
DscService dsc = context.DscService;
|
||
ExpressionSimplifier<string> e2 = new ExpressionSimplifier<string>();
|
||
string streamName = e2.Eval(mcExpr.Arguments[2]);
|
||
|
||
tableUri = DataPath.MakeDscStreamUri(dsc, streamName);
|
||
this.m_isTempOutput[i] = false;
|
||
}
|
||
|
||
//this block supports the scenario: q.ToHdfs()
|
||
else if (mcExpr.Method.Name == ReflectedNames.DryadLinqIQueryable_ToHdfsWorker)
|
||
{
|
||
string hdfsHeadNode = context.HdfsService;
|
||
ExpressionSimplifier<string> e2 = new ExpressionSimplifier<string>();
|
||
string streamName = e2.Eval(mcExpr.Arguments[2]);
|
||
|
||
tableUri = DataPath.MakeHdfsStreamUri(hdfsHeadNode, streamName);
|
||
this.m_isTempOutput[i] = false;
|
||
}
|
||
else {
|
||
throw new InvalidOperationException(); // should not occur.
|
||
}
|
||
|
||
this.m_outputTableUris[i] = tableUri;
|
||
|
||
}
|
||
this.Initialize(vertexCodeGen);
|
||
}
|
||
|
||
private void Initialize(VertexCodeGen vertexCodeGen)
|
||
{
|
||
this.m_codeGen = new HpcLinqCodeGen(this.m_context, vertexCodeGen);
|
||
this.m_queryPlan1 = null;
|
||
this.m_queryPlan2 = null;
|
||
this.m_queryPlan3 = null;
|
||
this.m_DryadLinqProgram = null;
|
||
this.m_queryPlan1 = null;
|
||
this.m_exprNodeInfoMap = new Dictionary<Expression, QueryNodeInfo>();
|
||
this.m_referencedQueryMap = new Dictionary<Expression, QueryNodeInfo>();
|
||
this.m_inputUriMap = new Dictionary<string, DryadInputNode>();
|
||
this.m_outputUriMap = new Dictionary<string, DryadOutputNode>();
|
||
this.queryExecutor = new JobExecutor(this.m_context);
|
||
|
||
// Initialize the data structures for the output tables
|
||
this.m_outputTypes = new Type[this.m_queryExprs.Length];
|
||
this.m_outputDatapaths = new string[this.m_queryExprs.Length];
|
||
this.m_queryNodeInfos = new QueryNodeInfo[this.m_queryExprs.Length];
|
||
|
||
for (int i = 0; i < this.m_queryExprs.Length; i++)
|
||
{
|
||
this.m_queryNodeInfos[i] = this.BuildNodeInfoGraph(this.m_queryExprs[i]);
|
||
this.m_queryNodeInfos[i] = new QueryNodeInfo(this.m_queryExprs[i], false, this.m_queryNodeInfos[i]);
|
||
|
||
this.m_outputDatapaths[i] = DataPath.GetDataPath(this.m_outputTableUris[i]);
|
||
|
||
Dictionary<string, string> args = DataPath.GetArguments(this.m_outputTableUris[i]);
|
||
|
||
if (!(DataPath.IsDsc(this.m_outputDatapaths[i]) || DataPath.IsHdfs(this.m_outputDatapaths[i])))
|
||
{
|
||
throw new DryadLinqException(HpcLinqErrorCode.UnrecognizedDataSource,
|
||
String.Format(SR.UnrecognizedDataSource, this.m_outputTableUris[i]));
|
||
}
|
||
}
|
||
}
|
||
|
||
internal HpcLinqContext Context
|
||
{
|
||
get { return this.m_context; }
|
||
}
|
||
|
||
internal HpcLinqCodeGen CodeGen
|
||
{
|
||
get { return this.m_codeGen; }
|
||
}
|
||
|
||
internal Dictionary<Expression, QueryNodeInfo> ReferencedQueryMap
|
||
{
|
||
get { return this.m_referencedQueryMap; }
|
||
}
|
||
|
||
/// <summary>
|
||
/// Probes the running assembly and its dependencies, and throws an exception
|
||
/// if any of them is targetted to x86. Returns silently if all managed assemblies
|
||
/// in the list are x64 or AnyCPU. Native or unloadable binaries are ignored.
|
||
/// </summary>
|
||
private void CheckAssemblyArchitectures()
|
||
{
|
||
// First create the list of assemblies to probe
|
||
// We use a stripped down version of the resource discovery logic in GenerateDryadProgram.
|
||
// i) We start with the same set of currently loaded binaries
|
||
// (== client app + its dependencies + dynamically loaded assemblies)
|
||
// ii) We take out user specified resource exclusions (this enables a workaround
|
||
// for x86 assemblies that must be loaded on the client side, think UI plugins,
|
||
// but aren't needed by the vertex code)
|
||
//
|
||
// The difference is we don't add the vertex DLL, or user resources.
|
||
List<string> resourcesToExclude = new List<string>();
|
||
resourcesToExclude.AddRange(this.m_context.Configuration.ResourcesToRemove.Select(x => x.ToLower(CultureInfo.InvariantCulture)));
|
||
|
||
IEnumerable<string> loadedAssemblyPaths = TypeSystem.GetLoadedNonSystemAssemblyPaths().Select(x => x.ToLower(CultureInfo.InvariantCulture));
|
||
var asembliesToCheck = loadedAssemblyPaths.Where(path => !resourcesToExclude.Contains(path));
|
||
foreach (string path in asembliesToCheck)
|
||
{
|
||
Assembly asm = null;
|
||
try
|
||
{
|
||
asm = Assembly.ReflectionOnlyLoadFrom(path);
|
||
}
|
||
catch
|
||
{
|
||
// silently ignore load errors
|
||
}
|
||
|
||
if (asm != null)
|
||
{
|
||
PortableExecutableKinds peKind;
|
||
ImageFileMachine machine;
|
||
asm.ManifestModule.GetPEKind(out peKind, out machine);
|
||
|
||
// machine will always be reported as "I386" for both true x86 and AnyCPU assemblies
|
||
// peKind will have the "Required32Bit" flag set only for x86 binaries. Therefore we use peKind to make our decision.
|
||
if ((peKind & PortableExecutableKinds.Required32Bit) != 0)
|
||
{
|
||
string offendingAssemblyName = Path.GetFileName(path);
|
||
throw new DryadLinqException(HpcLinqErrorCode.Binaries32BitNotSupported,
|
||
String.Format(SR.Binaries32BitNotSupported, offendingAssemblyName));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
internal DryadLinqQuery[] InvokeDryad()
|
||
{
|
||
lock (s_queryGenLock)
|
||
{
|
||
this.GenerateDryadProgram();
|
||
|
||
CheckAssemblyArchitectures();
|
||
|
||
// Invoke the background execution
|
||
this.queryExecutor.ExecuteAsync(this.m_DryadLinqProgram);
|
||
|
||
// Create the resulting partitioned tables
|
||
this.m_outputTables = new DryadLinqQuery[this.m_outputTableUris.Length];
|
||
MethodInfo minfo = typeof(Microsoft.Research.DryadLinq.DataProvider).GetMethod(
|
||
ReflectedNames.DataProvider_GetPartitionedTable,
|
||
BindingFlags.NonPublic | BindingFlags.Static);
|
||
for (int i = 0; i < this.m_outputTableUris.Length; i++)
|
||
{
|
||
MethodInfo minfo1 = minfo.MakeGenericMethod(this.m_outputTypes[i]);
|
||
object[] args = new object[] { this.m_context, this.m_outputTableUris[i] };
|
||
this.m_outputTables[i] = (DryadLinqQuery)minfo1.Invoke(null, args);
|
||
this.m_outputTables[i].QueryExecutor = this.queryExecutor;
|
||
}
|
||
return this.m_outputTables;
|
||
}
|
||
}
|
||
|
||
// Phase 1 of the query optimization
|
||
internal void GenerateQueryPlanPhase1()
|
||
{
|
||
if (this.m_queryPlan1 != null) return;
|
||
|
||
// Apply some simple rewrite rules
|
||
SimpleRewriter rewriter = new SimpleRewriter(this.m_exprNodeInfoMap.Values.ToList());
|
||
rewriter.Rewrite();
|
||
|
||
// Generate the query plan of phase1
|
||
var referencedNodes = this.m_referencedQueryMap.Values;
|
||
this.m_queryPlan1 = new DryadQueryNode[this.m_queryExprs.Length + referencedNodes.Count];
|
||
for (int i = 0; i < this.m_queryExprs.Length; i++)
|
||
{
|
||
this.m_queryPlan1[i] = this.Visit(this.m_queryNodeInfos[i].children[0].child);
|
||
}
|
||
int idx = this.m_queryExprs.Length;
|
||
foreach (QueryNodeInfo nodeInfo in referencedNodes)
|
||
{
|
||
// Add a Tee'd Merge
|
||
this.m_queryPlan1[idx] = this.Visit(nodeInfo.children[0].child);
|
||
DryadQueryNode mergeNode = new DryadMergeNode(true, false, nodeInfo.queryExpression,
|
||
this.m_queryPlan1[idx]);
|
||
this.m_queryPlan1[idx] = new DryadTeeNode(mergeNode.OutputTypes[0], true,
|
||
mergeNode.QueryExpression, mergeNode);
|
||
nodeInfo.queryNode = this.m_queryPlan1[idx];
|
||
idx++;
|
||
}
|
||
|
||
// Finally, add the output nodes.
|
||
Dictionary<DryadQueryNode, int> forkCounts = new Dictionary<DryadQueryNode, int>();
|
||
for (int i = 0; i < this.m_queryExprs.Length; i++)
|
||
{
|
||
DryadQueryNode queryNode = this.m_queryPlan1[i];
|
||
int cnt;
|
||
if (!forkCounts.TryGetValue(queryNode, out cnt))
|
||
{
|
||
cnt = queryNode.Parents.Count;
|
||
}
|
||
forkCounts[queryNode] = cnt + 1;
|
||
}
|
||
|
||
for (int i = 0; i < this.m_queryExprs.Length; i++)
|
||
{
|
||
HpcClientSideLog.Add("Query " + i + " Output: " + this.m_outputDatapaths[i]);
|
||
|
||
DryadQueryNode queryNode = this.m_queryPlan1[i];
|
||
if (TypeSystem.IsAnonymousType(queryNode.OutputTypes[0]))
|
||
{
|
||
throw new DryadLinqException(HpcLinqErrorCode.OutputTypeCannotBeAnonymous,
|
||
SR.OutputTypeCannotBeAnonymous);
|
||
}
|
||
|
||
// Add dummy Apply to make Dryad happy (it doesn't like to hook inputs straight to outputs)
|
||
if ((queryNode is DryadInputNode) || (forkCounts[queryNode] > 1))
|
||
{
|
||
// Add a dummy Apply
|
||
Type paramType = typeof(IEnumerable<>).MakeGenericType(queryNode.OutputTypes[0]);
|
||
ParameterExpression param = Expression.Parameter(paramType, "x");
|
||
Type type = typeof(Func<,>).MakeGenericType(paramType, paramType);
|
||
LambdaExpression applyExpr = Expression.Lambda(type, param, param);
|
||
DryadQueryNode applyNode = new DryadApplyNode(applyExpr, this.m_queryExprs[i], queryNode);
|
||
applyNode.OutputDataSetInfo = queryNode.OutputDataSetInfo;
|
||
queryNode = applyNode;
|
||
}
|
||
|
||
if (queryNode is DryadConcatNode)
|
||
{
|
||
// Again, we add dummy Apply in certain cases to make Dryad happy
|
||
((DryadConcatNode)queryNode).FixInputs();
|
||
}
|
||
|
||
// Add the output node
|
||
DscCompressionScheme outputScheme = this.m_context.Configuration.OutputDataCompressionScheme;
|
||
DryadOutputNode outputNode = new DryadOutputNode(this.m_context,
|
||
this.m_outputDatapaths[i],
|
||
this.m_isTempOutput[i],
|
||
outputScheme,
|
||
this.m_queryExprs[i],
|
||
queryNode);
|
||
|
||
this.m_queryPlan1[i] = outputNode;
|
||
|
||
if (this.m_outputUriMap.ContainsKey(this.m_outputDatapaths[i].ToLower()))
|
||
{
|
||
throw new DryadLinqException(HpcLinqErrorCode.MultipleOutputsWithSameDscUri,
|
||
String.Format(SR.MultipleOutputsWithSameDscUri, this.m_outputDatapaths[i]));
|
||
}
|
||
|
||
this.m_outputUriMap.Add(this.m_outputDatapaths[i].ToLower(), outputNode);
|
||
this.m_outputTypes[i] = this.m_queryPlan1[i].OutputTypes[0];
|
||
|
||
// Remove useless Tees to make Dryad happy
|
||
if ((queryNode is DryadTeeNode) && (forkCounts[queryNode] == 1))
|
||
{
|
||
DryadQueryNode teeChild = queryNode.Children[0];
|
||
teeChild.UpdateParent(queryNode, outputNode);
|
||
outputNode.UpdateChildren(queryNode, teeChild);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Phase 2 of the query optimization
|
||
internal DryadQueryNode[] GenerateQueryPlanPhase2()
|
||
{
|
||
if (this.m_queryPlan2 == null)
|
||
{
|
||
this.GenerateQueryPlanPhase1();
|
||
this.m_queryPlan2 = new DryadQueryNode[this.m_queryPlan1.Length];
|
||
for (int i = 0; i < this.m_queryPlan1.Length; i++)
|
||
{
|
||
this.m_queryPlan2[i] = this.VisitPhase2(this.m_queryPlan1[i]);
|
||
}
|
||
this.m_currentPhaseId++;
|
||
}
|
||
return this.m_queryPlan2;
|
||
}
|
||
|
||
private DryadQueryNode VisitPhase2(DryadQueryNode node)
|
||
{
|
||
DryadQueryNode resNode = node;
|
||
if (node.m_uniqueId == this.m_currentPhaseId)
|
||
{
|
||
if (node is DryadForkNode)
|
||
{
|
||
// For now, we require every branch of a fork be used:
|
||
DryadForkNode forkNode = (DryadForkNode)node;
|
||
for (int i = 0; i < forkNode.Parents.Count; i++)
|
||
{
|
||
if ((forkNode.Parents[i] is DryadTeeNode) &&
|
||
(forkNode.Parents[i].Parents.Count == 0))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.BranchOfForkNotUsed,
|
||
string.Format(SR.BranchOfForkNotUsed, i),
|
||
node.QueryExpression);
|
||
}
|
||
}
|
||
}
|
||
|
||
resNode = node.SuperNode;
|
||
if (resNode == null)
|
||
{
|
||
for (int i = 0; i < node.Children.Length; i++)
|
||
{
|
||
node.Children[i] = this.VisitPhase2(node.Children[i]);
|
||
}
|
||
resNode = node.PipelineReduce();
|
||
resNode.m_uniqueId++;
|
||
|
||
// Insert a Tee node if needed:
|
||
DryadQueryNode outputNode = resNode.OutputNode;
|
||
if (outputNode.IsForked &&
|
||
!(outputNode is DryadForkNode) &&
|
||
!(outputNode is DryadTeeNode))
|
||
{
|
||
resNode = resNode.InsertTee(true);
|
||
}
|
||
}
|
||
}
|
||
return resNode;
|
||
}
|
||
|
||
// Phase 3 of the query optimization
|
||
internal DryadQueryNode[] GenerateQueryPlanPhase3()
|
||
{
|
||
if (this.m_queryPlan3 == null)
|
||
{
|
||
this.GenerateQueryPlanPhase2();
|
||
this.m_queryPlan3 = this.m_queryPlan2;
|
||
for (int i = 0; i < this.m_queryPlan2.Length; i++)
|
||
{
|
||
this.VisitPhase3(this.m_queryPlan2[i]);
|
||
}
|
||
this.m_currentPhaseId++;
|
||
}
|
||
return this.m_queryPlan3;
|
||
}
|
||
|
||
private void VisitPhase3(DryadQueryNode node)
|
||
{
|
||
if (node.m_uniqueId == this.m_currentPhaseId)
|
||
{
|
||
node.m_uniqueId++;
|
||
|
||
// Remove some useless Tee nodes
|
||
foreach (DryadQueryNode child in node.Children)
|
||
{
|
||
if ((child is DryadTeeNode) && !child.IsForked)
|
||
{
|
||
DryadQueryNode teeChild = child.Children[0];
|
||
teeChild.UpdateParent(child, node);
|
||
node.UpdateChildren(child, teeChild);
|
||
}
|
||
}
|
||
|
||
// Remove some useless Merge nodes
|
||
if ((node is DryadMergeNode) &&
|
||
!node.IsForked &&
|
||
!(node.Parents[0] is DryadOutputNode) &&
|
||
!node.Children[0].IsDynamic &&
|
||
node.Children[0].PartitionCount == 1)
|
||
{
|
||
node.Children[0].UpdateParent(node, node.Parents[0]);
|
||
node.Parents[0].UpdateChildren(node, node.Children[0]);
|
||
}
|
||
|
||
// Add dynamic managers for tee nodes.
|
||
if ((StaticConfig.DynamicOptLevel & StaticConfig.NoDynamicOpt) != 0 &&
|
||
node is DryadTeeNode &&
|
||
node.DynamicManager.ManagerType == DynamicManagerType.None)
|
||
{
|
||
// insert a dynamic broadcast manager on Tee
|
||
node.DynamicManager = DynamicManager.Broadcast;
|
||
}
|
||
|
||
// Recurse on the children of node
|
||
foreach (DryadQueryNode child in node.Children)
|
||
{
|
||
this.VisitPhase3(child);
|
||
}
|
||
}
|
||
}
|
||
|
||
// The main method that generates the query plan.
|
||
internal DryadQueryNode[] QueryPlan()
|
||
{
|
||
return this.GenerateQueryPlanPhase3();
|
||
}
|
||
|
||
// Generate the vertex code for all the query nodes.
|
||
internal void CodeGenVisit()
|
||
{
|
||
if (!this.m_codeGenDone)
|
||
{
|
||
DryadQueryNode[] optimizedPlan = this.QueryPlan();
|
||
|
||
// make sure none of the outputs share a URI with inputs
|
||
foreach (var kvp in this.m_outputUriMap)
|
||
{
|
||
string outputPath = kvp.Key;
|
||
if(m_inputUriMap.ContainsKey(outputPath))
|
||
{
|
||
throw new DryadLinqException(HpcLinqErrorCode.OutputUriAlsoQueryInput,
|
||
String.Format(SR.OutputUriAlsoQueryInput, outputPath));
|
||
}
|
||
}
|
||
|
||
foreach (DryadQueryNode node in optimizedPlan)
|
||
{
|
||
this.CodeGenVisit(node);
|
||
}
|
||
this.m_currentPhaseId++;
|
||
this.m_codeGenDone = true;
|
||
}
|
||
}
|
||
|
||
private void CodeGenVisit(DryadQueryNode node)
|
||
{
|
||
if (node.m_uniqueId == this.m_currentPhaseId)
|
||
{
|
||
node.m_uniqueId++;
|
||
|
||
// We process the types first so that children will also know about all
|
||
// proxies/mappings that should be used.
|
||
node.CreateCodeAndMappingsForVertexTypes(false);
|
||
|
||
// Recurse on the children
|
||
foreach (DryadQueryNode child in node.Children)
|
||
{
|
||
this.CodeGenVisit(child);
|
||
}
|
||
|
||
if (node.NodeType == QueryNodeType.InputTable)
|
||
{
|
||
// not used as a vertex
|
||
string t = ((DryadInputNode)node).Table.DataSourceUri;
|
||
int index = t.LastIndexOf('/');
|
||
int bk = t.LastIndexOf('\\');
|
||
if (index < bk) index = bk;
|
||
node.m_vertexEntryMethod = t.Substring(index + 1);
|
||
}
|
||
else if (node.NodeType == QueryNodeType.OutputTable)
|
||
{
|
||
// not used as a vertex
|
||
string t = ((DryadOutputNode)node).MetaDataUri;
|
||
int index = t.LastIndexOf('/');
|
||
int bk = t.LastIndexOf('\\');
|
||
if (index < bk) index = bk;
|
||
int len = Math.Min(8, t.Length - index - 1);
|
||
node.m_vertexEntryMethod = t.Substring(index + 1, len);
|
||
}
|
||
else if (node.NodeType == QueryNodeType.Tee)
|
||
{
|
||
// not used as a vertex
|
||
node.m_vertexEntryMethod = HpcLinqCodeGen.MakeUniqueName("Tee");
|
||
// broadcast manager code generation
|
||
if (node.DynamicManager.ManagerType != DynamicManagerType.None)
|
||
{
|
||
node.DynamicManager.CreateVertexCode();
|
||
}
|
||
}
|
||
else if (node.NodeType == QueryNodeType.Concat)
|
||
{
|
||
// not used as a vertex
|
||
node.m_vertexEntryMethod = HpcLinqCodeGen.MakeUniqueName("Concat");
|
||
}
|
||
else
|
||
{
|
||
CodeMemberMethod vertexMethod = this.m_codeGen.AddVertexMethod(node);
|
||
node.m_vertexEntryMethod = vertexMethod.Name;
|
||
node.DynamicManager.CreateVertexCode();
|
||
}
|
||
}
|
||
}
|
||
|
||
// Assign unique ids to all the query nodes
|
||
private void AssignUniqueId()
|
||
{
|
||
if (this.m_currentPhaseId != -1)
|
||
{
|
||
//@@TODO: this should not be reachable. could change to Assert/InvalidOpEx
|
||
throw new DryadLinqException(HpcLinqErrorCode.Internal,
|
||
"Internal error: Optimization phase should be -1, not " +
|
||
this.m_currentPhaseId);
|
||
}
|
||
foreach (DryadQueryNode node in this.QueryPlan())
|
||
{
|
||
this.AssignUniqueId(node);
|
||
}
|
||
}
|
||
|
||
private void AssignUniqueId(DryadQueryNode node)
|
||
{
|
||
if (node.m_uniqueId == this.m_currentPhaseId)
|
||
{
|
||
foreach (Pair<string, DryadQueryNode> refChild in node.GetReferencedQueries())
|
||
{
|
||
this.AssignUniqueId(refChild.Value);
|
||
}
|
||
foreach (DryadQueryNode child in node.Children)
|
||
{
|
||
this.AssignUniqueId(child);
|
||
}
|
||
if (node.m_uniqueId == this.m_currentPhaseId)
|
||
{
|
||
node.m_uniqueId = this.m_nextVertexId++;
|
||
|
||
if (node.OutputNode is DryadForkNode)
|
||
{
|
||
foreach (DryadQueryNode pnode in node.Parents)
|
||
{
|
||
if (pnode.m_uniqueId == this.m_currentPhaseId)
|
||
{
|
||
pnode.m_uniqueId = this.m_nextVertexId++;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
private void CreateQueryPlan(XmlDocument queryDoc, XmlElement queryPlan)
|
||
{
|
||
this.AssignUniqueId();
|
||
|
||
HashSet<int> seen = new HashSet<int>();
|
||
foreach (DryadQueryNode node in this.QueryPlan())
|
||
{
|
||
node.AddToQueryPlan(queryDoc, queryPlan, seen);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Find the pdb file associated with a given filename.
|
||
/// </summary>
|
||
/// <param name="filename">Filename with debugging information.</param>
|
||
/// <returns>The associated pdb.</returns>
|
||
internal static string FindPDB(string filename)
|
||
{
|
||
string basename = Path.GetFileNameWithoutExtension(filename);
|
||
string directory = Path.GetDirectoryName(filename);
|
||
string pdbname = directory + Path.DirectorySeparatorChar + basename + ".pdb";
|
||
return (File.Exists(pdbname)) ? pdbname : null;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Add a resource to the Xml plan.
|
||
/// </summary>
|
||
/// <param name="queryDoc">Document holding the xml plan.</param>
|
||
/// <param name="parent">Parent node.</param>
|
||
/// <param name="resource">Resource to add.</param>
|
||
/// <returns>Handle to the inserted node.</returns>
|
||
private void AddResourceToPlan(XmlDocument queryDoc,
|
||
XmlElement parent,
|
||
string resource,
|
||
IEnumerable<string> resourcesToExclude)
|
||
{
|
||
AddResourceToPlan_Core(queryDoc, parent, resource, resourcesToExclude);
|
||
|
||
if (this.m_context.Configuration.CompileForVertexDebugging)
|
||
{
|
||
string pdb = FindPDB(resource);
|
||
if (pdb != null)
|
||
{
|
||
AddResourceToPlan_Core(queryDoc, parent, pdb, resourcesToExclude);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Add a resource to the plan unless it was specifically in the exclusions list.
|
||
private void AddResourceToPlan_Core(XmlDocument queryDoc,
|
||
XmlElement parent,
|
||
string resource,
|
||
IEnumerable<string> resourcesToExclude)
|
||
{
|
||
if (resourcesToExclude.Contains(resource, StringComparer.OrdinalIgnoreCase))
|
||
{
|
||
return;
|
||
}
|
||
XmlElement resourceElem = queryDoc.CreateElement("Resource");
|
||
resourceElem.InnerText = queryExecutor.AddResource(resource);
|
||
parent.AppendChild(resourceElem);
|
||
}
|
||
|
||
private void GenerateAppConfigResource(string appConfigPath)
|
||
{
|
||
// Generates an app config XML for the VertexHost which
|
||
//
|
||
// 1) specifies the server GC mode
|
||
//
|
||
// 2) requests a .NET runtime version equal to that of the client
|
||
// submitting the job (if the client has a .NET version higher than 3.5)
|
||
// also specifies <supportedRuntime version="v2.0.50727"/> as a fallback
|
||
// in case the cluster nodes don't have the client's newer .Net version
|
||
// (v2.0.50727 corresponds to .Net 3.5, both Sp1 and Sp2 pointing to this
|
||
// version ID, since it's actually the underlying CLR's version). This rule
|
||
// only kicks in if the query config has the MatchClientNetFrameworkVersion flag set.
|
||
//
|
||
// 3) disables Authenticode checks for the VH by specifying
|
||
// <generatePublisherEvidence enabled="false"/>. This is necessary
|
||
// because some cluster nodes don't have an Internet connection, in which
|
||
// case which the Authenticode check leads to a 20sec delay during process startup.
|
||
//
|
||
// NOTE: useLegacyV2RuntimeActivationPolicy="true" is needed becuase the VH binary is mixed mode.
|
||
string clientVersionString = "";
|
||
if (Environment.Version.Major > 2 && this.m_context.Configuration.MatchClientNetFrameworkVersion)
|
||
{
|
||
clientVersionString = String.Format(CultureInfo.InvariantCulture,
|
||
@" <supportedRuntime version=""v{0}.{1}""/>",
|
||
Environment.Version.Major,
|
||
Environment.Version.Minor);
|
||
// NOTE: We use the "v4.0" syntax instead of the longer "v4.0.30319" because as of .NET4
|
||
// the format of this app config tag has been simplified to "major.minor"
|
||
}
|
||
|
||
string appConfigBody =
|
||
@"<?xml version =""1.0""?>
|
||
<configuration>
|
||
<runtime>
|
||
<gcServer enabled=""true""/>
|
||
<gcConcurrent enabled=""false""/>
|
||
<generatePublisherEvidence enabled=""false""/>
|
||
</runtime>
|
||
|
||
<startup useLegacyV2RuntimeActivationPolicy=""true"">
|
||
";
|
||
|
||
appConfigBody += clientVersionString; // add the client specific version string if we generated one.
|
||
appConfigBody +=
|
||
@" <supportedRuntime version=""v2.0.50727""/>
|
||
</startup>
|
||
</configuration>
|
||
";
|
||
|
||
File.WriteAllText(appConfigPath, appConfigBody);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Generate the executable code.
|
||
/// </summary>
|
||
/// <returns>The path to the queryPlanXml location</returns>
|
||
internal string GenerateDryadProgram()
|
||
{
|
||
HpcLinqObjectStore.Clear();
|
||
|
||
// Any resource that we try to add will be tested against this list.
|
||
IEnumerable<string> resourcesToExclude = this.m_context.Configuration.ResourcesToRemove;
|
||
|
||
// BuildDryadLinqAssembly:
|
||
// 1. Performs query optimizations
|
||
// 2. Generate vertex code for all the nodes in the query plan
|
||
this.m_codeGen.BuildDryadLinqAssembly(this);
|
||
|
||
DryadQueryExplain explain = new DryadQueryExplain();
|
||
HpcClientSideLog.Add("{0}", explain.Explain(this));
|
||
|
||
// Finally, write out the query plan.
|
||
if (this.m_DryadLinqProgram == null)
|
||
{
|
||
int progId = Interlocked.Increment(ref s_uniqueProgId);
|
||
this.m_DryadLinqProgram = HpcLinqCodeGen.GetPathForGeneratedFile(DryadLinqProgram, progId);
|
||
this.m_queryGraph = HpcLinqCodeGen.GetPathForGeneratedFile(QueryGraph, progId);
|
||
}
|
||
|
||
XmlDocument queryDoc = new XmlDocument();
|
||
queryDoc.LoadXml("<Query></Query>");
|
||
|
||
// Write the assembly version information
|
||
Assembly dryadlinqassembly = Assembly.GetExecutingAssembly();
|
||
AssemblyName asmName = dryadlinqassembly.GetName();
|
||
XmlElement elem = queryDoc.CreateElement("DryadLinqVersion");
|
||
elem.InnerText = asmName.Version.ToString();
|
||
queryDoc.DocumentElement.AppendChild(elem);
|
||
|
||
// Add the cluster name element
|
||
elem = queryDoc.CreateElement("ClusterName");
|
||
elem.InnerText = this.m_context.Configuration.HeadNode;
|
||
queryDoc.DocumentElement.AppendChild(elem);
|
||
|
||
// Add the minimum number of nodes
|
||
int minComputeNodes = 1;
|
||
if (this.m_context.Configuration.JobMinNodes.HasValue)
|
||
{
|
||
minComputeNodes = this.m_context.Configuration.JobMinNodes.Value;
|
||
}
|
||
elem = queryDoc.CreateElement("MinimumComputeNodes");
|
||
elem.InnerText = minComputeNodes.ToString();
|
||
queryDoc.DocumentElement.AppendChild(elem);
|
||
|
||
// Add the maximum number of nodes
|
||
int maxComputeNodes = -1;
|
||
if (this.m_context.Configuration.JobMaxNodes.HasValue)
|
||
{
|
||
maxComputeNodes = this.m_context.Configuration.JobMaxNodes.Value;
|
||
}
|
||
elem = queryDoc.CreateElement("MaximumComputeNodes");
|
||
elem.InnerText = maxComputeNodes.ToString();
|
||
queryDoc.DocumentElement.AppendChild(elem);
|
||
|
||
// intermediate data compression
|
||
elem = queryDoc.CreateElement("IntermediateDataCompression");
|
||
elem.InnerText = ((int)this.m_context.Configuration.IntermediateDataCompressionScheme).ToString();
|
||
queryDoc.DocumentElement.AppendChild(elem);
|
||
|
||
// Speculative Duplication Node
|
||
elem = queryDoc.CreateElement("EnableSpeculativeDuplication");
|
||
elem.InnerText = this.m_context.Configuration.EnableSpeculativeDuplication.ToString();
|
||
queryDoc.DocumentElement.AppendChild(elem);
|
||
|
||
// Add the visualization element
|
||
//@@TODO[p2]: remove this element from the queryXML.
|
||
elem = queryDoc.CreateElement("Visualization");
|
||
elem.InnerText = "none";
|
||
queryDoc.DocumentElement.AppendChild(elem);
|
||
|
||
// Add the query name element
|
||
elem = queryDoc.CreateElement("QueryName");
|
||
if (String.IsNullOrEmpty(this.m_context.Configuration.JobFriendlyName))
|
||
{
|
||
elem.InnerText = asmName.Name;
|
||
}
|
||
else
|
||
{
|
||
elem.InnerText = this.m_context.Configuration.JobFriendlyName;
|
||
}
|
||
queryDoc.DocumentElement.AppendChild(elem);
|
||
|
||
|
||
|
||
// Add the XmlExecHostArgs element
|
||
elem = queryDoc.CreateElement("XmlExecHostArgs");
|
||
queryDoc.DocumentElement.AppendChild(elem);
|
||
|
||
// Add an element for each argument
|
||
string[] args = StaticConfig.XmlExecHostArgs.Split(new char[] {' '}, StringSplitOptions.RemoveEmptyEntries);
|
||
List<string> xmlExecResources = new List<string>();
|
||
for (int i = 0; i < args.Length; ++i)
|
||
{
|
||
string arg = args[i];
|
||
XmlElement argElem = queryDoc.CreateElement("Argument");
|
||
argElem.InnerText = arg;
|
||
elem.AppendChild(argElem);
|
||
if (arg.Equals("-bw") || arg.Equals("-r"))
|
||
{
|
||
xmlExecResources.Add(args[i+1]);
|
||
}
|
||
}
|
||
|
||
// Add the resources element
|
||
elem = queryDoc.CreateElement("Resources");
|
||
queryDoc.DocumentElement.AppendChild(elem);
|
||
|
||
// Add resource item for this LINQ DLL
|
||
AddResourceToPlan(queryDoc, elem, this.m_codeGen.GetTargetLocation(), resourcesToExclude);
|
||
|
||
// Add resource item for each loaded DLL that isn't a system DLL
|
||
IEnumerable<string> loadedAssemblyPaths = TypeSystem.GetLoadedNonSystemAssemblyPaths();
|
||
foreach(string assemblyPath in loadedAssemblyPaths)
|
||
{
|
||
AddResourceToPlan(queryDoc, elem, assemblyPath, resourcesToExclude);
|
||
}
|
||
|
||
// Add the xmlExec resources
|
||
foreach (string resourcePath in xmlExecResources)
|
||
{
|
||
AddResourceToPlan(queryDoc, elem, resourcePath, resourcesToExclude);
|
||
}
|
||
|
||
// Add codegen resources
|
||
foreach (string resourcePath in this.m_codeGen.VertexCodeGen.GetResources())
|
||
{
|
||
AddResourceToPlan(queryDoc, elem, resourcePath, resourcesToExclude);
|
||
}
|
||
|
||
// Create an app config file for the VertexHost process, and add it to the resources
|
||
string vertexHostAppConfigPath = HpcLinqCodeGen.GetPathForGeneratedFile(Path.ChangeExtension(VertexHostExe, "exe.config"), null);
|
||
GenerateAppConfigResource(vertexHostAppConfigPath);
|
||
AddResourceToPlan(queryDoc, elem, vertexHostAppConfigPath, resourcesToExclude);
|
||
|
||
// Save and add the object store as a resource
|
||
if (!HpcLinqObjectStore.IsEmpty)
|
||
{
|
||
HpcLinqObjectStore.Save();
|
||
AddResourceToPlan(queryDoc, elem, HpcLinqObjectStore.GetClientSideObjectStorePath(), resourcesToExclude);
|
||
}
|
||
|
||
// Add resource item for user-added resources
|
||
foreach (string userResource in this.m_context.Configuration.ResourcesToAdd.Distinct(StringComparer.OrdinalIgnoreCase))
|
||
{
|
||
AddResourceToPlan(queryDoc, elem, userResource, resourcesToExclude);
|
||
}
|
||
|
||
// Add the query plan element
|
||
XmlElement queryPlanElem = queryDoc.CreateElement("QueryPlan");
|
||
queryDoc.DocumentElement.AppendChild(queryPlanElem);
|
||
|
||
// Add the query tree as a sequence of nodes
|
||
this.CreateQueryPlan(queryDoc, queryPlanElem);
|
||
|
||
// Finally, save the DryadQuery doc to a file
|
||
queryDoc.Save(this.m_DryadLinqProgram);
|
||
|
||
return this.m_DryadLinqProgram;
|
||
}
|
||
|
||
private void BuildReferencedQuery(Expression expr)
|
||
{
|
||
ExpressionQuerySet querySet = new ExpressionQuerySet();
|
||
querySet.Visit(expr);
|
||
foreach (Expression qexpr in querySet.QuerySet)
|
||
{
|
||
QueryNodeInfo nodeInfo = BuildNodeInfoGraph(qexpr);
|
||
this.m_referencedQueryMap[qexpr] = new QueryNodeInfo(qexpr, false, nodeInfo);
|
||
}
|
||
}
|
||
|
||
private void BuildReferencedQuery(int startIdx, ReadOnlyCollection<Expression> exprs)
|
||
{
|
||
ExpressionQuerySet querySet = new ExpressionQuerySet();
|
||
for (int i = startIdx; i < exprs.Count; i++)
|
||
{
|
||
querySet.Visit(exprs[i]);
|
||
}
|
||
foreach (Expression qexpr in querySet.QuerySet)
|
||
{
|
||
QueryNodeInfo nodeInfo = BuildNodeInfoGraph(qexpr);
|
||
this.m_referencedQueryMap[qexpr] = new QueryNodeInfo(qexpr, false, nodeInfo);
|
||
}
|
||
}
|
||
|
||
//@@TODO: document what the 'NodeInfo' and 'ReferencedQuery' system does
|
||
private QueryNodeInfo BuildNodeInfoGraph(Expression expression)
|
||
{
|
||
QueryNodeInfo resNodeInfo = null;
|
||
if (this.m_exprNodeInfoMap.TryGetValue(expression, out resNodeInfo))
|
||
{
|
||
return resNodeInfo;
|
||
}
|
||
MethodCallExpression mcExpr = expression as MethodCallExpression;
|
||
if (mcExpr != null && mcExpr.Method.IsStatic && TypeSystem.IsQueryOperatorCall(mcExpr))
|
||
{
|
||
switch (mcExpr.Method.Name)
|
||
{
|
||
case "Join":
|
||
case "GroupJoin":
|
||
case "Union":
|
||
case "Intersect":
|
||
case "Except":
|
||
case "Zip":
|
||
case "SequenceEqual":
|
||
case "SequenceEqualAsQuery":
|
||
{
|
||
QueryNodeInfo child1 = BuildNodeInfoGraph(mcExpr.Arguments[0]);
|
||
QueryNodeInfo child2 = BuildNodeInfoGraph(mcExpr.Arguments[1]);
|
||
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1, child2);
|
||
this.BuildReferencedQuery(2, mcExpr.Arguments);
|
||
break;
|
||
}
|
||
case "Concat":
|
||
{
|
||
NewArrayExpression others = mcExpr.Arguments[1] as NewArrayExpression;
|
||
if (others == null)
|
||
{
|
||
QueryNodeInfo child1 = BuildNodeInfoGraph(mcExpr.Arguments[0]);
|
||
QueryNodeInfo child2 = BuildNodeInfoGraph(mcExpr.Arguments[1]);
|
||
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1, child2);
|
||
}
|
||
else
|
||
{
|
||
QueryNodeInfo[] infos = new QueryNodeInfo[others.Expressions.Count + 1];
|
||
infos[0] = BuildNodeInfoGraph(mcExpr.Arguments[0]);
|
||
for (int i = 0; i < others.Expressions.Count; ++i)
|
||
{
|
||
infos[i + 1] = BuildNodeInfoGraph(others.Expressions[i]);
|
||
}
|
||
resNodeInfo = new QueryNodeInfo(mcExpr, true, infos);
|
||
}
|
||
break;
|
||
}
|
||
case "Apply":
|
||
case "ApplyPerPartition":
|
||
{
|
||
QueryNodeInfo child1 = BuildNodeInfoGraph(mcExpr.Arguments[0]);
|
||
if (mcExpr.Arguments.Count == 2)
|
||
{
|
||
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1);
|
||
this.BuildReferencedQuery(mcExpr.Arguments[1]);
|
||
}
|
||
else
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(mcExpr.Arguments[2]);
|
||
if (lambda.Parameters.Count == 2)
|
||
{
|
||
// Apply with two sources
|
||
QueryNodeInfo child2 = BuildNodeInfoGraph(mcExpr.Arguments[1]);
|
||
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1, child2);
|
||
this.BuildReferencedQuery(mcExpr.Arguments[2]);
|
||
}
|
||
else
|
||
{
|
||
// Apply with multiple sources of the same type
|
||
NewArrayExpression others = (NewArrayExpression)mcExpr.Arguments[1];
|
||
QueryNodeInfo[] infos = new QueryNodeInfo[others.Expressions.Count + 1];
|
||
infos[0] = child1;
|
||
for (int i = 0; i < others.Expressions.Count; ++i)
|
||
{
|
||
infos[i + 1] = BuildNodeInfoGraph(others.Expressions[i]);
|
||
}
|
||
resNodeInfo = new QueryNodeInfo(mcExpr, true, infos);
|
||
this.BuildReferencedQuery(mcExpr.Arguments[2]);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "RangePartition":
|
||
{
|
||
QueryNodeInfo child1 = BuildNodeInfoGraph(mcExpr.Arguments[0]);
|
||
if (mcExpr.Arguments.Count == 6)
|
||
{
|
||
// This is a key part of handling for RangePartition( ... , IQueryable<> keysQuery, ...)
|
||
// The keys expression is established as a child[1] nodeInfo for the rangePartition node.
|
||
QueryNodeInfo child2 = BuildNodeInfoGraph(mcExpr.Arguments[2]);
|
||
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1, child2);
|
||
}
|
||
else
|
||
{
|
||
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1);
|
||
}
|
||
this.BuildReferencedQuery(mcExpr.Arguments[1]);
|
||
break;
|
||
}
|
||
default:
|
||
{
|
||
QueryNodeInfo child1 = BuildNodeInfoGraph(mcExpr.Arguments[0]);
|
||
resNodeInfo = new QueryNodeInfo(mcExpr, true, child1);
|
||
this.BuildReferencedQuery(1, mcExpr.Arguments);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
if (resNodeInfo == null)
|
||
{
|
||
resNodeInfo = new QueryNodeInfo(expression, false);
|
||
}
|
||
this.m_exprNodeInfoMap.Add(expression, resNodeInfo);
|
||
return resNodeInfo;
|
||
}
|
||
|
||
private static bool IsMergeNodeNeeded(DryadQueryNode node)
|
||
{
|
||
return node.IsDynamic || node.PartitionCount > 1;
|
||
}
|
||
|
||
internal DryadQueryNode Visit(QueryNodeInfo nodeInfo)
|
||
{
|
||
Expression expression = nodeInfo.queryExpression;
|
||
if (expression.NodeType == ExpressionType.Call)
|
||
{
|
||
MethodCallExpression mcExpr = (MethodCallExpression)expression;
|
||
if (mcExpr.Method.IsStatic && TypeSystem.IsQueryOperatorCall(mcExpr))
|
||
{
|
||
return this.VisitQueryOperatorCall(nodeInfo);
|
||
}
|
||
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
|
||
String.Format(SR.OperatorNotSupported, mcExpr.Method.Name),
|
||
expression);
|
||
}
|
||
else if (expression.NodeType == ExpressionType.Constant)
|
||
{
|
||
DryadInputNode inputNode = new DryadInputNode(this, (ConstantExpression)expression);
|
||
if (!this.m_inputUriMap.ContainsKey(inputNode.Table.DataSourceUri.ToLower()))
|
||
{
|
||
this.m_inputUriMap.Add(inputNode.Table.DataSourceUri.ToLower(), inputNode);
|
||
}
|
||
return inputNode;
|
||
}
|
||
else
|
||
{
|
||
string errMsg = "Can't handle expression of type " + expression.NodeType;
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.UnsupportedExpressionsType,
|
||
String.Format(SR.UnsupportedExpressionsType,expression.NodeType),
|
||
expression);
|
||
}
|
||
}
|
||
|
||
private static bool IsLambda(Expression expr, int n)
|
||
{
|
||
LambdaExpression lambdaExpr = HpcLinqExpression.GetLambda(expr);
|
||
return (lambdaExpr != null && lambdaExpr.Parameters.Count == n);
|
||
}
|
||
|
||
// Checks if the child of the source node is a groupby node without any result selectors.
|
||
private static bool IsGroupByWithoutResultSelector(Expression source)
|
||
{
|
||
bool isGroupBy = false;
|
||
if (source.NodeType == ExpressionType.Call)
|
||
{
|
||
MethodCallExpression expression = (MethodCallExpression)source;
|
||
if (expression.Method.IsStatic &&
|
||
TypeSystem.IsQueryOperatorCall(expression) &&
|
||
expression.Method.Name == "GroupBy")
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
isGroupBy = true;
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
isGroupBy = !IsLambda(expression.Arguments[2], 2);
|
||
}
|
||
else if (expression.Arguments.Count == 4)
|
||
{
|
||
isGroupBy = !(IsLambda(expression.Arguments[2], 2) ||
|
||
IsLambda(expression.Arguments[3], 2));
|
||
}
|
||
}
|
||
}
|
||
return isGroupBy;
|
||
}
|
||
|
||
private DryadQueryNode CreateOffset(bool isLong, Expression queryExpr, DryadQueryNode child)
|
||
{
|
||
// Count node
|
||
DryadQueryNode countNode = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
|
||
true, false, queryExpr, child);
|
||
|
||
// Apply node for x => Offsets(x)
|
||
Type paramType = typeof(IEnumerable<>).MakeGenericType(typeof(long));
|
||
ParameterExpression param = Expression.Parameter(paramType, "x");
|
||
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod("Offsets");
|
||
Expression body = Expression.Call(minfo, param, Expression.Constant(isLong, typeof(bool)));
|
||
Type type = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
LambdaExpression procFunc = Expression.Lambda(type, body, param);
|
||
DryadQueryNode mergeCountNode = new DryadMergeNode(true, true, queryExpr, countNode);
|
||
DryadQueryNode offsetsNode = new DryadApplyNode(procFunc, queryExpr, mergeCountNode);
|
||
|
||
// HashPartition
|
||
LambdaExpression keySelectExpr = IdentityFunction.Instance(typeof(IndexedValue<long>));
|
||
int pcount = child.OutputPartition.Count;
|
||
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr, null, null, pcount,
|
||
false, queryExpr, offsetsNode);
|
||
DryadQueryNode resNode = new DryadMergeNode(false, true, queryExpr, hdistNode);
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode PromoteConcat(QueryNodeInfo source,
|
||
DryadQueryNode sourceNode,
|
||
Func<DryadQueryNode, DryadQueryNode> func)
|
||
{
|
||
DryadQueryNode resNode = sourceNode;
|
||
if ((resNode is DryadConcatNode) && !source.IsForked)
|
||
{
|
||
DryadQueryNode[] children = resNode.Children;
|
||
DryadQueryNode[] newChildren = new DryadQueryNode[children.Length];
|
||
for (int i = 0; i < children.Length; i++)
|
||
{
|
||
children[i].Parents.Remove(resNode);
|
||
newChildren[i] = func(children[i]);
|
||
}
|
||
resNode = new DryadConcatNode(source.queryExpression, newChildren);
|
||
}
|
||
else
|
||
{
|
||
resNode = func(resNode);
|
||
}
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitSelect(QueryNodeInfo source,
|
||
QueryNodeType nodeType,
|
||
LambdaExpression selector,
|
||
LambdaExpression resultSelector,
|
||
MethodCallExpression queryExpr)
|
||
{
|
||
DryadQueryNode selectNode;
|
||
if (selector.Type.GetGenericArguments().Length == 2)
|
||
{
|
||
// If this select's child is a groupby node, push this select into its child, if
|
||
// 1. The groupby node is not tee'd, and
|
||
// 2. The groupby node has no result selector, and
|
||
// 3. The selector is decomposable
|
||
if (!source.IsForked &&
|
||
IsGroupByWithoutResultSelector(source.queryExpression) &&
|
||
Decomposition.GetDecompositionInfoList(selector, m_codeGen) != null)
|
||
{
|
||
MethodCallExpression expr = (MethodCallExpression)source.queryExpression;
|
||
LambdaExpression keySelectExpr = HpcLinqExpression.GetLambda(expr.Arguments[1]);
|
||
|
||
// Figure out elemSelectExpr and comparerExpr
|
||
LambdaExpression elemSelectExpr = null;
|
||
Expression comparerExpr = null;
|
||
if (expr.Arguments.Count == 3)
|
||
{
|
||
elemSelectExpr = HpcLinqExpression.GetLambda(expr.Arguments[2]);
|
||
if (elemSelectExpr == null)
|
||
{
|
||
comparerExpr = expr.Arguments[2];
|
||
}
|
||
}
|
||
else if (expr.Arguments.Count == 4)
|
||
{
|
||
elemSelectExpr = HpcLinqExpression.GetLambda(expr.Arguments[2]);
|
||
comparerExpr = expr.Arguments[3];
|
||
}
|
||
|
||
// Construct new query expression by building result selector expression
|
||
// and pushing it to groupby node.
|
||
selectNode = VisitGroupBy(source.children[0].child, keySelectExpr,
|
||
elemSelectExpr, selector, comparerExpr, queryExpr);
|
||
if (nodeType == QueryNodeType.SelectMany)
|
||
{
|
||
Type selectorRetType = selector.Type.GetGenericArguments()[1];
|
||
LambdaExpression id = IdentityFunction.Instance(selectorRetType);
|
||
selectNode = new DryadSelectNode(nodeType, id, resultSelector, queryExpr, selectNode);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
selectNode = this.PromoteConcat(
|
||
source, child,
|
||
x => new DryadSelectNode(nodeType, selector, resultSelector, queryExpr, x));
|
||
}
|
||
}
|
||
else
|
||
{
|
||
// The "indexed" version
|
||
DryadQueryNode child = this.Visit(source);
|
||
if (!child.IsDynamic && child.OutputPartition.Count == 1)
|
||
{
|
||
selectNode = this.PromoteConcat(
|
||
source, child,
|
||
x => new DryadSelectNode(nodeType, selector, resultSelector, queryExpr, x));
|
||
}
|
||
else
|
||
{
|
||
child.IsForked = true;
|
||
|
||
// Create (x, y) => Select(x, y, selector)
|
||
Type ptype1 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
|
||
Type ptype2 = typeof(IEnumerable<>).MakeGenericType(typeof(IndexedValue<long>));
|
||
ParameterExpression param1 = Expression.Parameter(ptype1, HpcLinqCodeGen.MakeUniqueName("x"));
|
||
ParameterExpression param2 = Expression.Parameter(ptype2, HpcLinqCodeGen.MakeUniqueName("y"));
|
||
|
||
string methodName = queryExpr.Method.Name;
|
||
Type[] selectorTypeArgs = selector.Type.GetGenericArguments();
|
||
Type typeArg2 = selectorTypeArgs[selectorTypeArgs.Length - 1];
|
||
if (nodeType == QueryNodeType.SelectMany)
|
||
{
|
||
if (resultSelector != null)
|
||
{
|
||
methodName += "Result";
|
||
}
|
||
typeArg2 = typeArg2.GetGenericArguments()[0];
|
||
}
|
||
|
||
string targetMethodName = methodName + "WithStartIndex";
|
||
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod(targetMethodName);
|
||
Expression body;
|
||
if (resultSelector == null)
|
||
{
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0], typeArg2);
|
||
body = Expression.Call(minfo, param1, param2, selector);
|
||
}
|
||
else
|
||
{
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0], typeArg2, resultSelector.Body.Type);
|
||
body = Expression.Call(minfo, param1, param2, selector, resultSelector);
|
||
}
|
||
Type type = typeof(Func<,,>).MakeGenericType(ptype1, ptype2, body.Type);
|
||
LambdaExpression procFunc = Expression.Lambda(type, body, param1, param2);
|
||
|
||
bool isLong = methodName.StartsWith("Long", StringComparison.Ordinal);
|
||
DryadQueryNode offsetNode = this.CreateOffset(isLong, queryExpr, child);
|
||
selectNode = new DryadApplyNode(procFunc, queryExpr, child, offsetNode);
|
||
}
|
||
}
|
||
return selectNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitWhere(QueryNodeInfo source,
|
||
LambdaExpression predicate,
|
||
MethodCallExpression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
DryadQueryNode whereNode;
|
||
if (predicate.Type.GetGenericArguments().Length == 2 ||
|
||
(!child.IsDynamic && child.OutputPartition.Count == 1))
|
||
{
|
||
whereNode = this.PromoteConcat(source, child, x => new DryadWhereNode(predicate, queryExpr, x));
|
||
}
|
||
else
|
||
{
|
||
// The "indexed" version
|
||
// Create (x, y) => DryadWhere(x, y, predicate)
|
||
Type ptype1 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
|
||
Type ptype2 = typeof(IEnumerable<>).MakeGenericType(typeof(IndexedValue<long>));
|
||
ParameterExpression param1 = Expression.Parameter(ptype1, HpcLinqCodeGen.MakeUniqueName("x"));
|
||
ParameterExpression param2 = Expression.Parameter(ptype2, HpcLinqCodeGen.MakeUniqueName("y"));
|
||
string targetMethod = queryExpr.Method.Name + "WithStartIndex";
|
||
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod(targetMethod);
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
|
||
Expression body = Expression.Call(minfo, param1, param2, predicate);
|
||
Type type = typeof(Func<,,>).MakeGenericType(ptype1, ptype2, body.Type);
|
||
LambdaExpression procFunc = Expression.Lambda(type, body, param1, param2);
|
||
|
||
child.IsForked = true;
|
||
bool isLong = (queryExpr.Method.Name == "LongWhere");
|
||
DryadQueryNode offsetNode = this.CreateOffset(isLong, queryExpr, child);
|
||
whereNode = new DryadApplyNode(procFunc, queryExpr, child, offsetNode);
|
||
}
|
||
return whereNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitJoin(QueryNodeInfo outerSource,
|
||
QueryNodeInfo innerSource,
|
||
QueryNodeType nodeType,
|
||
LambdaExpression outerKeySelector,
|
||
LambdaExpression innerKeySelector,
|
||
LambdaExpression resultSelector,
|
||
Expression comparerExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode outerChild = this.Visit(outerSource);
|
||
DryadQueryNode innerChild = this.Visit(innerSource);
|
||
DryadQueryNode joinNode = null;
|
||
|
||
Type keyType = outerKeySelector.Type.GetGenericArguments()[1];
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
|
||
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
|
||
queryExpr);
|
||
}
|
||
|
||
// The comparer object:
|
||
object comparer = null;
|
||
if (comparerExpr != null)
|
||
{
|
||
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
|
||
comparer = evaluator.Eval(comparerExpr);
|
||
}
|
||
|
||
if (outerChild.IsDynamic || innerChild.IsDynamic)
|
||
{
|
||
// Well, let us do the simplest thing for now
|
||
outerChild = new DryadHashPartitionNode(outerKeySelector,
|
||
comparerExpr,
|
||
StaticConfig.DefaultPartitionCount,
|
||
queryExpr,
|
||
outerChild);
|
||
if (IsMergeNodeNeeded(outerChild))
|
||
{
|
||
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
|
||
}
|
||
|
||
innerChild = new DryadHashPartitionNode(innerKeySelector,
|
||
comparerExpr,
|
||
StaticConfig.DefaultPartitionCount,
|
||
queryExpr,
|
||
innerChild);
|
||
if (IsMergeNodeNeeded(innerChild))
|
||
{
|
||
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
|
||
}
|
||
|
||
joinNode = new DryadJoinNode(nodeType,
|
||
"Hash" + nodeType,
|
||
outerKeySelector,
|
||
innerKeySelector,
|
||
resultSelector,
|
||
comparerExpr,
|
||
queryExpr,
|
||
outerChild,
|
||
innerChild);
|
||
return joinNode;
|
||
}
|
||
|
||
|
||
bool isOuterDescending = outerChild.OutputDataSetInfo.orderByInfo.IsDescending;
|
||
bool isInnerDescending = innerChild.OutputDataSetInfo.orderByInfo.IsDescending;
|
||
|
||
// Partition outer and inner if needed
|
||
if (outerChild.OutputPartition.ParType == PartitionType.Range &&
|
||
outerChild.OutputPartition.HasKeys &&
|
||
outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer))
|
||
{
|
||
if (innerChild.OutputPartition.ParType != PartitionType.Range ||
|
||
!innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) ||
|
||
!outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition))
|
||
{
|
||
// Range distribute inner using outer's partition.
|
||
innerChild = outerChild.OutputPartition.CreatePartitionNode(innerKeySelector, innerChild);
|
||
if (IsMergeNodeNeeded(innerChild))
|
||
{
|
||
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
|
||
}
|
||
}
|
||
}
|
||
else if (innerChild.OutputPartition.ParType == PartitionType.Range &&
|
||
innerChild.OutputPartition.HasKeys &&
|
||
innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer))
|
||
{
|
||
// Range distribute outer using inner's partition.
|
||
outerChild = innerChild.OutputPartition.CreatePartitionNode(outerKeySelector, outerChild);
|
||
if (IsMergeNodeNeeded(outerChild))
|
||
{
|
||
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
|
||
}
|
||
}
|
||
else if (outerChild.OutputPartition.ParType == PartitionType.Hash &&
|
||
outerChild.OutputPartition.IsPartitionedBy(outerKeySelector, comparer))
|
||
{
|
||
if (innerChild.OutputPartition.ParType != PartitionType.Hash ||
|
||
!innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer) ||
|
||
!outerChild.OutputPartition.IsSamePartition(innerChild.OutputPartition))
|
||
{
|
||
innerChild = new DryadHashPartitionNode(innerKeySelector,
|
||
comparerExpr,
|
||
outerChild.OutputPartition.Count,
|
||
queryExpr,
|
||
innerChild);
|
||
if (IsMergeNodeNeeded(innerChild))
|
||
{
|
||
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
|
||
}
|
||
}
|
||
}
|
||
else if (innerChild.OutputPartition.ParType == PartitionType.Hash &&
|
||
innerChild.OutputPartition.IsPartitionedBy(innerKeySelector, comparer))
|
||
{
|
||
outerChild = new DryadHashPartitionNode(outerKeySelector,
|
||
comparerExpr,
|
||
innerChild.OutputPartition.Count,
|
||
queryExpr,
|
||
outerChild);
|
||
if (IsMergeNodeNeeded(outerChild))
|
||
{
|
||
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
// No luck. Hash partition both outer and inner
|
||
int parCnt = Math.Max(outerChild.OutputPartition.Count, innerChild.OutputPartition.Count);
|
||
if (parCnt > 1)
|
||
{
|
||
outerChild = new DryadHashPartitionNode(outerKeySelector,
|
||
comparerExpr,
|
||
parCnt,
|
||
queryExpr,
|
||
outerChild);
|
||
if (IsMergeNodeNeeded(outerChild))
|
||
{
|
||
outerChild = new DryadMergeNode(false, false, queryExpr, outerChild);
|
||
}
|
||
|
||
innerChild = new DryadHashPartitionNode(innerKeySelector,
|
||
comparerExpr,
|
||
parCnt,
|
||
queryExpr,
|
||
innerChild);
|
||
if (IsMergeNodeNeeded(innerChild))
|
||
{
|
||
innerChild = new DryadMergeNode(false, false, queryExpr, innerChild);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Perform either merge or hash join
|
||
string opName = "Hash";
|
||
if (outerChild.IsOrderedBy(outerKeySelector, comparer))
|
||
{
|
||
if (!innerChild.IsOrderedBy(innerKeySelector, comparer) ||
|
||
isOuterDescending != isInnerDescending)
|
||
{
|
||
// Sort inner if unsorted
|
||
innerChild = new DryadOrderByNode(innerKeySelector, comparerExpr,
|
||
isOuterDescending, queryExpr, innerChild);
|
||
}
|
||
opName = "Merge";
|
||
}
|
||
else if (innerChild.IsOrderedBy(innerKeySelector, comparer))
|
||
{
|
||
if (!outerChild.IsOrderedBy(outerKeySelector, comparer) ||
|
||
isOuterDescending != isInnerDescending)
|
||
{
|
||
// Sort outer if unsorted
|
||
outerChild = new DryadOrderByNode(outerKeySelector, comparerExpr,
|
||
isInnerDescending, queryExpr, outerChild);
|
||
}
|
||
opName = "Merge";
|
||
}
|
||
|
||
joinNode = new DryadJoinNode(nodeType,
|
||
opName + nodeType,
|
||
outerKeySelector,
|
||
innerKeySelector,
|
||
resultSelector,
|
||
comparerExpr,
|
||
queryExpr,
|
||
outerChild,
|
||
innerChild);
|
||
return joinNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitDistinct(QueryNodeInfo source,
|
||
Expression comparerExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
|
||
Type keyType = child.OutputTypes[0];
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
|
||
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
|
||
queryExpr);
|
||
}
|
||
|
||
object comparer = null;
|
||
if (comparerExpr != null)
|
||
{
|
||
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
|
||
comparer = evaluator.Eval(comparerExpr);
|
||
}
|
||
|
||
LambdaExpression keySelectExpr = IdentityFunction.Instance(keyType);
|
||
if (!child.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
|
||
{
|
||
if (child.IsDynamic || child.OutputPartition.Count > 1)
|
||
{
|
||
child = new DryadDistinctNode(true, comparerExpr, queryExpr, child);
|
||
bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;
|
||
child = new DryadHashPartitionNode(keySelectExpr,
|
||
comparerExpr,
|
||
child.OutputPartition.Count,
|
||
isDynamic,
|
||
queryExpr,
|
||
child);
|
||
child = new DryadMergeNode(false, false, queryExpr, child);
|
||
}
|
||
}
|
||
DryadQueryNode resNode = new DryadDistinctNode(false, comparerExpr, queryExpr, child);
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitConcat(QueryNodeInfo source, MethodCallExpression queryExpr)
|
||
{
|
||
DryadQueryNode[] childs = new DryadQueryNode[source.children.Count];
|
||
for (int i = 0; i < source.children.Count; ++i)
|
||
{
|
||
childs[i] = this.Visit(source.children[i].child);
|
||
}
|
||
DryadQueryNode resNode = new DryadConcatNode(queryExpr, childs);
|
||
|
||
int parCount = resNode.OutputPartition.Count;
|
||
if (!resNode.IsDynamic && parCount > StaticConfig.MaxPartitionCount)
|
||
{
|
||
// Too many partitions, need to repartition
|
||
int newParCount = parCount / 2;
|
||
DryadQueryNode countNode = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
|
||
true, false, queryExpr, resNode);
|
||
DryadQueryNode mergeCountNode = new DryadMergeNode(true, false, queryExpr, countNode);
|
||
|
||
// Apply node for s => IndexedCount(s)
|
||
Type paramType = typeof(IEnumerable<>).MakeGenericType(typeof(long));
|
||
ParameterExpression param = Expression.Parameter(paramType, "s");
|
||
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("IndexedCount");
|
||
minfo = minfo.MakeGenericMethod(typeof(long));
|
||
Expression body = Expression.Call(minfo, param);
|
||
Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
LambdaExpression indexedCountFunc = Expression.Lambda(funcType, body, param);
|
||
DryadQueryNode indexedCountNode = new DryadApplyNode(indexedCountFunc, queryExpr, mergeCountNode);
|
||
|
||
// HashPartition(x => x.index, parCount)
|
||
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
|
||
Expression keySelectBody = Expression.Property(param, "Index");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
|
||
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
|
||
DryadQueryNode distCountNode = new DryadHashPartitionNode(keySelectExpr,
|
||
null,
|
||
parCount,
|
||
queryExpr,
|
||
indexedCountNode);
|
||
|
||
// Apply node for (x, y) => AddPartitionIndex(x, y, newParCount)
|
||
ParameterExpression param1 = Expression.Parameter(body.Type, "x");
|
||
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(resNode.OutputTypes[0]);
|
||
ParameterExpression param2 = Expression.Parameter(paramType2, "y");
|
||
minfo = typeof(HpcLinqHelper).GetMethod("AddPartitionIndex");
|
||
minfo = minfo.MakeGenericMethod(resNode.OutputTypes[0]);
|
||
body = Expression.Call(minfo, param1, param2, Expression.Constant(newParCount));
|
||
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
|
||
LambdaExpression addIndexFunc = Expression.Lambda(funcType, body, param1, param2);
|
||
DryadQueryNode addIndexNode = new DryadApplyNode(addIndexFunc, queryExpr, distCountNode, resNode);
|
||
|
||
// HashPartition(x => x.index, x => x.value, newParCount)
|
||
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
|
||
body = Expression.Property(param, "Index");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
keySelectExpr = Expression.Lambda(funcType, body, param);
|
||
body = Expression.Property(param, "Value");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
|
||
resNode = new DryadHashPartitionNode(keySelectExpr,
|
||
resultSelectExpr,
|
||
null,
|
||
newParCount,
|
||
false,
|
||
queryExpr,
|
||
addIndexNode);
|
||
resNode = new DryadMergeNode(true, true, queryExpr, resNode);
|
||
}
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitSetOperation(QueryNodeInfo source1,
|
||
QueryNodeInfo source2,
|
||
QueryNodeType nodeType,
|
||
Expression comparerExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child1 = this.Visit(source1);
|
||
DryadQueryNode child2 = this.Visit(source2);
|
||
DryadQueryNode resNode = null;
|
||
|
||
Type keyType = child1.OutputTypes[0];
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
|
||
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
|
||
queryExpr);
|
||
}
|
||
|
||
// The comparer object:
|
||
object comparer = null;
|
||
if (comparerExpr != null)
|
||
{
|
||
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
|
||
comparer = evaluator.Eval(comparerExpr);
|
||
}
|
||
|
||
LambdaExpression keySelectExpr = IdentityFunction.Instance(keyType);
|
||
if (child1.IsDynamic || child2.IsDynamic)
|
||
{
|
||
// Well, let us do the simplest thing for now
|
||
child1 = new DryadHashPartitionNode(keySelectExpr,
|
||
null,
|
||
StaticConfig.DefaultPartitionCount,
|
||
queryExpr,
|
||
child1);
|
||
if (IsMergeNodeNeeded(child1))
|
||
{
|
||
child1 = new DryadMergeNode(false, false, queryExpr, child1);
|
||
}
|
||
|
||
child2 = new DryadHashPartitionNode(keySelectExpr,
|
||
null,
|
||
StaticConfig.DefaultPartitionCount,
|
||
queryExpr,
|
||
child2);
|
||
if (IsMergeNodeNeeded(child2))
|
||
{
|
||
child2 = new DryadMergeNode(false, false, queryExpr, child2);
|
||
}
|
||
|
||
resNode = new DryadSetOperationNode(nodeType, nodeType.ToString(), comparerExpr,
|
||
queryExpr, child1, child2);
|
||
return resNode;
|
||
}
|
||
|
||
bool isDescending1 = child1.OutputDataSetInfo.orderByInfo.IsDescending;
|
||
bool isDescending2 = child2.OutputDataSetInfo.orderByInfo.IsDescending;
|
||
|
||
// Partition child1 and child2 if needed
|
||
if (child1.OutputPartition.ParType == PartitionType.Range &&
|
||
child1.OutputPartition.HasKeys &&
|
||
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
|
||
{
|
||
if (child2.OutputPartition.ParType != PartitionType.Range ||
|
||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
|
||
child1.OutputPartition.IsSamePartition(child2.OutputPartition))
|
||
{
|
||
// Range distribute child2 using child1's partition
|
||
child2 = child1.OutputPartition.CreatePartitionNode(keySelectExpr, child2);
|
||
if (IsMergeNodeNeeded(child2))
|
||
{
|
||
child2 = new DryadMergeNode(false, false, queryExpr, child2);
|
||
}
|
||
}
|
||
}
|
||
else if (child2.OutputPartition.ParType == PartitionType.Range &&
|
||
child2.OutputPartition.HasKeys &&
|
||
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
|
||
{
|
||
// Range distribute child1 using child2's partition
|
||
child1 = child2.OutputPartition.CreatePartitionNode(keySelectExpr, child1);
|
||
if (IsMergeNodeNeeded(child1))
|
||
{
|
||
child1 = new DryadMergeNode(false, false, queryExpr, child1);
|
||
}
|
||
}
|
||
else if (child1.OutputPartition.ParType == PartitionType.Hash &&
|
||
child1.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
|
||
{
|
||
if (child2.OutputPartition.ParType != PartitionType.Hash ||
|
||
!child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer) ||
|
||
!child1.OutputPartition.IsSamePartition(child2.OutputPartition))
|
||
{
|
||
// Hash distribute child2:
|
||
child2 = new DryadHashPartitionNode(keySelectExpr,
|
||
comparerExpr,
|
||
child1.OutputPartition.Count,
|
||
queryExpr,
|
||
child2);
|
||
if (IsMergeNodeNeeded(child2))
|
||
{
|
||
child2 = new DryadMergeNode(false, false, queryExpr, child2);
|
||
}
|
||
}
|
||
}
|
||
else if (child2.OutputPartition.ParType == PartitionType.Hash &&
|
||
child2.OutputPartition.IsPartitionedBy(keySelectExpr, comparer))
|
||
{
|
||
child1 = new DryadHashPartitionNode(keySelectExpr,
|
||
comparerExpr,
|
||
child2.OutputPartition.Count,
|
||
queryExpr,
|
||
child1);
|
||
if (IsMergeNodeNeeded(child1))
|
||
{
|
||
child1 = new DryadMergeNode(false, false, queryExpr, child1);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
// No luck. Hash distribute both child1 and child2, then perform hash operation
|
||
int parCnt = Math.Max(child1.OutputPartition.Count, child2.OutputPartition.Count);
|
||
if (parCnt > 1)
|
||
{
|
||
child1 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child1);
|
||
if (IsMergeNodeNeeded(child1))
|
||
{
|
||
child1 = new DryadMergeNode(false, false, queryExpr, child1);
|
||
}
|
||
|
||
child2 = new DryadHashPartitionNode(keySelectExpr, comparerExpr, parCnt, queryExpr, child2);
|
||
if (IsMergeNodeNeeded(child2))
|
||
{
|
||
child2 = new DryadMergeNode(false, false, queryExpr, child2);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Perform either hash or ordered operation
|
||
string opName = "";
|
||
if (child1.IsOrderedBy(keySelectExpr, comparer))
|
||
{
|
||
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
|
||
isDescending1 != isDescending2)
|
||
{
|
||
// Sort inner if unsorted
|
||
child2 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending1, queryExpr, child2);
|
||
}
|
||
opName = "Ordered";
|
||
}
|
||
else if (child2.IsOrderedBy(keySelectExpr, comparer))
|
||
{
|
||
if (!child1.IsOrderedBy(keySelectExpr, comparer) ||
|
||
isDescending1 != isDescending2)
|
||
{
|
||
// Sort outer if unsorted
|
||
child1 = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending2, queryExpr, child1);
|
||
}
|
||
opName = "Ordered";
|
||
}
|
||
|
||
resNode = new DryadSetOperationNode(nodeType, opName + nodeType, comparerExpr, queryExpr, child1, child2);
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitContains(QueryNodeInfo source,
|
||
Expression valueExpr,
|
||
Expression comparerExpr,
|
||
bool isQuery,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
|
||
Type keyType = child.OutputTypes[0];
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
|
||
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
|
||
queryExpr);
|
||
}
|
||
|
||
DryadQueryNode resNode = this.PromoteConcat(
|
||
source, child,
|
||
x => new DryadContainsNode(valueExpr, comparerExpr, queryExpr, x));
|
||
resNode = new DryadBasicAggregateNode(null, AggregateOpType.Any, false, isQuery, queryExpr, resNode);
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitQuantifier(QueryNodeInfo source,
|
||
LambdaExpression lambda,
|
||
AggregateOpType aggType,
|
||
bool isQuery,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
DryadQueryNode resNode = this.PromoteConcat(
|
||
source, child,
|
||
x => new DryadBasicAggregateNode(
|
||
lambda, aggType, true, isQuery, queryExpr, x));
|
||
resNode = new DryadBasicAggregateNode(null, aggType, false, isQuery, queryExpr, resNode);
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitAggregate(QueryNodeInfo source,
|
||
Expression seed,
|
||
LambdaExpression funcLambda,
|
||
LambdaExpression resultLambda,
|
||
bool isQuery,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
DryadQueryNode resNode = child;
|
||
if (HpcLinqExpression.IsAssociative(funcLambda))
|
||
{
|
||
LambdaExpression combinerLambda = HpcLinqExpression.GetAssociativeCombiner(funcLambda);
|
||
ResourceAttribute attrib = AttributeSystem.GetResourceAttrib(funcLambda);
|
||
bool funcIsExpensive = (attrib != null && attrib.IsExpensive);
|
||
resNode = this.PromoteConcat(
|
||
source, child,
|
||
delegate(DryadQueryNode x) {
|
||
DryadQueryNode y = new DryadAggregateNode("AssocAggregate", seed,
|
||
funcLambda, combinerLambda, resultLambda,
|
||
1, isQuery, queryExpr, x, false);
|
||
return new DryadAggregateNode("AssocTreeAggregate", seed,
|
||
funcLambda, combinerLambda, resultLambda,
|
||
2, isQuery, queryExpr, y, false);
|
||
});
|
||
resNode = new DryadAggregateNode("AssocAggregate", seed, funcLambda, combinerLambda, resultLambda,
|
||
3, isQuery, queryExpr, resNode, funcIsExpensive);
|
||
}
|
||
else
|
||
{
|
||
resNode = new DryadAggregateNode("Aggregate", seed, funcLambda, null, resultLambda, 3,
|
||
isQuery, queryExpr, child, false);
|
||
}
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitBasicAggregate(QueryNodeInfo source,
|
||
LambdaExpression lambda,
|
||
AggregateOpType aggType,
|
||
bool isQuery,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
if (aggType == AggregateOpType.Min || aggType == AggregateOpType.Max)
|
||
{
|
||
Type elemType = child.OutputTypes[0];
|
||
if (lambda != null)
|
||
{
|
||
elemType = lambda.Body.Type;
|
||
}
|
||
if (!TypeSystem.HasDefaultComparer(elemType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.AggregationOperatorRequiresIComparable,
|
||
String.Format(SR.AggregationOperatorRequiresIComparable, aggType ),
|
||
queryExpr);
|
||
}
|
||
}
|
||
DryadQueryNode resNode = this.PromoteConcat(
|
||
source, child,
|
||
x => new DryadBasicAggregateNode(lambda, aggType, true, isQuery, queryExpr, x));
|
||
|
||
switch (aggType)
|
||
{
|
||
case AggregateOpType.Count:
|
||
case AggregateOpType.LongCount:
|
||
{
|
||
resNode = new DryadBasicAggregateNode(null, AggregateOpType.Sum, false,
|
||
isQuery, queryExpr, resNode);
|
||
break;
|
||
}
|
||
case AggregateOpType.Sum:
|
||
case AggregateOpType.Min:
|
||
case AggregateOpType.Max:
|
||
case AggregateOpType.Average:
|
||
{
|
||
resNode = new DryadBasicAggregateNode(null, aggType, false,
|
||
isQuery, queryExpr, resNode);
|
||
break;
|
||
}
|
||
default:
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
|
||
String.Format(SR.OperatorNotSupported, aggType),
|
||
queryExpr);
|
||
}
|
||
}
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitGroupBy(QueryNodeInfo source,
|
||
LambdaExpression keySelectExpr,
|
||
LambdaExpression elemSelectExpr,
|
||
LambdaExpression resultSelectExpr,
|
||
Expression comparerExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
|
||
ExpressionInfo einfo = new ExpressionInfo(keySelectExpr);
|
||
if (einfo.IsExpensive)
|
||
{
|
||
// Any method call that is not tagged as "expensive=false" will be deemed expensive.
|
||
// if the keySelector is expensive, we rewrite the query so that the key-function is invoked only once
|
||
// and the record key passed around via a Pair<TKey,TRecord>.
|
||
// keyFunc becomes pair=>pair.Key
|
||
// elementSelector must be rewritten so that references to (record) become (pair.Value)
|
||
|
||
Type[] vkTypes = keySelectExpr.Type.GetGenericArguments();
|
||
Type pairType = typeof(Pair<,>).MakeGenericType(vkTypes[1], vkTypes[0]);
|
||
ParameterExpression pairParam = Expression.Parameter(pairType, "e");
|
||
|
||
// Add Select(x => new Pair<K,S>(key(x), x))
|
||
ParameterExpression valueParam = keySelectExpr.Parameters[0];
|
||
Expression body = Expression.New(pairType.GetConstructors()[0], keySelectExpr.Body, valueParam);
|
||
Type delegateType = typeof(Func<,>).MakeGenericType(valueParam.Type, body.Type);
|
||
LambdaExpression selectExpr = Expression.Lambda(delegateType, body, valueParam);
|
||
child = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, child);
|
||
|
||
// Change keySelector to e => e.Key
|
||
PropertyInfo keyInfo = pairParam.Type.GetProperty("Key");
|
||
body = Expression.Property(pairParam, keyInfo);
|
||
delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type);
|
||
keySelectExpr = Expression.Lambda(delegateType, body, pairParam);
|
||
|
||
// Add or change elementSelector with e.Value
|
||
PropertyInfo valueInfo = pairParam.Type.GetProperty("Value");
|
||
body = Expression.Property(pairParam, valueInfo);
|
||
if (elemSelectExpr != null)
|
||
{
|
||
ParameterSubst subst = new ParameterSubst(elemSelectExpr.Parameters[0], body);
|
||
body = subst.Visit(elemSelectExpr.Body);
|
||
}
|
||
delegateType = typeof(Func<,>).MakeGenericType(pairParam.Type, body.Type);
|
||
elemSelectExpr = Expression.Lambda(delegateType, body, pairParam);
|
||
}
|
||
|
||
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
|
||
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
|
||
queryExpr);
|
||
}
|
||
Type elemType;
|
||
if (elemSelectExpr == null)
|
||
{
|
||
elemType = keySelectExpr.Type.GetGenericArguments()[0];
|
||
}
|
||
else
|
||
{
|
||
elemType = elemSelectExpr.Type.GetGenericArguments()[1];
|
||
}
|
||
|
||
// The comparer object:
|
||
object comparer = null;
|
||
if (comparerExpr != null)
|
||
{
|
||
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
|
||
comparer = evaluator.Eval(comparerExpr);
|
||
}
|
||
|
||
LambdaExpression keySelectExpr1 = keySelectExpr;
|
||
LambdaExpression elemSelectExpr1 = elemSelectExpr;
|
||
LambdaExpression resultSelectExpr1 = resultSelectExpr;
|
||
LambdaExpression seedExpr1 = null;
|
||
LambdaExpression accumulateExpr1 = null;
|
||
|
||
List<DecompositionInfo> dInfoList = null;
|
||
if (resultSelectExpr != null)
|
||
{
|
||
dInfoList = Decomposition.GetDecompositionInfoList(resultSelectExpr, this.m_codeGen);
|
||
}
|
||
|
||
String groupByOpName = "GroupBy";
|
||
DryadQueryNode groupByNode = child;
|
||
bool isPartitioned = child.OutputPartition.IsPartitionedBy(keySelectExpr, comparer);
|
||
if (dInfoList != null)
|
||
{
|
||
// ** Decomposable GroupBy-Reduce
|
||
// This block creates the first GroupByNode and does some preparation for subsequent nodes.
|
||
if (child.IsOrderedBy(keySelectExpr, comparer))
|
||
{
|
||
groupByOpName = "OrderedGroupBy";
|
||
}
|
||
|
||
int dcnt = dInfoList.Count;
|
||
ParameterExpression keyParam;
|
||
if (resultSelectExpr.Parameters.Count == 1)
|
||
{
|
||
keyParam = Expression.Parameter(keyType, HpcLinqCodeGen.MakeUniqueName("k"));
|
||
}
|
||
else
|
||
{
|
||
keyParam = resultSelectExpr.Parameters[0];
|
||
}
|
||
|
||
// Seed:
|
||
ParameterExpression param2 = Expression.Parameter(
|
||
elemType, HpcLinqCodeGen.MakeUniqueName("e"));
|
||
Expression zeroExpr = Expression.Constant(0, typeof(int));
|
||
Expression seedBody = zeroExpr;
|
||
if (dcnt != 0)
|
||
{
|
||
LambdaExpression seed = dInfoList[dcnt-1].Seed;
|
||
ParameterSubst subst = new ParameterSubst(seed.Parameters[0], param2);
|
||
seedBody = subst.Visit(seed.Body);
|
||
for (int i = dcnt - 2; i >= 0; i--)
|
||
{
|
||
seed = dInfoList[i].Seed;
|
||
subst = new ParameterSubst(seed.Parameters[0], param2);
|
||
Expression firstExpr = subst.Visit(seed.Body);
|
||
Type newPairType = typeof(Pair<,>).MakeGenericType(firstExpr.Type, seedBody.Type);
|
||
seedBody = Expression.New(newPairType.GetConstructors()[0], firstExpr, seedBody);
|
||
}
|
||
}
|
||
LambdaExpression seedExpr = Expression.Lambda(seedBody, param2);
|
||
|
||
// Accumulate:
|
||
ParameterExpression param1 = Expression.Parameter(
|
||
seedBody.Type, HpcLinqCodeGen.MakeUniqueName("a"));
|
||
Expression accumulateBody = zeroExpr;
|
||
if (dcnt != 0)
|
||
{
|
||
accumulateBody = Decomposition.AccumulateList(param1, param2, dInfoList, 0);
|
||
}
|
||
LambdaExpression accumulateExpr = Expression.Lambda(accumulateBody, param1, param2);
|
||
|
||
// Now prepare for the merge-aggregator and/or in the secondary group-by.
|
||
// keySelectExpr1: e => e.Key
|
||
Type reducerResType = typeof(Pair<,>).MakeGenericType(keyParam.Type, accumulateBody.Type);
|
||
ParameterExpression reducerResParam = Expression.Parameter(reducerResType, "e");
|
||
PropertyInfo keyInfo = reducerResParam.Type.GetProperty("Key");
|
||
Expression body = Expression.Property(reducerResParam, keyInfo);
|
||
Type delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type);
|
||
keySelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam);
|
||
|
||
// elemSelectExpr1: e => e.Value
|
||
PropertyInfo valueInfo = reducerResParam.Type.GetProperty("Value");
|
||
body = Expression.Property(reducerResParam, valueInfo);
|
||
delegateType = typeof(Func<,>).MakeGenericType(reducerResParam.Type, body.Type);
|
||
elemSelectExpr1 = Expression.Lambda(delegateType, body, reducerResParam);
|
||
|
||
// SeedExpr1
|
||
param2 = Expression.Parameter(elemSelectExpr1.Body.Type,
|
||
HpcLinqCodeGen.MakeUniqueName("e"));
|
||
seedExpr1 = Expression.Lambda(param2, param2);
|
||
|
||
// AccumulateExpr1
|
||
Expression recursiveAccumulateBody = zeroExpr;
|
||
if (dcnt != 0)
|
||
{
|
||
recursiveAccumulateBody = Decomposition.RecursiveAccumulateList(param1, param2, dInfoList, 0);
|
||
}
|
||
accumulateExpr1 = Expression.Lambda(recursiveAccumulateBody, param1, param2);
|
||
|
||
// resultSelectExpr1
|
||
resultSelectExpr1 = null;
|
||
|
||
// The first groupByNode.
|
||
// If the input was already correctly partitioned, this will be the only groupByNode.
|
||
bool isPartial = StaticConfig.GroupByLocalAggregationIsPartial && !isPartitioned;
|
||
groupByNode = new DryadGroupByNode(
|
||
groupByOpName, keySelectExpr, elemSelectExpr, null,
|
||
seedExpr, accumulateExpr, accumulateExpr1, comparerExpr,
|
||
isPartial, queryExpr, child);
|
||
}
|
||
else
|
||
{
|
||
// Can't do partial aggregation.
|
||
// Use sort, mergesort, and ordered groupby, if TKey implements IComparable.
|
||
if ((comparer != null && TypeSystem.IsComparer(comparer, keyType)) ||
|
||
(comparer == null && TypeSystem.HasDefaultComparer(keyType)))
|
||
{
|
||
if (!child.IsOrderedBy(keySelectExpr, comparer))
|
||
{
|
||
groupByNode = new DryadOrderByNode(keySelectExpr, comparerExpr, true, queryExpr, child);
|
||
}
|
||
groupByOpName = "OrderedGroupBy";
|
||
}
|
||
|
||
// Add a GroupByNode if it is partitioned or has elementSelector.
|
||
// If the input was already correctly partitioned, this will be the only groupByNode.
|
||
if (isPartitioned)
|
||
{
|
||
groupByNode = new DryadGroupByNode(groupByOpName,
|
||
keySelectExpr,
|
||
elemSelectExpr,
|
||
resultSelectExpr,
|
||
null, // seed
|
||
null, // accumulate
|
||
null, // recursiveAccumulate
|
||
comparerExpr,
|
||
false, // isPartial
|
||
queryExpr,
|
||
groupByNode);
|
||
}
|
||
else if (elemSelectExpr != null)
|
||
{
|
||
// Local GroupBy without resultSelector:
|
||
groupByNode = new DryadGroupByNode(groupByOpName,
|
||
keySelectExpr,
|
||
elemSelectExpr,
|
||
null, // resultSelect
|
||
null, // seed
|
||
null, // accumulate
|
||
null, // recursiveAccumulate
|
||
comparerExpr,
|
||
StaticConfig.GroupByLocalAggregationIsPartial, // isPartial
|
||
queryExpr,
|
||
groupByNode);
|
||
|
||
// keySelectExpr1: g => g.Key
|
||
ParameterExpression groupParam = Expression.Parameter(groupByNode.OutputTypes[0], "g");
|
||
PropertyInfo keyInfo = groupParam.Type.GetProperty("Key");
|
||
Expression body = Expression.Property(groupParam, keyInfo);
|
||
Type delegateType = typeof(Func<,>).MakeGenericType(groupParam.Type, body.Type);
|
||
keySelectExpr1 = Expression.Lambda(delegateType, body, groupParam);
|
||
|
||
// No elementSelector
|
||
elemSelectExpr1 = null;
|
||
|
||
// resultSelectExpr1
|
||
ParameterExpression keyParam;
|
||
Type groupType = typeof(IEnumerable<>).MakeGenericType(groupByNode.OutputTypes[0]);
|
||
groupParam = Expression.Parameter(groupType, HpcLinqCodeGen.MakeUniqueName("g"));
|
||
if (resultSelectExpr == null)
|
||
{
|
||
// resultSelectExpr1: (k, g) => MakeDryadLinqGroup(k, g)
|
||
keyParam = Expression.Parameter(keySelectExpr1.Body.Type, HpcLinqCodeGen.MakeUniqueName("k"));
|
||
MethodInfo groupingInfo = typeof(HpcLinqEnumerable).GetMethod("MakeHpcLinqGroup");
|
||
groupingInfo = groupingInfo.MakeGenericMethod(keyParam.Type, elemType);
|
||
body = Expression.Call(groupingInfo, keyParam, groupParam);
|
||
}
|
||
else
|
||
{
|
||
// resultSelectExpr1: (k, g) => resultSelectExpr(k, FlattenGroups(g))
|
||
keyParam = resultSelectExpr.Parameters[0];
|
||
MethodInfo flattenInfo = typeof(HpcLinqEnumerable).GetMethod("FlattenGroups");
|
||
flattenInfo = flattenInfo.MakeGenericMethod(keyParam.Type, elemType);
|
||
Expression groupExpr = Expression.Call(flattenInfo, groupParam);
|
||
ParameterSubst subst = new ParameterSubst(resultSelectExpr.Parameters[1], groupExpr);
|
||
body = subst.Visit(resultSelectExpr.Body);
|
||
}
|
||
delegateType = typeof(Func<,,>).MakeGenericType(keyParam.Type, groupParam.Type, body.Type);
|
||
resultSelectExpr1 = Expression.Lambda(delegateType, body, keyParam, groupParam);
|
||
}
|
||
}
|
||
|
||
// At this point, the first GroupByNode has been created.
|
||
DryadMergeNode mergeNode = null;
|
||
DryadQueryNode groupByNode1 = groupByNode;
|
||
if (!isPartitioned)
|
||
{
|
||
// Create HashPartitionNode, MergeNode, and second GroupByNode
|
||
|
||
// Note, if we are doing decomposable-GroupByReduce, there is still some work to go after this
|
||
// - attach the combiner to the first merge-node
|
||
// - attach the combiner to the merge-node
|
||
// - attach finalizer to second GroupBy
|
||
int parCount = (groupByNode.IsDynamic) ? StaticConfig.DefaultPartitionCount : groupByNode.OutputPartition.Count;
|
||
bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;
|
||
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr1, comparerExpr, parCount,
|
||
isDynamic, queryExpr, groupByNode);
|
||
|
||
// Create the Merge Node
|
||
if (groupByOpName == "OrderedGroupBy")
|
||
{
|
||
// Mergesort with the same keySelector of the hash partition
|
||
mergeNode = new DryadMergeNode(keySelectExpr1, comparerExpr, true, false, queryExpr, hdistNode);
|
||
}
|
||
else
|
||
{
|
||
// Random merge
|
||
mergeNode = new DryadMergeNode(false, false, queryExpr, hdistNode);
|
||
}
|
||
groupByNode1 = new DryadGroupByNode(groupByOpName, keySelectExpr1, elemSelectExpr1,
|
||
resultSelectExpr1, seedExpr1, accumulateExpr1,
|
||
accumulateExpr1, comparerExpr, false, queryExpr,
|
||
mergeNode);
|
||
}
|
||
|
||
// Final tidy-up for decomposable GroupBy-Reduce pattern.
|
||
// - attach combiner to first GroupByNode
|
||
// - attache combiner to MergeNode as an aggregator
|
||
// - build a SelectNode to project out results and call finalizer on them.
|
||
if (dInfoList != null)
|
||
{
|
||
// Add dynamic aggregator to the merge-node, if applicable
|
||
if (StaticConfig.GroupByDynamicReduce && !isPartitioned)
|
||
{
|
||
mergeNode.AddAggregateNode(groupByNode1);
|
||
}
|
||
|
||
// Add the final Select node
|
||
Type keyResultPairType = typeof(Pair<,>).MakeGenericType(keyType, seedExpr1.Body.Type);
|
||
ParameterExpression keyResultPairParam = Expression.Parameter(keyResultPairType,
|
||
HpcLinqCodeGen.MakeUniqueName("e"));
|
||
PropertyInfo valuePropInfo_1 = keyResultPairType.GetProperty("Value");
|
||
Expression combinedValueExpr = Expression.Property(keyResultPairParam, valuePropInfo_1);
|
||
|
||
// First, build the combinerList
|
||
int dcnt = dInfoList.Count;
|
||
Expression[] combinerList = new Expression[dcnt];
|
||
for (int i = 0; i < dcnt; i++)
|
||
{
|
||
if (i + 1 == dcnt)
|
||
{
|
||
combinerList[i] = combinedValueExpr;
|
||
}
|
||
else
|
||
{
|
||
PropertyInfo keyPropInfo = combinedValueExpr.Type.GetProperty("Key");
|
||
combinerList[i] = Expression.Property(combinedValueExpr, keyPropInfo);
|
||
PropertyInfo valuePropInfo = combinedValueExpr.Type.GetProperty("Value");
|
||
combinedValueExpr = Expression.Property(combinedValueExpr, valuePropInfo);
|
||
}
|
||
LambdaExpression finalizerExpr = dInfoList[i].FinalReducer;
|
||
if (finalizerExpr != null)
|
||
{
|
||
ParameterSubst subst = new ParameterSubst(finalizerExpr.Parameters[0], combinerList[i]);
|
||
combinerList[i] = subst.Visit(finalizerExpr.Body);
|
||
}
|
||
}
|
||
|
||
// Build the funcList
|
||
Expression[] funcList = new Expression[dcnt];
|
||
for (int i = 0; i < dcnt; i++)
|
||
{
|
||
funcList[i] = dInfoList[i].Func;
|
||
}
|
||
|
||
// Apply the substitutions
|
||
CombinerSubst combinerSubst = new CombinerSubst(resultSelectExpr, keyResultPairParam, funcList, combinerList);
|
||
Expression finalizerSelectBody = combinerSubst.Visit();
|
||
|
||
// Finally, the Select node
|
||
Type delegateType = typeof(Func<,>).MakeGenericType(keyResultPairType, finalizerSelectBody.Type);
|
||
LambdaExpression selectExpr = Expression.Lambda(delegateType, finalizerSelectBody, keyResultPairParam);
|
||
groupByNode1 = new DryadSelectNode(QueryNodeType.Select, selectExpr, null, queryExpr, groupByNode1);
|
||
}
|
||
return groupByNode1;
|
||
}
|
||
|
||
// Creates an "auto-sampling range-partition sub-query"
|
||
private DryadQueryNode CreateRangePartition(bool isDynamic,
|
||
LambdaExpression keySelectExpr,
|
||
LambdaExpression resultSelectExpr,
|
||
Expression comparerExpr,
|
||
Expression isDescendingExpr,
|
||
Expression queryExpr,
|
||
Expression partitionCountExpr,
|
||
DryadQueryNode child)
|
||
{
|
||
// Make child a Tee node
|
||
child.IsForked = true;
|
||
|
||
// The partition count
|
||
Expression countExpr = null;
|
||
|
||
if (isDescendingExpr == null)
|
||
{
|
||
isDescendingExpr = Expression.Constant(false, typeof(bool)); //default for isDescending is false.
|
||
}
|
||
|
||
// NOTE: for MayRTM, isDynamic should never be true
|
||
if (!isDynamic)
|
||
{
|
||
if (partitionCountExpr != null)
|
||
{
|
||
countExpr = partitionCountExpr;
|
||
}
|
||
else
|
||
{
|
||
// If partitionCount was not explicitly set, use the child's partition count.
|
||
countExpr = Expression.Constant(child.OutputPartition.Count);
|
||
}
|
||
}
|
||
|
||
Type recordType = child.OutputTypes[0];
|
||
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
|
||
|
||
// Create x => Phase1Sampling(x_1, keySelector, denv)
|
||
Type lambdaParamType1 = typeof(IEnumerable<>).MakeGenericType(recordType);
|
||
ParameterExpression lambdaParam1 = Expression.Parameter(lambdaParamType1, "x_1");
|
||
|
||
ParameterExpression denvParam = Expression.Parameter(typeof(HpcLinqVertexEnv), "denv");
|
||
|
||
MethodInfo minfo = typeof(HpcLinqSampler).GetMethod("Phase1Sampling");
|
||
Expression body = Expression.Call(minfo.MakeGenericMethod(recordType, keyType),
|
||
lambdaParam1, keySelectExpr, denvParam);
|
||
Type type = typeof(Func<,>).MakeGenericType(lambdaParam1.Type, body.Type);
|
||
LambdaExpression samplingExpr = Expression.Lambda(type, body, lambdaParam1);
|
||
|
||
// Create the Sampling node
|
||
DryadApplyNode samplingNode = new DryadApplyNode(samplingExpr, queryExpr, child);
|
||
|
||
// Create x => RangeSampler(x, keySelectExpr, comparer, isDescendingExpr)
|
||
Type lambdaParamType = typeof(IEnumerable<>).MakeGenericType(keyType);
|
||
ParameterExpression lambdaParam = Expression.Parameter(lambdaParamType, "x_2");
|
||
|
||
//For RTM, isDynamic should never be true.
|
||
//string methodName = (isDynamic) ? "RangeSampler_Dynamic" : "RangeSampler_Static";
|
||
Debug.Assert(isDynamic == false, "Internal error: isDynamic is true.");
|
||
string methodName = "RangeSampler_Static";
|
||
|
||
minfo = typeof(HpcLinqSampler).GetMethod(methodName);
|
||
minfo = minfo.MakeGenericMethod(keyType);
|
||
Expression comparerArgExpr = comparerExpr;
|
||
if (comparerExpr == null)
|
||
{
|
||
if (!TypeSystem.HasDefaultComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
|
||
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
|
||
queryExpr);
|
||
}
|
||
comparerArgExpr = Expression.Constant(null, typeof(IComparer<>).MakeGenericType(keyType));
|
||
}
|
||
|
||
Expression lastArg;
|
||
if (isDynamic)
|
||
{
|
||
lastArg = denvParam;
|
||
}
|
||
else
|
||
{
|
||
lastArg = countExpr;
|
||
}
|
||
|
||
body = Expression.Call(minfo, lambdaParam, comparerArgExpr, isDescendingExpr, lastArg);
|
||
type = typeof(Func<,>).MakeGenericType(lambdaParam.Type, body.Type);
|
||
LambdaExpression samplerExpr = Expression.Lambda(type, body, lambdaParam);
|
||
|
||
// Create the sample node
|
||
DryadQueryNode sampleDataNode = new DryadMergeNode(false, true, queryExpr, samplingNode);
|
||
DryadQueryNode sampleNode = new DryadApplyNode(samplerExpr, queryExpr, sampleDataNode);
|
||
sampleNode.IsForked = true;
|
||
|
||
// Create the range distribute node
|
||
DryadQueryNode resNode = new DryadRangePartitionNode(keySelectExpr,
|
||
resultSelectExpr,
|
||
null,
|
||
comparerExpr,
|
||
isDescendingExpr,
|
||
countExpr,
|
||
queryExpr,
|
||
child,
|
||
sampleNode);
|
||
resNode = new DryadMergeNode(false, true, queryExpr, resNode);
|
||
|
||
// Set the dynamic manager for sampleNode
|
||
if (isDynamic)
|
||
{
|
||
sampleDataNode.DynamicManager = new DynamicRangeDistributor(resNode);
|
||
}
|
||
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitOrderBy(QueryNodeInfo source,
|
||
LambdaExpression keySelectExpr,
|
||
Expression comparerExpr,
|
||
bool isDescending,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
|
||
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
|
||
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
|
||
queryExpr);
|
||
}
|
||
|
||
DryadQueryNode resNode = child;
|
||
if (child.OutputPartition.ParType == PartitionType.Range &&
|
||
child.OutputPartition.IsPartitionedBy(keySelectExpr, comparerExpr, isDescending))
|
||
{
|
||
// Only need to sort each partition
|
||
resNode = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending, queryExpr, child);
|
||
}
|
||
else
|
||
{
|
||
Expression isDescendingExpr = Expression.Constant(isDescending);
|
||
bool dynamicOptEnabled = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicRangePartitionLevel) != 0;
|
||
if (dynamicOptEnabled || child.IsDynamic)
|
||
{
|
||
// NOTE: for MayRTM, this path should not be taken.
|
||
resNode = this.CreateRangePartition(true, keySelectExpr, null, comparerExpr,
|
||
isDescendingExpr, queryExpr, null, child);
|
||
resNode = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending, queryExpr, resNode);
|
||
}
|
||
else
|
||
{
|
||
if (child.OutputPartition.Count > 1)
|
||
{
|
||
resNode = this.CreateRangePartition(false, keySelectExpr, null, comparerExpr,
|
||
isDescendingExpr, queryExpr, null, child);
|
||
}
|
||
resNode = new DryadOrderByNode(keySelectExpr, comparerExpr, isDescending, queryExpr, resNode);
|
||
}
|
||
}
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode FirstStagePartitionOp(string opName,
|
||
QueryNodeType nodeType,
|
||
Expression controlExpr,
|
||
MethodCallExpression queryExpr,
|
||
DryadQueryNode child)
|
||
{
|
||
if (nodeType == QueryNodeType.TakeWhile)
|
||
{
|
||
Type ptype = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
|
||
ParameterExpression param = Expression.Parameter(ptype, HpcLinqCodeGen.MakeUniqueName("x"));
|
||
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod("GroupTakeWhile");
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
|
||
Expression body = Expression.Call(minfo, param, controlExpr);
|
||
Type type = typeof(Func<,>).MakeGenericType(ptype, body.Type);
|
||
LambdaExpression procFunc = Expression.Lambda(type, body, param);
|
||
return new DryadApplyNode(procFunc, queryExpr, child);
|
||
}
|
||
else
|
||
{
|
||
return new DryadPartitionOpNode(opName, nodeType, controlExpr, true, queryExpr, child);
|
||
}
|
||
}
|
||
|
||
private DryadQueryNode VisitPartitionOp(string opName,
|
||
QueryNodeInfo source,
|
||
QueryNodeType nodeType,
|
||
Expression controlExpr,
|
||
MethodCallExpression queryExpr)
|
||
{
|
||
DryadQueryNode resNode;
|
||
if (nodeType == QueryNodeType.TakeWhile &&
|
||
controlExpr.Type.GetGenericArguments().Length != 2)
|
||
{
|
||
// The "indexed" version.
|
||
resNode = this.Visit(source);
|
||
|
||
// The following block used to be skipped for resNode.OutputPartition.Count == 1,
|
||
// which causes compilation error (bug 13593)
|
||
// @@TODO[p3] : implement a working optimization for nPartition==1 that calls
|
||
// directly to Linq TakeWhile.
|
||
// Note: the test is: if (resNode.IsDynamic || resNode.OutputPartition.Count > 1)
|
||
{
|
||
resNode.IsForked = true;
|
||
|
||
bool isLong = (queryExpr.Method.Name == "LongTakeWhile");
|
||
DryadQueryNode offsetNode = this.CreateOffset(isLong, queryExpr, resNode);
|
||
|
||
// Create (x, y) => GroupIndexedTakeWhile(x, y, controlExpr)
|
||
Type ptype1 = typeof(IEnumerable<>).MakeGenericType(resNode.OutputTypes[0]);
|
||
Type ptype2 = typeof(IEnumerable<>).MakeGenericType(typeof(IndexedValue<long>));
|
||
ParameterExpression param1 = Expression.Parameter(ptype1, HpcLinqCodeGen.MakeUniqueName("x"));
|
||
ParameterExpression param2 = Expression.Parameter(ptype2, HpcLinqCodeGen.MakeUniqueName("y"));
|
||
string methodName = "GroupIndexed" + queryExpr.Method.Name;
|
||
MethodInfo minfo = typeof(HpcLinqEnumerable).GetMethod(methodName);
|
||
minfo = minfo.MakeGenericMethod(resNode.OutputTypes[0]);
|
||
Expression body = Expression.Call(minfo, param1, param2, controlExpr);
|
||
Type type = typeof(Func<,,>).MakeGenericType(ptype1, ptype2, body.Type);
|
||
LambdaExpression procFunc = Expression.Lambda(type, body, param1, param2);
|
||
resNode = new DryadApplyNode(procFunc, queryExpr, resNode, offsetNode);
|
||
}
|
||
}
|
||
else if (!source.IsForked &&
|
||
(nodeType == QueryNodeType.Take || nodeType == QueryNodeType.TakeWhile) &&
|
||
(source.OperatorName == "OrderBy" || source.OperatorName == "OrderByDescending"))
|
||
{
|
||
resNode = this.Visit(source.children[0].child);
|
||
|
||
bool isDescending = (source.OperatorName == "OrderByDescending");
|
||
MethodCallExpression sourceQueryExpr = (MethodCallExpression)source.queryExpression;
|
||
LambdaExpression keySelectExpr = HpcLinqExpression.GetLambda(sourceQueryExpr.Arguments[1]);
|
||
Expression comparerExpr = null;
|
||
if (sourceQueryExpr.Arguments.Count == 3)
|
||
{
|
||
comparerExpr = sourceQueryExpr.Arguments[2];
|
||
}
|
||
resNode = this.PromoteConcat(
|
||
source.children[0].child,
|
||
resNode,
|
||
delegate(DryadQueryNode x) {
|
||
DryadQueryNode y = new DryadOrderByNode(keySelectExpr, comparerExpr,
|
||
isDescending, sourceQueryExpr, x);
|
||
return FirstStagePartitionOp(opName, nodeType, controlExpr, queryExpr, y);
|
||
});
|
||
if (resNode.IsDynamic || resNode.OutputPartition.Count > 1)
|
||
{
|
||
// Need a mergesort
|
||
resNode = new DryadMergeNode(keySelectExpr, comparerExpr, isDescending, false, sourceQueryExpr, resNode);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
resNode = this.Visit(source);
|
||
if (nodeType == QueryNodeType.Take || nodeType == QueryNodeType.TakeWhile)
|
||
{
|
||
resNode = this.PromoteConcat(
|
||
source, resNode,
|
||
x => FirstStagePartitionOp(opName, nodeType, controlExpr, queryExpr, x));
|
||
}
|
||
}
|
||
resNode = new DryadPartitionOpNode(opName, nodeType, controlExpr, false, queryExpr, resNode);
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitZip(QueryNodeInfo first,
|
||
QueryNodeInfo second,
|
||
LambdaExpression resultSelector,
|
||
MethodCallExpression queryExpr)
|
||
{
|
||
DryadQueryNode child1 = this.Visit(first);
|
||
DryadQueryNode child2 = this.Visit(second);
|
||
|
||
if (child1.IsDynamic || child2.IsDynamic)
|
||
{
|
||
// Well, let us for now do it on a single machine
|
||
child1 = new DryadMergeNode(true, false, queryExpr, child1);
|
||
child2 = new DryadMergeNode(true, false, queryExpr, child2);
|
||
|
||
// Apply node for (x, y) => Zip(x, y, resultSelector)
|
||
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(child1.OutputTypes[0]);
|
||
ParameterExpression param1 = Expression.Parameter(paramType1, "s1");
|
||
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child2.OutputTypes[0]);
|
||
ParameterExpression param2 = Expression.Parameter(paramType2, "s2");
|
||
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("Zip");
|
||
minfo = minfo.MakeGenericMethod(child1.OutputTypes[0]);
|
||
Expression body = Expression.Call(minfo, param1, param2, resultSelector);
|
||
Type funcType = typeof(Func<,>).MakeGenericType(param1.Type, param2.Type, body.Type);
|
||
LambdaExpression procFunc = Expression.Lambda(funcType, body, param1, param2);
|
||
return new DryadApplyNode(procFunc, queryExpr, child1, child2);
|
||
}
|
||
else
|
||
{
|
||
int parCount1 = child1.OutputPartition.Count;
|
||
int parCount2 = child2.OutputPartition.Count;
|
||
|
||
// Count nodes
|
||
DryadQueryNode countNode1 = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
|
||
true, false, queryExpr, child1);
|
||
DryadQueryNode countNode2 = new DryadBasicAggregateNode(null, AggregateOpType.LongCount,
|
||
true, false, queryExpr, child2);
|
||
countNode1 = new DryadMergeNode(true, false, queryExpr, countNode1);
|
||
countNode2 = new DryadMergeNode(true, false, queryExpr, countNode2);
|
||
|
||
// Apply node for (x, y) => ZipCount(x, y)
|
||
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(typeof(long));
|
||
ParameterExpression param1 = Expression.Parameter(paramType1, "x");
|
||
ParameterExpression param2 = Expression.Parameter(paramType1, "y");
|
||
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("ZipCount");
|
||
Expression body = Expression.Call(minfo, param1, param2);
|
||
Type funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
|
||
LambdaExpression zipCount = Expression.Lambda(funcType, body, param1, param2);
|
||
DryadQueryNode indexedCountNode = new DryadApplyNode(zipCount, queryExpr, countNode1, countNode2);
|
||
|
||
// HashPartition(x => x.index, parCount2)
|
||
ParameterExpression param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
|
||
Expression keySelectBody = Expression.Property(param, "Index");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
|
||
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
|
||
DryadQueryNode distCountNode = new DryadHashPartitionNode(keySelectExpr,
|
||
null,
|
||
parCount2,
|
||
queryExpr,
|
||
indexedCountNode);
|
||
|
||
// Apply node for (x, y) => AssignPartitionIndex(x, y)
|
||
param1 = Expression.Parameter(body.Type, "x");
|
||
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child2.OutputTypes[0]);
|
||
param2 = Expression.Parameter(paramType2, "y");
|
||
minfo = typeof(HpcLinqHelper).GetMethod("AssignPartitionIndex");
|
||
minfo = minfo.MakeGenericMethod(child2.OutputTypes[0]);
|
||
body = Expression.Call(minfo, param1, param2);
|
||
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
|
||
LambdaExpression assignIndex = Expression.Lambda(funcType, body, param1, param2);
|
||
DryadQueryNode addIndexNode = new DryadApplyNode(assignIndex, queryExpr, distCountNode, child2);
|
||
|
||
// HashPartition(x => x.index, x => x.value, parCount1)
|
||
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
|
||
body = Expression.Property(param, "Index");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
keySelectExpr = Expression.Lambda(funcType, body, param);
|
||
body = Expression.Property(param, "Value");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
|
||
DryadQueryNode newChild2 = new DryadHashPartitionNode(keySelectExpr,
|
||
resultSelectExpr,
|
||
null,
|
||
parCount1,
|
||
false,
|
||
queryExpr,
|
||
addIndexNode);
|
||
newChild2 = new DryadMergeNode(true, true, queryExpr, newChild2);
|
||
|
||
// Finally the zip node
|
||
return new DryadZipNode(resultSelector, queryExpr, child1, newChild2);
|
||
}
|
||
}
|
||
|
||
// Basic plan: (reverse all partitions) then (reverse data in each partition)
|
||
// The main complication is to perform the first step.
|
||
// Approach:
|
||
// - tee the input.
|
||
// - have a dummy apply node that produces the singleton {0} at each partition
|
||
// - merge to get a seq {0,0,..} whose length = nPartition.
|
||
// - convert that seq to { (0,n), (1,n), ...}
|
||
// - hash-partition to send one item to each of the n workers.
|
||
// - use binary-apply to attach targetIndex to each source item
|
||
// Apply( seq1 = indexCountPair, seq2 = original data) => ({tgt, item0}, {tgt, item1}, .. )
|
||
// - hash-partition to move items to target partition.
|
||
// - use local LINQ reverse to do the local data reversal.
|
||
private DryadQueryNode VisitReverse(QueryNodeInfo source, Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
if (child.IsDynamic)
|
||
{
|
||
throw new DryadLinqException("Reverse is only supported for static partition count");
|
||
}
|
||
|
||
child.IsForked = true;
|
||
|
||
// Apply node for s => ValueZero(s)
|
||
Type paramType = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
|
||
ParameterExpression param = Expression.Parameter(paramType, "s");
|
||
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("ValueZero");
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
|
||
Expression body = Expression.Call(minfo, param);
|
||
Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
LambdaExpression procFunc = Expression.Lambda(funcType, body, param);
|
||
DryadQueryNode valueZeroNode = new DryadApplyNode(procFunc, queryExpr, child);
|
||
|
||
// Apply node for s => ReverseIndex(s)
|
||
paramType = typeof(IEnumerable<>).MakeGenericType(typeof(int));
|
||
param = Expression.Parameter(paramType, "s");
|
||
minfo = typeof(HpcLinqHelper).GetMethod("MakeIndexCountPairs");
|
||
body = Expression.Call(minfo, param);
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
procFunc = Expression.Lambda(funcType, body, param);
|
||
DryadQueryNode mergeZeroNode = new DryadMergeNode(true, true, queryExpr, valueZeroNode);
|
||
DryadQueryNode indexCountNode = new DryadApplyNode(procFunc, queryExpr, mergeZeroNode);
|
||
|
||
// HashPartition to distribute the indexCounts -- one to each partition.
|
||
// each partition will receive (myPartitionID, pcount).
|
||
int pcount = child.OutputPartition.Count;
|
||
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
|
||
Expression keySelectBody = Expression.Property(param, "Index");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
|
||
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
|
||
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr,
|
||
null,
|
||
pcount,
|
||
queryExpr,
|
||
indexCountNode);
|
||
|
||
// Apply node for (x, y) => AddIndexForReverse(x, y)
|
||
ParameterExpression param1 = Expression.Parameter(body.Type, "x");
|
||
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
|
||
ParameterExpression param2 = Expression.Parameter(paramType2, "y");
|
||
minfo = typeof(HpcLinqHelper).GetMethod("AddIndexForReverse");
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
|
||
body = Expression.Call(minfo, param1, param2);
|
||
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
|
||
LambdaExpression addIndexFunc = Expression.Lambda(funcType, body, param1, param2);
|
||
DryadQueryNode addIndexNode = new DryadApplyNode(addIndexFunc, queryExpr, hdistNode, child);
|
||
|
||
// HashPartition(x => x.index, x => x.value, pcount)
|
||
// Moves all data to correct target partition. (each worker will direct all its items to one target partition)
|
||
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
|
||
body = Expression.Property(param, "Index");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
keySelectExpr = Expression.Lambda(funcType, body, param);
|
||
body = Expression.Property(param, "Value");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
LambdaExpression resultSelectExpr = Expression.Lambda(funcType, body, param);
|
||
DryadQueryNode reversePartitionNode = new DryadHashPartitionNode(
|
||
keySelectExpr, resultSelectExpr, null,
|
||
pcount, false, queryExpr, addIndexNode);
|
||
|
||
// Reverse node
|
||
paramType = typeof(IEnumerable<>).MakeGenericType(reversePartitionNode.OutputTypes[0]);
|
||
param = Expression.Parameter(paramType, "x");
|
||
minfo = typeof(HpcLinqVertex).GetMethod("Reverse");
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
|
||
body = Expression.Call(minfo, param);
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
procFunc = Expression.Lambda(funcType, body, param);
|
||
DryadQueryNode resNode = new DryadMergeNode(true, true, queryExpr, reversePartitionNode);
|
||
resNode = new DryadApplyNode(procFunc, queryExpr, resNode);
|
||
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitSequenceEqual(QueryNodeInfo source1,
|
||
QueryNodeInfo source2,
|
||
Expression comparerExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child1 = this.Visit(source1);
|
||
DryadQueryNode child2 = this.Visit(source2);
|
||
|
||
Type elemType = child1.OutputTypes[0];
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(elemType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerExpressionMustBeSpecifiedOrElementTypeMustBeIEquatable,
|
||
String.Format(SR.ComparerExpressionMustBeSpecifiedOrElementTypeMustBeIEquatable, elemType),
|
||
queryExpr);
|
||
}
|
||
|
||
// Well, let us do it on a single machine for now
|
||
child1 = new DryadMergeNode(true, false, queryExpr, child1);
|
||
child2 = new DryadMergeNode(true, false, queryExpr, child2);
|
||
|
||
// Apply node for (x, y) => SequenceEqual(x, y, c)
|
||
Type paramType = typeof(IEnumerable<>).MakeGenericType(elemType);
|
||
ParameterExpression param1 = Expression.Parameter(paramType, "s1");
|
||
ParameterExpression param2 = Expression.Parameter(paramType, "s2");
|
||
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("SequenceEqual");
|
||
minfo = minfo.MakeGenericMethod(elemType);
|
||
if (comparerExpr == null)
|
||
{
|
||
comparerExpr = Expression.Constant(null, typeof(IEqualityComparer<>).MakeGenericType(elemType));
|
||
}
|
||
Expression body = Expression.Call(minfo, param1, param2, comparerExpr);
|
||
Type funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
|
||
LambdaExpression procFunc = Expression.Lambda(funcType, body, param1, param2);
|
||
return new DryadApplyNode(procFunc, queryExpr, child1, child2);
|
||
}
|
||
|
||
private DryadQueryNode VisitHashPartition(QueryNodeInfo source,
|
||
LambdaExpression keySelectExpr,
|
||
Expression comparerExpr,
|
||
Expression countExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
|
||
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
|
||
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType),
|
||
queryExpr);
|
||
}
|
||
|
||
|
||
bool isDynamic = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicHashPartitionLevel) != 0;
|
||
|
||
int nOutputPartitions;
|
||
if (countExpr != null)
|
||
{
|
||
ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
|
||
nOutputPartitions = evaluator.Eval(countExpr);
|
||
isDynamic = false;
|
||
}
|
||
else
|
||
{
|
||
// Note: For MayRTM, isDynamic will never be true.
|
||
nOutputPartitions = (isDynamic) ? 1 : child.OutputPartition.Count;
|
||
}
|
||
|
||
DryadQueryNode resNode = new DryadHashPartitionNode(
|
||
keySelectExpr, null, comparerExpr, nOutputPartitions,
|
||
isDynamic, queryExpr, child);
|
||
resNode = new DryadMergeNode(false, true, queryExpr, resNode);
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitRangePartition(QueryNodeInfo source,
|
||
LambdaExpression keySelectExpr,
|
||
Expression keysExpr,
|
||
Expression comparerExpr,
|
||
Expression isDescendingExpr,
|
||
Expression partitionCountExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
|
||
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
|
||
string.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
|
||
queryExpr);
|
||
}
|
||
|
||
DryadQueryNode resNode;
|
||
if (keysExpr == null)
|
||
{
|
||
// case: no keys are provided -- create range partitioner with auto-separator-selection
|
||
bool dynamicOptEnabled = (StaticConfig.DynamicOptLevel & StaticConfig.DynamicRangePartitionLevel) != 0;
|
||
bool useDynamic = (dynamicOptEnabled || child.IsDynamic);
|
||
|
||
// NOTE: for MayRTM, useDynamic should always be false
|
||
resNode = this.CreateRangePartition(useDynamic, keySelectExpr, null, comparerExpr, isDescendingExpr,
|
||
queryExpr, partitionCountExpr, child);
|
||
}
|
||
else
|
||
{
|
||
// case: keys are local enum (eg an array) -- create range partitioner with keys input via Object-store.
|
||
resNode = new DryadRangePartitionNode(keySelectExpr, null, keysExpr, comparerExpr,
|
||
isDescendingExpr, null, queryExpr, child);
|
||
resNode = new DryadMergeNode(false, true, queryExpr, resNode);
|
||
}
|
||
return resNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitMultiApply(QueryNodeInfo source,
|
||
LambdaExpression procLambda,
|
||
bool perPartition,
|
||
bool isFirstOnly,
|
||
MethodCallExpression queryExpr)
|
||
{
|
||
DryadQueryNode[] childs = new DryadQueryNode[source.children.Count];
|
||
for (int i = 0; i < source.children.Count; ++i)
|
||
{
|
||
childs[i] = this.Visit(source.children[i].child);
|
||
}
|
||
|
||
bool isDynamic = childs.Any(x => x.IsDynamic);
|
||
if (perPartition && !isDynamic)
|
||
{
|
||
// Homomorphic case.
|
||
if (isFirstOnly)
|
||
{
|
||
for (int i = 1; i < childs.Length; ++i)
|
||
{
|
||
childs[i] = new DryadTeeNode(childs[i].OutputTypes[0], true, queryExpr, childs[i]);
|
||
childs[i].ConOpType = ConnectionOpType.CrossProduct;
|
||
childs[i] = new DryadMergeNode(childs[0].OutputPartition.Count, queryExpr, childs[i]);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
int count = childs[0].OutputPartition.Count;
|
||
for (int i = 1; i < childs.Length; ++i)
|
||
{
|
||
if (childs[i].OutputPartition.Count != count)
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.HomomorphicApplyNeedsSamePartitionCount,
|
||
SR.HomomorphicApplyNeedsSamePartitionCount,
|
||
queryExpr);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
else
|
||
{
|
||
// Non-homomorphic case.
|
||
for (int i = 0; i < childs.Length; ++i)
|
||
{
|
||
if (childs[i].IsDynamic || childs[i].OutputPartition.Count > 1)
|
||
{
|
||
childs[i] = new DryadMergeNode(true, false, queryExpr, childs[i]);
|
||
}
|
||
}
|
||
}
|
||
DryadQueryNode applyNode = new DryadApplyNode(procLambda, true, queryExpr, childs);
|
||
return applyNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitApply(QueryNodeInfo source1,
|
||
QueryNodeInfo source2,
|
||
LambdaExpression procLambda,
|
||
bool perPartition,
|
||
bool isFirstOnly,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child1 = this.Visit(source1);
|
||
|
||
DryadQueryNode applyNode;
|
||
if (source2 == null)
|
||
{
|
||
// Unary-apply case:
|
||
if (perPartition)
|
||
{
|
||
//homomorphic
|
||
applyNode = this.PromoteConcat(source1, child1, x => new DryadApplyNode(procLambda, queryExpr, x));
|
||
}
|
||
else
|
||
{
|
||
//non-homomorphic
|
||
if (child1.IsDynamic || child1.OutputPartition.Count > 1)
|
||
{
|
||
child1 = new DryadMergeNode(true, false, queryExpr, child1);
|
||
}
|
||
applyNode = new DryadApplyNode(procLambda, queryExpr, child1);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
// Binary-apply case:
|
||
DryadQueryNode child2 = this.Visit(source2);
|
||
|
||
if (perPartition && isFirstOnly)
|
||
{
|
||
// The function is left homomorphic:
|
||
if (!child2.IsForked && (child1.IsDynamic || child1.OutputPartition.Count > 1))
|
||
{
|
||
// The normal cases..
|
||
if (IsMergeNodeNeeded(child2))
|
||
{
|
||
if (child1.IsDynamic)
|
||
{
|
||
child2 = new DryadMergeNode(true, false, queryExpr, child2);
|
||
child2.IsForked = true;
|
||
}
|
||
else
|
||
{
|
||
// Rather than do full merge and broadcast, which has lots of data-movement
|
||
// 1. Tee output2 with output cross-product
|
||
// 2. Do a merge-stage which will have input1.nPartition nodes each performing a merge.
|
||
// This acheives a distribution of the entire input2 to the Apply nodes with least data-movement.
|
||
child2 = new DryadTeeNode(child2.OutputTypes[0], true, queryExpr, child2);
|
||
child2.ConOpType = ConnectionOpType.CrossProduct;
|
||
child2 = new DryadMergeNode(child1.OutputPartition.Count, queryExpr, child2);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
// the right-data is alread a single partition, so just tee it.
|
||
// this will provide a copy to each of the apply nodes.
|
||
child2 = new DryadTeeNode(child2.OutputTypes[0], true, queryExpr, child2);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
// Less common cases..
|
||
// a full merge of the right-data may be necessary.
|
||
if (child2.IsDynamic || child2.OutputPartition.Count > 1)
|
||
{
|
||
child2 = new DryadMergeNode(true, false, queryExpr, child2);
|
||
if (child1.IsDynamic || child1.OutputPartition.Count > 1)
|
||
{
|
||
child2.IsForked = true;
|
||
}
|
||
}
|
||
}
|
||
applyNode = new DryadApplyNode(procLambda, queryExpr, child1, child2);
|
||
}
|
||
else if (perPartition && !isFirstOnly && !child1.IsDynamic && !child2.IsDynamic)
|
||
{
|
||
// Full homomorphic
|
||
// No merging occurs.
|
||
// NOTE: We generally expect that both the left and right datasets have matching partitionCount.
|
||
// however, we don't test for it yet as users might know what they are doing, and it makes
|
||
// LocalDebug inconsistent as LocalDebug doesn't throw in that situation.
|
||
applyNode = new DryadApplyNode(procLambda, queryExpr, child1, child2);
|
||
}
|
||
else
|
||
{
|
||
// Non-homomorphic
|
||
// Full merges of both data sets is necessary.
|
||
if (child1.IsDynamic || child1.OutputPartition.Count > 1)
|
||
{
|
||
child1 = new DryadMergeNode(true, false, queryExpr, child1);
|
||
}
|
||
if (child2.IsDynamic || child2.OutputPartition.Count > 1)
|
||
{
|
||
child2 = new DryadMergeNode(true, false, queryExpr, child2);
|
||
}
|
||
applyNode = new DryadApplyNode(procLambda, queryExpr, child1, child2);
|
||
}
|
||
}
|
||
return applyNode;
|
||
}
|
||
|
||
private DryadQueryNode VisitFork(QueryNodeInfo source,
|
||
LambdaExpression forkLambda,
|
||
Expression keysExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
return new DryadForkNode(forkLambda, keysExpr, queryExpr, child);
|
||
}
|
||
|
||
private DryadQueryNode VisitForkChoose(QueryNodeInfo source,
|
||
Expression indexExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
|
||
int index = evaluator.Eval(indexExpr);
|
||
return ((DryadForkNode)child).Parents[index];
|
||
}
|
||
|
||
private DryadQueryNode VisitAssumeHashPartition(QueryNodeInfo source,
|
||
LambdaExpression keySelectExpr,
|
||
Expression keysExpr,
|
||
Expression comparerExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
|
||
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultEqualityComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable,
|
||
String.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIEquatable, keyType.FullName),
|
||
queryExpr);
|
||
}
|
||
|
||
object comparer = null;
|
||
if (comparerExpr != null)
|
||
{
|
||
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
|
||
comparer = evaluator.Eval(comparerExpr);
|
||
}
|
||
DataSetInfo outputInfo = new DataSetInfo(child.OutputDataSetInfo);
|
||
outputInfo.partitionInfo = PartitionInfo.CreateHash(keySelectExpr,
|
||
child.OutputPartition.Count,
|
||
comparer,
|
||
keyType);
|
||
child.OutputDataSetInfo = outputInfo;
|
||
return child;
|
||
}
|
||
|
||
private DryadQueryNode VisitAssumeRangePartition(QueryNodeInfo source,
|
||
LambdaExpression keySelectExpr,
|
||
Expression keysExpr,
|
||
Expression comparerExpr,
|
||
Expression isDescendingExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
|
||
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
|
||
String.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
|
||
queryExpr);
|
||
}
|
||
|
||
|
||
object comparer = null;
|
||
if (comparerExpr != null)
|
||
{
|
||
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
|
||
comparer = evaluator.Eval(comparerExpr);
|
||
}
|
||
|
||
object keys = null;
|
||
if (keysExpr != null)
|
||
{
|
||
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
|
||
keys = evaluator.Eval(keysExpr);
|
||
}
|
||
|
||
//count the number of keys provided.
|
||
if (keys != null)
|
||
{
|
||
int nSeparators = 0;
|
||
var ie = ((IEnumerable)keys).GetEnumerator();
|
||
while (ie.MoveNext())
|
||
{
|
||
nSeparators++;
|
||
}
|
||
|
||
if (!child.IsDynamic && nSeparators != child.PartitionCount - 1)
|
||
{
|
||
throw DryadLinqException.Create(
|
||
HpcLinqErrorCode.BadSeparatorCount,
|
||
String.Format(SR.BadSeparatorCount, nSeparators, child.PartitionCount - 1),
|
||
queryExpr);
|
||
}
|
||
}
|
||
|
||
|
||
bool? isDescending = null;
|
||
if (isDescendingExpr != null)
|
||
{
|
||
ExpressionSimplifier<bool> evaluator = new ExpressionSimplifier<bool>();
|
||
isDescending = evaluator.Eval(isDescendingExpr);
|
||
}
|
||
|
||
DataSetInfo outputInfo = new DataSetInfo(child.OutputDataSetInfo);
|
||
outputInfo.partitionInfo = PartitionInfo.CreateRange(keySelectExpr,
|
||
keys,
|
||
comparer,
|
||
isDescending,
|
||
child.OutputPartition.Count,
|
||
keyType);
|
||
child.OutputDataSetInfo = outputInfo;
|
||
return child;
|
||
}
|
||
|
||
private DryadQueryNode VisitAssumeOrderBy(QueryNodeInfo source,
|
||
LambdaExpression keySelectExpr,
|
||
Expression comparerExpr,
|
||
Expression isDescendingExpr,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
|
||
Type keyType = keySelectExpr.Type.GetGenericArguments()[1];
|
||
if (comparerExpr == null && !TypeSystem.HasDefaultComparer(keyType))
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable,
|
||
String.Format(SR.ComparerMustBeSpecifiedOrKeyTypeMustBeIComparable, keyType),
|
||
queryExpr);
|
||
}
|
||
|
||
object comparer = null;
|
||
if (comparerExpr != null)
|
||
{
|
||
ExpressionSimplifier<object> evaluator = new ExpressionSimplifier<object>();
|
||
comparer = evaluator.Eval(comparerExpr);
|
||
}
|
||
|
||
ExpressionSimplifier<bool> bevaluator = new ExpressionSimplifier<bool>();
|
||
bool isDescending = bevaluator.Eval(isDescendingExpr);
|
||
|
||
DataSetInfo outputInfo = new DataSetInfo(child.OutputDataSetInfo);
|
||
outputInfo.orderByInfo = OrderByInfo.Create(keySelectExpr, comparer, isDescending, keyType);
|
||
child.OutputDataSetInfo = outputInfo;
|
||
return child;
|
||
}
|
||
|
||
private DryadQueryNode VisitSlidingWindow(QueryNodeInfo source,
|
||
LambdaExpression procLambda,
|
||
Expression windowSizeExpr,
|
||
Expression queryExpr)
|
||
{
|
||
// var windows = source.Apply(s => HpcLinqHelper.Last(s, windowSize));
|
||
// var slided = windows.Apply(s => HpcLinqHelper.Slide(s)).HashPartition(x => x.Index);
|
||
// slided.Apply(source, (x, y) => HpcLinqHelper.ProcessWindows(x, y, procFunc, windowSize));
|
||
DryadQueryNode child = this.Visit(source);
|
||
if (child.IsDynamic)
|
||
{
|
||
throw new DryadLinqException("SlidingWindow is only supported for static partition count");
|
||
}
|
||
|
||
ExpressionSimplifier<int> evaluator = new ExpressionSimplifier<int>();
|
||
Expression windowSize = Expression.Constant(evaluator.Eval(windowSizeExpr), typeof(int));
|
||
|
||
child.IsForked = true;
|
||
|
||
// Apply node for s => Last(s, windowSize)
|
||
Type paramType = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
|
||
ParameterExpression param = Expression.Parameter(paramType, HpcLinqCodeGen.MakeUniqueName("s"));
|
||
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("Last");
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
|
||
Expression body = Expression.Call(minfo, param, windowSize);
|
||
Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
LambdaExpression procFunc = Expression.Lambda(funcType, body, param);
|
||
DryadQueryNode lastNode = new DryadApplyNode(procFunc, queryExpr, child);
|
||
lastNode = new DryadMergeNode(true, true, queryExpr, lastNode);
|
||
|
||
// Apply node for s => Slide(s)
|
||
param = Expression.Parameter(body.Type, HpcLinqCodeGen.MakeUniqueName("s"));
|
||
minfo = typeof(HpcLinqHelper).GetMethod("Slide");
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
|
||
body = Expression.Call(minfo, param);
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
procFunc = Expression.Lambda(funcType, body, param);
|
||
DryadQueryNode slideNode = new DryadApplyNode(procFunc, queryExpr, lastNode);
|
||
|
||
// Hash partition to distribute from partition i to i+1
|
||
int pcount = child.OutputPartition.Count;
|
||
param = Expression.Parameter(body.Type.GetGenericArguments()[0], "x");
|
||
Expression keySelectBody = Expression.Property(param, "Index");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, keySelectBody.Type);
|
||
LambdaExpression keySelectExpr = Expression.Lambda(funcType, keySelectBody, param);
|
||
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr,
|
||
null,
|
||
pcount,
|
||
queryExpr,
|
||
slideNode);
|
||
|
||
// Apply node for (x, y) => ProcessWindows(x, y, proclambda, windowSize)
|
||
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(body.Type);
|
||
ParameterExpression param1 = Expression.Parameter(paramType1, HpcLinqCodeGen.MakeUniqueName("x"));
|
||
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
|
||
ParameterExpression param2 = Expression.Parameter(paramType2, HpcLinqCodeGen.MakeUniqueName("y"));
|
||
minfo = typeof(HpcLinqHelper).GetMethod("ProcessWindows");
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0], procLambda.Body.Type);
|
||
body = Expression.Call(minfo, param1, param2, procLambda, windowSize);
|
||
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
|
||
procFunc = Expression.Lambda(funcType, body, param1, param2);
|
||
return new DryadApplyNode(procFunc, queryExpr, hdistNode, child);
|
||
}
|
||
|
||
private DryadQueryNode VisitApplyWithPartitionIndex(
|
||
QueryNodeInfo source,
|
||
LambdaExpression procLambda,
|
||
Expression queryExpr)
|
||
{
|
||
// var indices = source.Apply(s => ValueZero(s)).Apply(s => AssignIndex(s));
|
||
// indices.Apply(source, (x, y) => ProcessWithIndex(x, y, procFunc));
|
||
DryadQueryNode child = this.Visit(source);
|
||
if (child.IsDynamic)
|
||
{
|
||
throw new DryadLinqException("ApplyWithPartitionIndex is only supported for static partition count");
|
||
}
|
||
|
||
child.IsForked = true;
|
||
|
||
// Apply node for s => ValueZero(s)
|
||
Type paramType = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
|
||
ParameterExpression param = Expression.Parameter(paramType, "s");
|
||
MethodInfo minfo = typeof(HpcLinqHelper).GetMethod("ValueZero");
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0]);
|
||
Expression body = Expression.Call(minfo, param);
|
||
Type funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
LambdaExpression procFunc = Expression.Lambda(funcType, body, param);
|
||
DryadQueryNode valueZeroNode = new DryadApplyNode(procFunc, queryExpr, child);
|
||
valueZeroNode = new DryadMergeNode(true, true, queryExpr, valueZeroNode);
|
||
|
||
// Apply node for s => AssignIndex(s)
|
||
paramType = typeof(IEnumerable<>).MakeGenericType(typeof(int));
|
||
param = Expression.Parameter(paramType, "s");
|
||
minfo = typeof(HpcLinqHelper).GetMethod("AssignIndex");
|
||
body = Expression.Call(minfo, param);
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, body.Type);
|
||
procFunc = Expression.Lambda(funcType, body, param);
|
||
DryadQueryNode assignIndexNode = new DryadApplyNode(procFunc, queryExpr, valueZeroNode);
|
||
|
||
// HashPartition to distribute the indices -- one to each partition.
|
||
int pcount = child.OutputPartition.Count;
|
||
param = Expression.Parameter(body.Type, "x");
|
||
funcType = typeof(Func<,>).MakeGenericType(param.Type, param.Type);
|
||
LambdaExpression keySelectExpr = Expression.Lambda(funcType, param, param);
|
||
DryadQueryNode hdistNode = new DryadHashPartitionNode(keySelectExpr,
|
||
null,
|
||
pcount,
|
||
queryExpr,
|
||
assignIndexNode);
|
||
|
||
// Apply node for (x, y) => ProcessWithIndex(x, y, procLambda));
|
||
Type paramType1 = typeof(IEnumerable<>).MakeGenericType(child.OutputTypes[0]);
|
||
ParameterExpression param1 = Expression.Parameter(paramType1, HpcLinqCodeGen.MakeUniqueName("x"));
|
||
Type paramType2 = typeof(IEnumerable<>).MakeGenericType(typeof(int));
|
||
ParameterExpression param2 = Expression.Parameter(paramType2, HpcLinqCodeGen.MakeUniqueName("y"));
|
||
minfo = typeof(HpcLinqHelper).GetMethod("ProcessWithIndex");
|
||
minfo = minfo.MakeGenericMethod(child.OutputTypes[0], procLambda.Body.Type);
|
||
body = Expression.Call(minfo, param1, param2, procLambda);
|
||
funcType = typeof(Func<,,>).MakeGenericType(param1.Type, param2.Type, body.Type);
|
||
procFunc = Expression.Lambda(funcType, body, param1, param2);
|
||
return new DryadApplyNode(procFunc, queryExpr, child, hdistNode);
|
||
}
|
||
|
||
private DryadQueryNode VisitFirst(QueryNodeInfo source,
|
||
LambdaExpression lambda,
|
||
AggregateOpType aggType,
|
||
bool isQuery,
|
||
Expression queryExpr)
|
||
{
|
||
DryadQueryNode child = this.Visit(source);
|
||
DryadQueryNode resNode = this.PromoteConcat(
|
||
source, child,
|
||
x => new DryadBasicAggregateNode(lambda, aggType, true, isQuery, queryExpr, x));
|
||
return new DryadBasicAggregateNode(null, aggType, false, isQuery, queryExpr, resNode);
|
||
}
|
||
|
||
private DryadQueryNode VisitThenBy(QueryNodeInfo source,
|
||
LambdaExpression keySelectExpr,
|
||
bool isDescending,
|
||
Expression queryExpr)
|
||
{
|
||
// YY: This makes it hard to maintain OrderByInfo.
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
|
||
String.Format(SR.OperatorNotSupported, "ThenBy"),
|
||
queryExpr);
|
||
}
|
||
|
||
private DryadQueryNode VisitDefaultIfEmpty(QueryNodeInfo source,
|
||
Expression defaultValueExpr,
|
||
MethodCallExpression queryExpr)
|
||
{
|
||
// YY: Not very useful. We could add it later.
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
|
||
String.Format(SR.OperatorNotSupported, "DefaultIfEmpty"),
|
||
queryExpr);
|
||
}
|
||
|
||
private DryadQueryNode VisitElementAt(string opName,
|
||
QueryNodeInfo source,
|
||
Expression indexExpr,
|
||
Expression queryExpr)
|
||
{
|
||
// YY: Not very useful. We could add it later.
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
|
||
String.Format(SR.OperatorNotSupported, opName),
|
||
queryExpr);
|
||
}
|
||
|
||
private DryadQueryNode VisitOfType(QueryNodeInfo source,
|
||
Type ofType,
|
||
Expression queryExpr)
|
||
{
|
||
// YY: Not very useful.
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
|
||
String.Format(SR.OperatorNotSupported, "OfType"),
|
||
queryExpr);
|
||
}
|
||
|
||
private DryadQueryNode VisitQueryOperatorCall(QueryNodeInfo nodeInfo)
|
||
{
|
||
DryadQueryNode resNode = nodeInfo.queryNode;
|
||
if (resNode != null) return resNode;
|
||
|
||
MethodCallExpression expression = (MethodCallExpression)nodeInfo.queryExpression;
|
||
string methodName = expression.Method.Name;
|
||
|
||
#region LINQMETHODS
|
||
switch (methodName)
|
||
{
|
||
case "Aggregate":
|
||
case "AggregateAsQuery":
|
||
{
|
||
bool isQuery = (methodName == "AggregateAsQuery");
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression funcLambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (funcLambda != null && funcLambda.Parameters.Count == 2)
|
||
{
|
||
resNode = this.VisitAggregate(nodeInfo.children[0].child,
|
||
null,
|
||
funcLambda,
|
||
null,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
LambdaExpression funcLambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
if (funcLambda != null && funcLambda.Parameters.Count == 2)
|
||
{
|
||
resNode = this.VisitAggregate(nodeInfo.children[0].child,
|
||
expression.Arguments[1],
|
||
funcLambda,
|
||
null,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 4)
|
||
{
|
||
LambdaExpression funcLambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
LambdaExpression resultLambda = HpcLinqExpression.GetLambda(expression.Arguments[3]);
|
||
if (funcLambda != null && funcLambda.Parameters.Count == 2 &&
|
||
resultLambda != null && resultLambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitAggregate(nodeInfo.children[0].child,
|
||
expression.Arguments[1],
|
||
funcLambda,
|
||
resultLambda,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Select":
|
||
case "LongSelect":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && (lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
|
||
{
|
||
resNode = this.VisitSelect(nodeInfo.children[0].child,
|
||
QueryNodeType.Select,
|
||
lambda,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "SelectMany":
|
||
case "LongSelectMany":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count <= 2)
|
||
{
|
||
resNode = this.VisitSelect(nodeInfo.children[0].child,
|
||
QueryNodeType.SelectMany,
|
||
lambda,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
LambdaExpression lambda1 = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
if (lambda1 != null && (lambda1.Parameters.Count == 1 || lambda1.Parameters.Count == 2) &&
|
||
lambda2 != null && lambda2.Parameters.Count == 2)
|
||
{
|
||
resNode = this.VisitSelect(nodeInfo.children[0].child,
|
||
QueryNodeType.SelectMany,
|
||
lambda1,
|
||
lambda2,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Join":
|
||
case "GroupJoin":
|
||
{
|
||
QueryNodeType nodeType = (methodName == "Join") ? QueryNodeType.Join : QueryNodeType.GroupJoin;
|
||
if (expression.Arguments.Count == 5)
|
||
{
|
||
LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
LambdaExpression lambda3 = HpcLinqExpression.GetLambda(expression.Arguments[3]);
|
||
LambdaExpression lambda4 = HpcLinqExpression.GetLambda(expression.Arguments[4]);
|
||
if (lambda2 != null && lambda2.Parameters.Count == 1 &&
|
||
lambda3 != null && lambda3.Parameters.Count == 1 &&
|
||
lambda4 != null && lambda4.Parameters.Count == 2)
|
||
{
|
||
resNode = this.VisitJoin(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
nodeType,
|
||
lambda2,
|
||
lambda3,
|
||
lambda4,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 6)
|
||
{
|
||
LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
LambdaExpression lambda3 = HpcLinqExpression.GetLambda(expression.Arguments[3]);
|
||
LambdaExpression lambda4 = HpcLinqExpression.GetLambda(expression.Arguments[4]);
|
||
if (lambda2 != null && lambda2.Parameters.Count == 1 &&
|
||
lambda3 != null && lambda3.Parameters.Count == 1 &&
|
||
lambda4 != null && lambda4.Parameters.Count == 2)
|
||
{
|
||
resNode = this.VisitJoin(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
nodeType,
|
||
lambda2,
|
||
lambda3,
|
||
lambda4,
|
||
expression.Arguments[5],
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "OfType":
|
||
{
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
Type ofType = expression.Method.GetGenericArguments()[0];
|
||
resNode = this.VisitOfType(nodeInfo.children[0].child, ofType, expression);
|
||
}
|
||
break;
|
||
}
|
||
case "Where":
|
||
case "LongWhere":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && (lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
|
||
{
|
||
resNode = this.VisitWhere(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "First":
|
||
case "FirstOrDefault":
|
||
case "FirstAsQuery":
|
||
{
|
||
AggregateOpType aggType = (methodName == "FirstOrDefault") ? AggregateOpType.FirstOrDefault : AggregateOpType.First;
|
||
bool isQuery = (methodName == "FirstAsQuery");
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitFirst(nodeInfo.children[0].child, null, aggType, isQuery, expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitFirst(nodeInfo.children[0].child,
|
||
lambda,
|
||
aggType,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Single":
|
||
case "SingleOrDefault":
|
||
case "SingleAsQuery":
|
||
{
|
||
AggregateOpType aggType = (methodName == "SingleOrDefault") ? AggregateOpType.SingleOrDefault : AggregateOpType.Single;
|
||
bool isQuery = (methodName == "SingleAsQuery");
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitFirst(nodeInfo.children[0].child, null, aggType, isQuery, expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitFirst(nodeInfo.children[0].child,
|
||
lambda,
|
||
aggType,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Last":
|
||
case "LastOrDefault":
|
||
case "LastAsQuery":
|
||
{
|
||
AggregateOpType aggType = (methodName == "LastOrDefault") ? AggregateOpType.LastOrDefault : AggregateOpType.Last;
|
||
bool isQuery = (methodName == "LastAsQuery");
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitFirst(nodeInfo.children[0].child, null, aggType, isQuery, expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitFirst(nodeInfo.children[0].child,
|
||
lambda,
|
||
aggType,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Distinct":
|
||
{
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitDistinct(nodeInfo.children[0].child, null, expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitDistinct(nodeInfo.children[0].child,
|
||
expression.Arguments[1],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "DefaultIfEmpty":
|
||
{
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitDefaultIfEmpty(nodeInfo.children[0].child,
|
||
null,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitDefaultIfEmpty(nodeInfo.children[0].child,
|
||
expression.Arguments[1],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "Concat":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitConcat(nodeInfo, expression);
|
||
}
|
||
break;
|
||
}
|
||
case "Union":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
QueryNodeType.Union,
|
||
null,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
QueryNodeType.Union,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "Intersect":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
QueryNodeType.Intersect,
|
||
null,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
QueryNodeType.Intersect,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "Except":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
QueryNodeType.Except,
|
||
null,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
resNode = this.VisitSetOperation(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
QueryNodeType.Except,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "Any":
|
||
case "AnyAsQuery":
|
||
{
|
||
bool isQuery = (methodName == "AnyAsQuery");
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
Type type = expression.Method.GetGenericArguments()[0];
|
||
ParameterExpression param = Expression.Parameter(type, "x");
|
||
Type delegateType = typeof(Func<,>).MakeGenericType(type, typeof(bool));
|
||
Expression body = Expression.Constant(true);
|
||
LambdaExpression lambda = Expression.Lambda(delegateType, body, param);
|
||
resNode = this.VisitQuantifier(nodeInfo.children[0].child,
|
||
lambda,
|
||
AggregateOpType.Any,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitQuantifier(nodeInfo.children[0].child,
|
||
lambda,
|
||
AggregateOpType.Any,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "All":
|
||
case "AllAsQuery":
|
||
{
|
||
bool isQuery = (methodName == "AllAsQuery");
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitQuantifier(nodeInfo.children[0].child,
|
||
lambda,
|
||
AggregateOpType.All,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Count":
|
||
case "CountAsQuery":
|
||
{
|
||
bool isQuery = (methodName == "CountAsQuery");
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
null,
|
||
AggregateOpType.Count,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
lambda,
|
||
AggregateOpType.Count,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "LongCount":
|
||
case "LongCountAsQuery":
|
||
{
|
||
bool isQuery = (methodName == "LongCountAsQuery");
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
null,
|
||
AggregateOpType.LongCount,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
lambda,
|
||
AggregateOpType.LongCount,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Sum":
|
||
case "SumAsQuery":
|
||
{
|
||
bool isQuery = (methodName == "SumAsQuery");
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
null,
|
||
AggregateOpType.Sum,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
lambda,
|
||
AggregateOpType.Sum,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Min":
|
||
case "MinAsQuery":
|
||
{
|
||
bool isQuery = (methodName == "MinAsQuery");
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
null,
|
||
AggregateOpType.Min,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
lambda,
|
||
AggregateOpType.Min,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Max":
|
||
case "MaxAsQuery":
|
||
{
|
||
bool isQuery = (methodName == "MaxAsQuery");
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
null,
|
||
AggregateOpType.Max,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
lambda,
|
||
AggregateOpType.Max,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Average":
|
||
case "AverageAsQuery":
|
||
{
|
||
bool isQuery = (methodName == "AverageAsQuery");
|
||
if (expression.Arguments.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
null,
|
||
AggregateOpType.Average,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitBasicAggregate(nodeInfo.children[0].child,
|
||
lambda,
|
||
AggregateOpType.Average,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "GroupBy":
|
||
{
|
||
// groupby can take 2, 3, 4, or 5 arguments.
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
//Supplied arguments are as follows:(source, key selector)
|
||
LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (keySelExpr != null && keySelExpr.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
|
||
keySelExpr,
|
||
null,
|
||
null,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
//Supplied arguments are as follows:(source, key selector, element selector/result selector/comparer)
|
||
LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
if (keySelExpr != null && lambda2 == null)
|
||
{
|
||
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
|
||
keySelExpr,
|
||
null,
|
||
null,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
else if (keySelExpr != null && keySelExpr.Parameters.Count == 1 && lambda2 != null)
|
||
{
|
||
LambdaExpression elemSelExpr = null;
|
||
LambdaExpression resSelExpr = null;
|
||
if (lambda2.Parameters.Count == 1)
|
||
{
|
||
elemSelExpr = lambda2;
|
||
}
|
||
else if (lambda2.Parameters.Count == 2)
|
||
{
|
||
resSelExpr = lambda2;
|
||
}
|
||
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
|
||
keySelExpr,
|
||
elemSelExpr,
|
||
resSelExpr,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 4)
|
||
{
|
||
//Argument-0 is source and Argument-1 is key selector expression
|
||
LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
LambdaExpression lambda2 = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
LambdaExpression lambda3 = HpcLinqExpression.GetLambda(expression.Arguments[3]);
|
||
if (keySelExpr != null && keySelExpr.Parameters.Count == 1 && lambda3 == null)
|
||
{
|
||
//Argument-2 can be either result selector, element selector and argument-3 is comparer
|
||
LambdaExpression elemSelExpr = null;
|
||
LambdaExpression resSelExpr = null;
|
||
if (lambda2.Parameters.Count == 1)
|
||
{
|
||
elemSelExpr = lambda2;
|
||
}
|
||
else if (lambda2.Parameters.Count == 2)
|
||
{
|
||
resSelExpr = lambda2;
|
||
}
|
||
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
|
||
keySelExpr,
|
||
elemSelExpr,
|
||
resSelExpr,
|
||
expression.Arguments[3],
|
||
expression);
|
||
}
|
||
else if (keySelExpr != null && keySelExpr.Parameters.Count == 1 &&
|
||
lambda2 != null && lambda2.Parameters.Count == 1 &&
|
||
lambda3 != null && lambda3.Parameters.Count == 2)
|
||
{
|
||
//Argument-2 is element selector and argument-3 is result selector
|
||
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
|
||
keySelExpr,
|
||
lambda2,
|
||
lambda3,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 5)
|
||
{
|
||
//Supplied arguments are as follows:(source, key selector, element selector, result selector, comparer)
|
||
LambdaExpression keySelExpr = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
LambdaExpression elemSelExpr = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
LambdaExpression resSelExpr = HpcLinqExpression.GetLambda(expression.Arguments[3]);
|
||
if (keySelExpr != null && keySelExpr.Parameters.Count == 1 &&
|
||
elemSelExpr != null && elemSelExpr.Parameters.Count == 1 &&
|
||
resSelExpr != null && resSelExpr.Parameters.Count == 2)
|
||
{
|
||
resNode = this.VisitGroupBy(nodeInfo.children[0].child,
|
||
keySelExpr,
|
||
elemSelExpr,
|
||
resSelExpr,
|
||
expression.Arguments[4],
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "OrderBy":
|
||
case "OrderByDescending":
|
||
{
|
||
bool isDescending = (methodName == "OrderByDescending");
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitOrderBy(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
isDescending,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitOrderBy(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
isDescending,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "ThenBy":
|
||
case "ThenByDescending":
|
||
{
|
||
bool isDescending = (methodName == "ThenByDescending");
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitThenBy(nodeInfo.children[0].child,
|
||
lambda,
|
||
isDescending,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "ElementAt":
|
||
case "ElementAtOrDefault":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitElementAt(methodName,
|
||
nodeInfo.children[0].child,
|
||
expression.Arguments[1],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "Take":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitPartitionOp(methodName,
|
||
nodeInfo.children[0].child,
|
||
QueryNodeType.Take,
|
||
expression.Arguments[1],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "TakeWhile":
|
||
case "LongTakeWhile":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null &&
|
||
(lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
|
||
{
|
||
resNode = this.VisitPartitionOp("TakeWhile",
|
||
nodeInfo.children[0].child,
|
||
QueryNodeType.TakeWhile,
|
||
lambda,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Skip":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitPartitionOp(methodName,
|
||
nodeInfo.children[0].child,
|
||
QueryNodeType.Skip,
|
||
expression.Arguments[1],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "SkipWhile":
|
||
case "LongSkipWhile":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null &&
|
||
(lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
|
||
{
|
||
resNode = this.VisitPartitionOp(methodName,
|
||
nodeInfo.children[0].child,
|
||
QueryNodeType.SkipWhile,
|
||
lambda,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Contains":
|
||
case "ContainsAsQuery":
|
||
{
|
||
bool isQuery = (methodName == "ContainsAsQuery");
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitContains(nodeInfo.children[0].child,
|
||
expression.Arguments[1],
|
||
null,
|
||
isQuery,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
resNode = this.VisitContains(nodeInfo.children[0].child,
|
||
expression.Arguments[1],
|
||
expression.Arguments[2],
|
||
isQuery,
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "Reverse":
|
||
{
|
||
resNode = this.VisitReverse(nodeInfo.children[0].child, expression);
|
||
break;
|
||
}
|
||
case "SequenceEqual":
|
||
case "SequenceEqualAsQuery":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitSequenceEqual(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
null,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
resNode = this.VisitSequenceEqual(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "Zip":
|
||
{
|
||
if (expression.Arguments.Count == 3)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
if (lambda != null && lambda.Parameters.Count == 2)
|
||
{
|
||
resNode = this.VisitZip(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
lambda,
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "HashPartition":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitHashPartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
if (expression.Arguments[2].Type == typeof(int))
|
||
{
|
||
resNode = this.VisitHashPartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
else
|
||
{
|
||
resNode = this.VisitHashPartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 4)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitHashPartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
expression.Arguments[3],
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "RangePartition":
|
||
{
|
||
//overloads:
|
||
//
|
||
// 2-param:
|
||
// (source, keySelector)
|
||
//
|
||
// 3-param:
|
||
// (source, keySelector, pcount)
|
||
// (source, keySelector, isDescending)
|
||
// (source, keySelector, rangeSeparators)
|
||
//
|
||
// 4-param:
|
||
// (source, keySelector, isDescending, pcount)
|
||
// (source, keySelector, keyComparer, isDescending)
|
||
// (source, keySelector, rangeSeparators, keyComparer)
|
||
//
|
||
// 5-param:
|
||
// (source, keySelector, keyComparer, isDescending, pcount)
|
||
// (source, keySelector, rangeSeparators, keyComparer, isDescending)
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
// Case: (source, keySelector)
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
null,
|
||
null,
|
||
null,
|
||
expression);
|
||
}
|
||
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
if (expression.Arguments[2].Type == typeof(int))
|
||
{
|
||
// Case: (source, keySelector, pcount)
|
||
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
null,
|
||
null,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
if (expression.Arguments[2].Type == typeof(bool))
|
||
{
|
||
// Case: (source, keySelector, isDescending)
|
||
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
null,
|
||
expression.Arguments[2],
|
||
null,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments[2].Type.IsArray)
|
||
{
|
||
// Case: RangePartition(keySelector, TKey[] keys)
|
||
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
null,
|
||
null,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 4)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
if (expression.Arguments[2].Type == typeof(bool))
|
||
{
|
||
//case: (source, keySelector, isDescending, pcount)
|
||
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
null,
|
||
expression.Arguments[2],
|
||
expression.Arguments[3],
|
||
expression);
|
||
}
|
||
else if (expression.Arguments[3].Type == typeof(bool))
|
||
{
|
||
//case: (source, keySelector, keyComparer, isDescending)
|
||
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
expression.Arguments[2],
|
||
expression.Arguments[3],
|
||
null,
|
||
expression);
|
||
}
|
||
else if (expression.Arguments[2].Type.IsArray)
|
||
{
|
||
//case: (source, keySelector, rangeSeparators, keyComparer)
|
||
|
||
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
expression.Arguments[3],
|
||
null,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 5)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
if (expression.Arguments[3].Type == typeof(bool))
|
||
{
|
||
// case: (source, keySelector, keyComparer, isDescending, pcount)
|
||
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
expression.Arguments[2],
|
||
expression.Arguments[3],
|
||
expression.Arguments[4],
|
||
expression);
|
||
}
|
||
else if (expression.Arguments[4].Type == typeof(bool))
|
||
{
|
||
// case: (source, keySelector, rangeSeparators, keyComparer, isDescending)
|
||
resNode = this.VisitRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
expression.Arguments[3],
|
||
expression.Arguments[4],
|
||
null,
|
||
expression);
|
||
}
|
||
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Apply":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null &&
|
||
(lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
|
||
{
|
||
resNode = this.VisitApply(nodeInfo.children[0].child,
|
||
null,
|
||
lambda,
|
||
false,
|
||
false,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
if (lambda != null)
|
||
{
|
||
if (lambda.Parameters.Count == 2)
|
||
{
|
||
resNode = this.VisitApply(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
lambda,
|
||
false,
|
||
false,
|
||
expression);
|
||
}
|
||
else
|
||
{
|
||
// Apply with multiple sources of the same type
|
||
resNode = this.VisitMultiApply(nodeInfo, lambda, false, false, expression);
|
||
}
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "ApplyPerPartition":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null &&
|
||
(lambda.Parameters.Count == 1 || lambda.Parameters.Count == 2))
|
||
{
|
||
resNode = this.VisitApply(nodeInfo.children[0].child,
|
||
null,
|
||
lambda,
|
||
true,
|
||
false,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 4)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[2]);
|
||
ExpressionSimplifier<bool> evaluator = new ExpressionSimplifier<bool>();
|
||
bool isFirstOnly = evaluator.Eval(expression.Arguments[3]);
|
||
if (lambda != null)
|
||
{
|
||
if (lambda.Parameters.Count == 2)
|
||
{
|
||
resNode = this.VisitApply(nodeInfo.children[0].child,
|
||
nodeInfo.children[1].child,
|
||
lambda,
|
||
true,
|
||
isFirstOnly,
|
||
expression);
|
||
}
|
||
else
|
||
{
|
||
// Apply with multiple sources of the same type
|
||
resNode = this.VisitMultiApply(nodeInfo, lambda, true, isFirstOnly, expression);
|
||
}
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "Fork":
|
||
{
|
||
if (expression.Arguments.Count == 2) // ForkSelect and ForkApply
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitFork(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 3) // ForkByKey
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitFork(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "ForkChoose":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
resNode = this.VisitForkChoose(nodeInfo.children[0].child,
|
||
expression.Arguments[1],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "AssumeHashPartition":
|
||
{
|
||
if (expression.Arguments.Count == 2)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitAssumeHashPartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
null,
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 3)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitAssumeHashPartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "AssumeRangePartition":
|
||
{
|
||
if (expression.Arguments.Count == 3)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
if (expression.Arguments[2].Type.IsArray)
|
||
{
|
||
resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
null,
|
||
null,
|
||
expression);
|
||
}
|
||
else
|
||
{
|
||
resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
null,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 4)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
if (expression.Arguments[2].Type.IsArray)
|
||
{
|
||
resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
expression.Arguments[3],
|
||
null,
|
||
expression);
|
||
}
|
||
else
|
||
{
|
||
resNode = this.VisitAssumeRangePartition(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
expression.Arguments[2],
|
||
expression.Arguments[3],
|
||
expression);
|
||
}
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "AssumeOrderBy":
|
||
{
|
||
if (expression.Arguments.Count == 3)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitAssumeOrderBy(nodeInfo.children[0].child,
|
||
lambda,
|
||
null,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
}
|
||
else if (expression.Arguments.Count == 4)
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitAssumeOrderBy(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
expression.Arguments[3],
|
||
expression);
|
||
}
|
||
}
|
||
break;
|
||
}
|
||
case "SlidingWindow":
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitSlidingWindow(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression.Arguments[2],
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case "SelectWithPartitionIndex":
|
||
case "ApplyWithPartitionIndex":
|
||
{
|
||
LambdaExpression lambda = HpcLinqExpression.GetLambda(expression.Arguments[1]);
|
||
if (lambda != null && lambda.Parameters.Count == 1)
|
||
{
|
||
resNode = this.VisitApplyWithPartitionIndex(nodeInfo.children[0].child,
|
||
lambda,
|
||
expression);
|
||
}
|
||
break;
|
||
}
|
||
case ReflectedNames.DryadLinqIQueryable_ToDscWorker: // was case "ToPartitionedTableLazy":
|
||
{
|
||
//SHOULD NOT VISIT.. The DryadQueryGen ctors should be interrogating ToDsc nodes directly.
|
||
//Later if we do allow ToDsc in the middle of query chain, then we need to either
|
||
// 1. update the source node with an outputeTableUri
|
||
// OR 2. create an actual node and handle it later on (tee, etc)
|
||
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ToDscUsedIncorrectly,
|
||
String.Format(SR.ToDscUsedIncorrectly),
|
||
expression);
|
||
}
|
||
case ReflectedNames.DryadLinqIQueryable_ToHdfsWorker: // was case "ToPartitionedTableLazy":
|
||
{
|
||
//SHOULD NOT VISIT.. The DryadQueryGen ctors should be interrogating ToDsc nodes directly.
|
||
//Later if we do allow ToDsc in the middle of query chain, then we need to either
|
||
// 1. update the source node with an outputeTableUri
|
||
// OR 2. create an actual node and handle it later on (tee, etc)
|
||
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.ToHdfsUsedIncorrectly, String.Format(SR.ToHdfsUsedIncorrectly), expression);
|
||
}
|
||
}
|
||
#endregion
|
||
|
||
if (resNode == null)
|
||
{
|
||
throw DryadLinqException.Create(HpcLinqErrorCode.OperatorNotSupported,
|
||
String.Format(SR.OperatorNotSupported, methodName),
|
||
expression);
|
||
}
|
||
resNode.IsForked = resNode.IsForked || nodeInfo.IsForked;
|
||
nodeInfo.queryNode = resNode;
|
||
return resNode;
|
||
}
|
||
}
|
||
}
|