berkus
/
Dryad
mirror of https://github.com/MicrosoftResearch/Dryad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

version 0.1.5 release

This commit is contained in:
Dennis Fetterly 2014-06-10 16:05:37 -07:00
parent 93be3e4b66
commit 5c260ffe0b
78 changed files with 6603 additions and 1975 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -19,3 +19,5 @@ x64/
build/
[Bb]in/
[Oo]bj/
/Calypso/packages
/DryadLinqTests/packages

72
ClusterInterface/ClusterInterface.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -45,75 +45,81 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@ -147,10 +153,10 @@
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

4
ClusterInterface/HttpScheduler.cs
View File

@ -90,10 +90,10 @@ namespace Microsoft.Research.Dryad.ClusterInterface
return fileServer.Uri.AbsoluteUri;
}
public IProcess NewProcess(IProcessWatcher watcher, string commandLine)
public IProcess NewProcess(IProcessWatcher watcher, string commandLine, string commandLineArguments)
{
ISchedulerProcess process = scheduler.NewProcess();
return new Process(process, watcher, commandLine, logger);
return new Process(process, watcher, commandLine, commandLineArguments, logger);
}
public void ScheduleProcess(IProcess ip, List<Affinity> affinities)

3
ClusterInterface/Interfaces.cs
View File

@ -368,8 +368,9 @@ namespace Microsoft.Research.Dryad.ClusterInterface
/// the application on the process' lifecycle</param>
/// <param name="commandLine">the command line to use to start the process on the remote
/// computer</param>
/// <param name="commandLineArguments">arguments to provide to the remote process</param>
/// <returns>a handle to the new process</returns>
IProcess NewProcess(IProcessWatcher watcher, string commandLine);
IProcess NewProcess(IProcessWatcher watcher, string commandLine, string commandLineArguments);
/// <summary>
/// request that a Process object, return from NewProcess, be scheduled according

23
ClusterInterface/Process.cs
View File

@ -56,11 +56,6 @@ namespace Microsoft.Research.Dryad.ClusterInterface
/// </summary>
private State state;
/// <summary>
/// string used to start the remote process
/// </summary>
private string commandLine;
/// <summary>
/// this is the handle that the scheduler supplies that is used to refer
/// to the process
@ -114,11 +109,12 @@ namespace Microsoft.Research.Dryad.ClusterInterface
/// <summary>
/// construct a new object to represent the lifecycle of a process being scheduled
/// </summary>
public Process(ISchedulerProcess p, IProcessWatcher w, string cmd, ILogger l)
public Process(ISchedulerProcess p, IProcessWatcher w, string cmd, string cmdLineArgs, ILogger l)
{
schedulerProcess = p;
state = State.Initializing;
commandLine = cmd;
CommandLine = cmd;
CommandLineArguments = cmdLineArgs;
watcher = w;
cancelTask = new TaskCompletionSource<bool>();
directory = null;
@ -137,7 +133,12 @@ namespace Microsoft.Research.Dryad.ClusterInterface
/// <summary>
/// the string used to start the remote process
/// </summary>
public string CommandLine { get { return commandLine; } }
public string CommandLine { get; private set; }
/// <summary>
/// arguments provided when starting the remote process
/// </summary>
public string CommandLineArguments { get; private set; }
/// <summary>
/// the local directory of the process at the daemon's host computer
@ -198,9 +199,13 @@ namespace Microsoft.Research.Dryad.ClusterInterface
logger.Log("Process " + Id + " scheduling itself as " + directory + " on computer " + computer.Name + " at " + computer.Host);
ToMatched(computer, DateTime.Now.ToFileTimeUtc());
StringBuilder payload = new StringBuilder();
payload.AppendLine(CommandLine);
payload.AppendLine(CommandLineArguments);
Task<string> bail = interrupt.ContinueWith((t) => "");
Task<string> upload = PostRequest(computer, directory + "?op=create", Encoding.UTF8.GetBytes(commandLine));
Task<string> upload = PostRequest(computer, directory + "?op=create", Encoding.UTF8.GetBytes(payload.ToString()));
Task<string> completed = await Task.WhenAny(bail, upload);
if (completed == bail)

6
ClusterInterface/Properties/AssemblyInfo.cs
View File

@ -28,7 +28,7 @@ using System.Runtime.InteropServices;
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyProduct("DryadHttpClusterInterface")]
[assembly: AssemblyProduct("Dryad")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.3.0")]
[assembly: AssemblyFileVersion("0.1.3.0")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

2
ClusterInterface/app.config
View File

@ -4,7 +4,7 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />

24
ClusterInterface/packages.config
View File

@ -1,21 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages>

6
Dryad.sln
View File

@ -78,8 +78,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalScheduler", "LocalSche
{A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6} = {A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6}
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadLinqTests", "DryadLinqTests\DryadLinqTests.csproj", "{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64
@ -146,10 +144,6 @@ Global
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Debug|x64.Build.0 = Debug|x64
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.ActiveCfg = Release|x64
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.Build.0 = Release|x64
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Debug|x64.ActiveCfg = Debug|x64
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Debug|x64.Build.0 = Debug|x64
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Release|x64.ActiveCfg = Release|x64
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

58
DryadLinqGraphManager/DryadLinqGraphManager.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -65,68 +65,68 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core">
@ -138,13 +138,13 @@
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@ -212,10 +212,10 @@
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

6
DryadLinqGraphManager/Properties/AssemblyInfo.cs
View File

@ -29,7 +29,7 @@ using System.Runtime.InteropServices;
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyProduct("DryadLinqGraphManager")]
[assembly: AssemblyProduct("Dryad")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.3.0")]
[assembly: AssemblyFileVersion("0.1.3.0")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

10
DryadLinqGraphManager/app.config
View File

@ -19,12 +19,20 @@
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.0.7" newVersion="1.1.0.7" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.0.7" newVersion="1.1.0.7" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

24
DryadLinqGraphManager/packages.config
View File

@ -1,21 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages>

24
DryadLinqTests/App.config
View File

@ -5,6 +5,14 @@
</startup>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
@ -18,12 +26,20 @@
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
</assemblyBinding>
</runtime>

370
DryadLinqTests/ApplyAndForkTests.cs
View File

@ -3,11 +3,38 @@ using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace DryadLinqTests
{
public class ApplyAndForkTests
{
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" ApplyAndForkTests ");
TestLog.Message(" **********************");
var tests = new Dictionary<string, Action>()
{
{"NonHomomorphicUnaryApply", () => NonHomomorphicUnaryApply(context) },
{"HomomorphicUnaryApply", () => HomomorphicUnaryApply(context) },
{"NonHomomorphicBinaryApply", () => NonHomomorphicBinaryApply(context) },
{"LeftHomomorphicBinaryApply", () => LeftHomomorphicBinaryApply(context) },
{"FullHomomorphicBinaryApply_DifferentDataSets", () => FullHomomorphicBinaryApply_DifferentDataSets(context) },
{"FullHomomorphicBinaryApply_IdenticalDataSets", () => FullHomomorphicBinaryApply_IdenticalDataSets(context) },
{"Aggregate_WithCombiner", () => Aggregate_WithCombiner(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static IEnumerable<int> NonHomomorphic_Unary_Func(IEnumerable<int> input)
{
return input;
@ -57,177 +84,328 @@ namespace DryadLinqTests
throw new Exception("a node received empty right-data");
}
public static bool NonHomomorphicUnaryApply()
public static bool NonHomomorphicUnaryApply(DryadLinqContext context)
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
string testName = "NonHomomorphicUnaryApply";
TestLog.TestStart(testName);
bool passed = true;
try
{
string outFile = "unittest/output/NonHomomorphicUnaryApply.out";
IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/NonHomomorphicUnaryApply";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Validate.outFileExists(outFile);
result[0] = q1;
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (DryadLinqException)
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool HomomorphicUnaryApply()
public static bool HomomorphicUnaryApply(DryadLinqContext context)
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
string testName = "HomomorphicUnaryApply";
TestLog.TestStart(testName);
bool passed = true;
try
{
string outFile = "unittest/output/HomomorphicUnaryApply.out";
IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/HomomorphicUnaryApply";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Validate.outFileExists(outFile);
result[0] = q1;
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (DryadLinqException)
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool NonHomomorphicBinaryApply()
public static bool NonHomomorphicBinaryApply(DryadLinqContext context)
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
string testName = "NonHomomorphicBinaryApply";
TestLog.TestStart(testName);
bool passed = true;
try
{
string outFile = "unittest/output/NonHomomorphicBinaryApply.out";
IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/NonHomomorphicBinaryApply";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Validate.outFileExists(outFile);
result[0] = q1;
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (DryadLinqException)
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool LeftHomomorphicBinaryApply()
public static bool LeftHomomorphicBinaryApply(DryadLinqContext context)
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
string testName = "LeftHomomorphicBinaryApply";
TestLog.TestStart(testName);
bool passed = true;
try
{
string outFile = "unittest/output/LeftHomomorphicBinaryApply.out";
IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/LeftHomomorphicBinaryApply";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Validate.outFileExists(outFile);
result[0] = q1;
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (DryadLinqException)
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool FullHomomorphicBinaryApply_DifferentDataSets()
public static bool FullHomomorphicBinaryApply_DifferentDataSets(DryadLinqContext context)
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
string testName = "FullHomomorphicBinaryApply_DifferentDataSets";
TestLog.TestStart(testName);
bool passed = true;
try
{
string outFile = "unittest/output/FullHomomorphicBinaryApply_DifferentDataSets.out";
IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/FullHomomorphicBinaryApply_DifferentDataSets";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> pt2 = simple.Select(x => x.First());
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Validate.outFileExists(outFile);
result[0] = q1;
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (DryadLinqException)
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool FullHomomorphicBinaryApply_IdenticalDataSets()
public static bool FullHomomorphicBinaryApply_IdenticalDataSets(DryadLinqContext context)
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
string testName = "FullHomomorphicBinaryApply_IdenticalDataSets";
TestLog.TestStart(testName);
bool passed = true;
try
{
string outFile = "unittest/output/FullHomomorphicBinaryApply_2.out";
IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/FullHomomorphicBinaryApply_2";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
passed &= Validate.outFileExists(outFile);
result[0] = q1;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (DryadLinqException)
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
[Associative(typeof(AssociativeRecursive1))]
public static string IntToStringCSVAggregator(string agg, int next)
public static string IntToStringCSVAggregator(string agg, int next)
{
return agg + "," + next.ToString();
}
@ -243,27 +421,29 @@ namespace DryadLinqTests
}
}
public static bool Aggregate_WithCombiner()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
public static bool Aggregate_WithCombiner(DryadLinqContext context)
{
string testName = "Aggregate_WithCombiner";
TestLog.TestStart(testName);
bool passed = true;
try
{
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
}
}
catch (DryadLinqException)
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
}

1532
DryadLinqTests/BasicAPITests.cs
View File

File diff suppressed because it is too large Load Diff

53
DryadLinqTests/DryadLinqFSharpTests/App.config Normal file
View File

@ -0,0 +1,53 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="FSharp.Core" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
<bindingRedirect oldVersion="4.0.0.0" newVersion="4.3.0.0" />
<bindingRedirect oldVersion="2.3.5.0" newVersion="4.3.0.0" />
<bindingRedirect oldVersion="2.0.0.0" newVersion="4.3.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

219
DryadLinqTests/DryadLinqFSharpTests/DryadLinqFSharpTests.fsproj Normal file
View File

@ -0,0 +1,219 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>d0153361-1b5d-4715-8ee3-aa824f89cbc3</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>DryadLinqFSharpTests</RootNamespace>
<AssemblyName>DryadLinqFSharpTests</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<Name>DryadLinqFSharpTests</Name>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<Tailcalls>false</Tailcalls>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<WarningLevel>3</WarningLevel>
<PlatformTarget>AnyCPU</PlatformTarget>
<DocumentationFile>bin\Debug\DryadLinqFSharpTests.XML</DocumentationFile>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<Tailcalls>true</Tailcalls>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<WarningLevel>3</WarningLevel>
<PlatformTarget>AnyCPU</PlatformTarget>
<DocumentationFile>bin\Release\DryadLinqFSharpTests.XML</DocumentationFile>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Data.Edm">
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Data.OData">
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Data.Services.Client">
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Hadoop.Client">
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Msagl">
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Msagl.Drawing">
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Msagl.GraphViewerGdi">
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Research.DryadLinq">
<HintPath>..\packages\Microsoft.Research.Dryad.0.1.4-beta001\lib\net45\Microsoft.Research.DryadLinq.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge">
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils">
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage">
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="mscorlib" />
<Reference Include="FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Private>True</Private>
</Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="System.Data.Linq" />
<Reference Include="System.Drawing" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Numerics" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.ServiceModel" />
<Reference Include="System.ServiceModel.Discovery" />
<Reference Include="System.Spatial">
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.Xml" />
<Reference Include="System.Xml.Linq" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<Compile Include="Program.fs" />
<None Include="packages.config" />
</ItemGroup>
<PropertyGroup>
<MinimumVisualStudioVersion Condition="'$(MinimumVisualStudioVersion)' == ''">11</MinimumVisualStudioVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<Tailcalls>false</Tailcalls>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<WarningLevel>3</WarningLevel>
<DocumentationFile>bin\Debug\DryadLinqFSharpTests.XML</DocumentationFile>
<Prefer32Bit>true</Prefer32Bit>
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<Tailcalls>true</Tailcalls>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<WarningLevel>3</WarningLevel>
<DocumentationFile>bin\Release\DryadLinqFSharpTests.XML</DocumentationFile>
<Prefer32Bit>true</Prefer32Bit>
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<Import Project="$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets" Condition=" Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets" Condition="Exists('..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

46
DryadLinqTests/DryadLinqFSharpTests/Program.fs Normal file
View File

@ -0,0 +1,46 @@
// Learn more about F# at http://fsharp.net
// See the 'F# Tutorial' project for more help.
open System
open System.Collections.Generic
open System.Linq
open System.IO
open Microsoft.Research.DryadLinq
let context = new DryadLinqContext(1)
let inputUri = @"partfile:///d:/DryadLinqTemp/PartFiles/foo.pt"
let input = context.FromStore<LineRecord>(inputUri)
type Pair = { word : string; count : int }
let Histogram (source : IQueryable<LineRecord>) (k : int) =
let words = source.SelectMany(fun (x : LineRecord) -> x.Line.Split(' ') |> Seq.ofArray)
let groups = words.GroupBy(fun x -> x)
let counts = groups.Select(fun (x : IGrouping<string, string>) -> { word = x.Key; count = x.Count() })
let ordered = counts.OrderByDescending(fun x -> x.count)
ordered.Take(k)
let Histogram1 (source : IQueryable<LineRecord>) (k : int) =
query {
for ws in source do
for w in ws.Line.Split(' ') do
groupBy w into g
select { word = g.Key; count = g.Count() } into winfo
sortByDescending winfo.count
take k
}
let Histogram2 =
query {
for ws in input do
for w in ws.Line.Split(' ') do
groupBy w into g
select { word = g.Key; count = g.Count() } into winfo
sortByDescending winfo.count
take 10
}
for x in Histogram1 input 10 do System.Console.WriteLine(x.word + " : " + x.count.ToString())
//let foo = Histogram input 10
//foo.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/xxx.pt").SubmitAndWait()

23
DryadLinqTests/DryadLinqFSharpTests/packages.config Normal file
View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Dryad" version="0.1.4-beta001" targetFramework="net45" />
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages>

178
DryadLinqTests/DryadLinqTests.csproj
View File

@ -1,34 +1,36 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}</ProjectGuid>
<ProjectGuid>{4EA15885-003F-4B4D-A7DB-C172C2F23161}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>DryadLinqTests</RootNamespace>
<AssemblyName>DryadLinqTests</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
<RestorePackages>true</RestorePackages>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
@ -53,91 +55,110 @@
<ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
<HintPath>packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
<HintPath>packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
<HintPath>packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
<HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<Reference Include="Microsoft.Msagl">
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<Reference Include="Microsoft.Msagl.Drawing">
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Msagl.GraphViewerGdi">
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.DryadLinq, Version=0.1.3.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>packages\Microsoft.Research.Dryad.0.1.4-beta001\lib\net45\Microsoft.Research.DryadLinq.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge">
<HintPath>packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils">
<HintPath>packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
<HintPath>packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<HintPath>packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
<HintPath>packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
<HintPath>packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
<HintPath>packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
<HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
<HintPath>packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<HintPath>packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
<HintPath>packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Data.Linq" />
<Reference Include="System.Drawing" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.ServiceModel" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
<HintPath>packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" />
@ -150,53 +171,38 @@
<Compile Include="ApplyAndForkTests.cs" />
<Compile Include="BasicAPITests.cs" />
<Compile Include="GroupByReduceTests.cs" />
<Compile Include="MiscBugFixTests.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="RangePartitionAPICoverageTests.cs" />
<Compile Include="SerializationTests.cs" />
<Compile Include="SerializationTestTypes.cs" />
<Compile Include="SimpleTests.cs" />
<Compile Include="TestLog.cs" />
<Compile Include="TypesInQueryTests.cs" />
<Compile Include="Utils.cs" />
<Compile Include="Validate.cs" />
<Compile Include="WordCount.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\LinqToDryad\LinqToDryad.csproj">
<Project>{d33c34cc-6db2-417c-88b7-299830711774}</Project>
<Name>LinqToDryad</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<BootstrapperPackage Include=".NETFramework,Version=v4.5">
<Visible>False</Visible>
<ProductName>Microsoft .NET Framework 4.5 %28x86 and x64%29</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>false</Install>
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Import Project="packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
<Error Condition="!Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets" Condition="Exists('packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

46
DryadLinqTests/DryadLinqTests.sln Normal file
View File

@ -0,0 +1,46 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2012
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadLinqTests", "DryadLinqTests.csproj", "{4EA15885-003F-4B4D-A7DB-C172C2F23161}"
EndProject
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "DryadLinqFSharpTests", "DryadLinqFSharpTests\DryadLinqFSharpTests.fsproj", "{D0153361-1B5D-4715-8EE3-AA824F89CBC3}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|Mixed Platforms = Debug|Mixed Platforms
Debug|x64 = Debug|x64
Release|Any CPU = Release|Any CPU
Release|Mixed Platforms = Release|Mixed Platforms
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|x64.ActiveCfg = Debug|x64
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|x64.Build.0 = Debug|x64
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Any CPU.Build.0 = Release|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Mixed Platforms.ActiveCfg = Release|x64
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Mixed Platforms.Build.0 = Release|x64
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|x64.ActiveCfg = Release|x64
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|x64.Build.0 = Release|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|x64.ActiveCfg = Debug|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|x64.Build.0 = Debug|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Any CPU.Build.0 = Release|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Mixed Platforms.ActiveCfg = Release|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Mixed Platforms.Build.0 = Release|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|x64.ActiveCfg = Release|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

942
DryadLinqTests/GroupByReduceTests.cs
View File

File diff suppressed because it is too large Load Diff

948
DryadLinqTests/MiscBugFixTests.cs Normal file
View File

@ -0,0 +1,948 @@
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Text.RegularExpressions;
namespace DryadLinqTests
{
public static class MiscBugFixTests
{
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" MiscBugFixTests ");
TestLog.Message(" **********************");
var tests = new Dictionary<string, Action>()
{
{"Bug11447_GroupByWithComparer", () => Bug11447_GroupByWithComparer(context) },
{"Bug12584_HashPartitionOutputCount", () => Bug12584_HashPartitionOutputCount(context) },
{"Bug13108_DisableSequenceEquals", () => Bug13108_DisableSequenceEquals(context) },
{"Bug13529_and_Bug13593_IndexedOperatorCompilation", () => Bug13529_and_Bug13593_IndexedOperatorCompilation(context) },
{"Bug13130_ReverseOperator", () => Bug13130_ReverseOperator(context) },
{"Bug13736_IndexedTakeWhile", () => Bug13736_IndexedTakeWhile(context) },
{"Bug13534_HashPartitionNegIndexIsError", () => Bug13534_HashPartitionNegIndexIsError(context) },
{"Bug13474_and_Bug13483_FromDscOnBadFileSet", () => Bug13474_and_Bug13483_FromDscOnBadFileSet(context) },
{"Bug13637_EmptyFilesInFilesets", () => Bug13637_EmptyFilesInFilesets(context) },
{"Bug13637_LocalDebugProducingZeroRecords", () => Bug13637_LocalDebugProducingZeroRecords(context) },
{"Bug13245_FromDsc_Submit_throws", () => Bug13245_FromDsc_Submit_throws(context) },
{"Bug13245_QueryUsingNonHpcLinqOperator", () => Bug13245_QueryUsingNonHpcLinqOperator(context) },
{"Bug13302_ConfigEnvironmentCleanup", () => Bug13302_ConfigEnvironmentCleanup(context) },
{"Bug13970_MismatchedDataTypes", () => Bug13970_MismatchedDataTypes(context) },
{"Bug14010_AlreadyDisposedContext", () => Bug14010_AlreadyDisposedContext(context) },
{"Bug14256_LeaseOnTempDscFileset", () => Bug14256_LeaseOnTempDscFileset(context) },
{"Bug14189_OrderPreservation", () => Bug14189_OrderPreservation(context) },
{"Bug14190_MergeJoin_DecreasingOrder", () => Bug14190_MergeJoin_DecreasingOrder(context) },
{"Bug14192_MultiApplySubExpressionReuse", () => Bug14192_MultiApplySubExpressionReuse(context) },
{"Bug14870_LongIndexTakeWhile", () => Bug14870_LongIndexTakeWhile(context) },
{"Bug15159_NotOperatorForNullableBool", () => Bug15159_NotOperatorForNullableBool(context) },
{"Bug15371_NoDataMembersForSerialization", () => Bug15371_NoDataMembersForSerialization(context) },
{"Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes", () => Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static bool Bug11447_GroupByWithComparer(DryadLinqContext context)
{
string testName = "Bug11447_GroupByWithComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
result[0] = a;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
result[1] = a;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug12584_HashPartitionOutputCount(DryadLinqContext context) // ToDo
{
string testName = "Bug12584_HashPartitionOutputCount";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug12584_HashPartitionOutputCount";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.HashPartition(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
info.Wait();
//// partition verification are only valid for cluster execution..
//// check that nOutputPartitions == nInputPartitions.
//// Note: this is today's behavior, but we don't strictly guarantee this and the rules may change.
//var inFS = context.DscService.GetFileSet(DataGenerators.SIMPLE_FILESET_NAME);
//var inPCount = inFS.GetFiles().Count();
//var outFS = context.DscService.GetFileSet("DevUnitTest/Bug12584_out");
//var outPCount = outFS.GetFiles().Count();
//passed &= TestUtils.Assert(outPCount == inPCount, "Output nPartitions should be equal to input nPartitions.");
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13108_DisableSequenceEquals(DryadLinqContext context)
{
string testName = "Bug13108_DisableSequenceEquals";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
bool result = pt1.Select(x => x).SequenceEqual(pt2.Select(x => x));
passed &= false; // NotSupportedException should have been thrown
}
}
catch (Exception Ex)
{
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13529_and_Bug13593_IndexedOperatorCompilation(DryadLinqContext context)
{
string testName = "Bug13529_and_Bug13593_IndexedOperatorCompilation";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
result[0] = pt1.Select((x, i) => x)
.LongSelect((x, i) => x)
.LongSelectMany((x, i) => new[] { x })
.SelectMany((x, i) => new[] { x })
.Where((x, i) => true)
.LongWhere((x, i) => true)
.TakeWhile((x, i) => true)
.LongTakeWhile((x, i) => true)
.SkipWhile((x, i) => false)
.LongSkipWhile((x, i) => false)
.ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
result[1] = pt1.Select((x, i) => x)
.LongSelect((x, i) => x)
.LongSelectMany((x, i) => new[] { x })
.SelectMany((x, i) => new[] { x })
.Where((x, i) => true)
.LongWhere((x, i) => true)
.TakeWhile((x, i) => true)
.LongTakeWhile((x, i) => true)
.SkipWhile((x, i) => false)
.LongSkipWhile((x, i) => false)
.ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13130_ReverseOperator(DryadLinqContext context)
{
string testName = "Bug13130_ReverseOperator";
TestLog.TestStart(testName);
bool passed = true;
//data set #1
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
result[0] = pt1.Reverse().ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
result[1] = pt1.Reverse().ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
//data set #2 ToDo
//data set #3 ToDo
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13736_IndexedTakeWhile(DryadLinqContext context)
{
string testName = "Bug13736_IndexedTakeWhile";
TestLog.TestStart(testName);
bool passed = true;
// dataset #1
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var pt2 = pt1.TakeWhile((x, i) => i < 6).ToArray();
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 6).ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var pt2 = pt1.TakeWhile((x, i) => i < 6).ToArray();
result[1] = pt1.AsEnumerable().TakeWhile((x, i) => i < 6).ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
// dataset #2
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var pt2 = pt1.TakeWhile((x, i) => i < 125).ToArray();
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 125).ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetGroupByReduceDataSet(context);
var pt2 = pt1.TakeWhile((x, i) => i < 125).ToArray();
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 125).ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception)
{
passed &= false;
}
}
catch (Exception)
{
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13534_HashPartitionNegIndexIsError(DryadLinqContext context)
{
string testName = "Bug13534_HashPartitionNegIndexIsError";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
pt1.HashPartition(x => x, 0); // exception should be thrown
passed &= false;
}
}
catch (Exception)
{
}
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
pt1.HashPartition(x => x, -1); // exception should be thrown
passed &= false;
}
}
catch (Exception)
{
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13474_and_Bug13483_FromDscOnBadFileSet(DryadLinqContext context)
{
// ToDo
return false;
}
public static bool Bug13637_EmptyFilesInFilesets(DryadLinqContext context)
{
// ToDo
return false;
}
public static bool Bug13637_LocalDebugProducingZeroRecords(DryadLinqContext context)
{
string testName = "Bug13637_LocalDebugProducingZeroRecords";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug13637_LocalDebugProducingZeroRecords";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt2 = pt1.Where(x => false).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var queryInfo2 = pt2.Submit();
queryInfo2.Wait();
int[] data2 = pt2.ToArray();
passed &= (data2.Length == 0);
// query producing no records -> anonymous output
IQueryable<int> pt3 = pt1.Where(x => false);
var queryInfo3 = pt3.Submit();
queryInfo3.Wait();
int[] data3 = pt3.ToArray();
passed &= (data3.Length == 0);
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13245_FromDsc_Submit_throws(DryadLinqContext context)
{
string testName = "Bug13245_FromDsc_Submit_throws";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.Submit();
passed &= false; // should throw ArgEx as the input isn't well formed
}
}
catch (ArgumentException)
{
}
catch (Exception)
{
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static IQueryable<int> Blah(this IQueryable<int> source)
{
return Enumerable.Range(1, 1).AsQueryable();
}
public static bool Bug13245_QueryUsingNonHpcLinqOperator(DryadLinqContext context)
{
string testName = "Bug13245_QueryUsingNonHpcLinqOperator";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.Blah().Submit();
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static IEnumerable<int> CheckEnvVar(IEnumerable<int> en)
{
if (Environment.GetEnvironmentVariable("DummyEnvVar") == null)
throw new Exception("the expected environment variable is not defined");
return en;
}
public static bool Bug13302_ConfigEnvironmentCleanup(DryadLinqContext context)
{
string testName = "Bug13302_ConfigEnvironmentCleanup";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.Apply((en) => CheckEnvVar(en)).Submit();
info.Wait();
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13970_MismatchedDataTypes(DryadLinqContext context)
{
string testName = "Bug13970_MismatchedDataTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetGroupByReduceDataSet(context);
double? output = pt1.AverageAsQuery().Single();
passed &= false;
}
}
catch (Exception)
{
passed &= true; // expected
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14010_AlreadyDisposedContext(DryadLinqContext context)
{
string testName = "Bug14010_AlreadyDisposedContext";
TestLog.TestStart(testName);
bool passed = true;
context.LocalDebug = false;
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
ctx.Dispose();
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
int output = pt1.Select(x => x).First();
passed &= false;
}
catch (Exception)
{
passed &= true;
}
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
ctx.Dispose();
int output = pt1.Select(x => x).First();
passed &= false;
}
catch (Exception)
{
passed &= true;
}
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
ctx.Dispose();
IQueryable<int> query = pt1.Select(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, "abc"), true);
var info = DryadLinqQueryable.Submit(query);
passed &= false;
}
catch (Exception)
{
passed &= true;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14256_LeaseOnTempDscFileset(DryadLinqContext context) // TODO
{
string testName = "Bug14256_LeaseOnTempDscFileset";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14189_OrderPreservation(DryadLinqContext context)
{
string testName = "Bug14189_OrderPreservation";
TestLog.TestStart(testName);
bool passed = true;
try
{
IGrouping<int, int>[] clusterSorted, localSorted;
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
var output = pt1.OrderBy(x => x % 4)
.Select(x => x).Select(x => x).Where(x => true) // this pipeline was not preserving order correctly.
.GroupBy(x => x % 4)
.ToArray();
passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
// sort back on the key for deterministic output.
clusterSorted = output.OrderBy(x => x.Key).ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
var output = pt1.OrderBy(x => x % 4)
.Select(x => x).Select(x => x).Where(x => true) // this pipeline was not preserving order correctly.
.GroupBy(x => x % 4)
.ToArray();
passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
// sort back on the key for deterministic output.
localSorted = output.OrderBy(x => x.Key).ToArray();
}
// check that each group of output has the same elements as the LINQ groups.
for (int i = 0; i < 4; i++)
{
var a = clusterSorted[i].OrderBy(x => x);
var b = localSorted[i].OrderBy(x => x);
passed &= a.SequenceEqual(b); //the output should match linq. Error for group: + i);
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14190_MergeJoin_DecreasingOrder(DryadLinqContext context) // TODO
{
string testName = "Bug14190_MergeJoin_DecreasingOrder";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug14190_MergeJoin_DecreasingOrder";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14192_MultiApplySubExpressionReuse(DryadLinqContext context)
{
string testName = "Bug14192_MultiApplySubExpressionReuse";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug14192_MultiApplySubExpressionReuse";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var data = pt1.Select(x => x);
var info = pt1.Apply(new[] { pt1, pt1 }, (sources) => new int[] { 1, 2, 3 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
info.Wait();
// ToDo
//string queryPlan = TestUtils.GetRecentQueryXmlAsText();
//int nVerticesInPlan = TestUtils.GetVertexStageCount(queryPlan);
//passed &= (nVerticesInPlan == 7); // "Only seven vertices should appear (before bug, there were 10 of which the last three were extraneous.");
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14870_LongIndexTakeWhile(DryadLinqContext context)
{
string testName = "Bug14870_LongIndexTakeWhile";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
// ToDo - move to data generator
int[][] data = new[] {
new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { }, new int[] { } };
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.LongTakeWhile((x, i) => i < 8).ToArray();
passed &= (output.Length == 8); // "Only eight items should be returned."
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug15159_NotOperatorForNullableBool(DryadLinqContext context)
{
string testName = "Bug15159_NotOperatorForNullableBool";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<bool?>[] result = new IEnumerable<bool?>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
result[0] = output.ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
result[1] = output.ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public class NoDataMembersClass
{
}
public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
{
string testName = "Bug15371_NoDataMembersForSerialization";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var result = context.FromStore<NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
passed &= false;
}
}
catch (DryadLinqException Ex)
{
passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(DryadLinqContext context)
{
string testName = "Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
// use an anonymous type with nullable fields, and make sure we at least have some null values in the stream
var output = pt1.Select(x => new { FirstField = (x % 2 == 0) ? new int?(x) : default(int?), SecondField = x.ToString() })
.GroupBy(x => x.FirstField, y => y.SecondField); // use of GB ensures we exercise the emitted GetHashCode() overload
passed &= (output.Count() != 0); // "Query return 0 length output"
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
}
}

57
DryadLinqTests/Program.cs
View File

@ -17,14 +17,15 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Threading.Tasks;
using System.Reflection;
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
namespace DryadLinqTests
@ -33,32 +34,38 @@ namespace DryadLinqTests
{
public static void Main(string[] args)
{
//BasicAPITests.ToStoreThrowsForNonQuery();
//BasicAPITests.ToStoreGetEnumeratorThrows();
//BasicAPITests.GetEnumeratorNonToStoreTerminated();
//BasicAPITests.ToStoreSubmitGetEnumerator();
//BasicAPITests.SubmitNonToStoreTerminated();
//BasicAPITests.MaterializeToStoreTerminated();
//BasicAPITests.MaterializeNonToStoreTerminated();
//BasicAPITests.EnumeratePlainData();
SimpleTests.Run(args);
//BasicAPITests.RepeatSubmit();
//BasicAPITests.RepeatMaterialize();
//BasicAPITests.MaterializeMentionsSameQueryTwice();
//BasicAPITests.QueryOnDataBackedDLQ();
//BasicAPITests.Bug11781_CountandFirstOrDefault();
//BasicAPITests.Bug11782_Aggregate();
//BasicAPITests.Bug11782_LowLevelQueryableManipulation();
//BasicAPITests.Bug11638_LongWhere();
//BasicAPITests.AssumeRangePartition();
//BasicAPITests.Bug11638_LongMethods();
//BasicAPITests.ContextConfigIsReadOnly();
//BasicAPITests.ToggleSpeculativeDuplication();
//BasicAPITests.Bug15068_ConfigResourcesAPI();
//BasicAPITests.Bug_16341_SubmitThrowsForDifferentContexts();
//BasicAPITests.Bug14449_ContextShouldExposeVersionIDs();
//BasicAPITests.Bug_16341_VariousTestsForSubmit();
/*
*
Config conf = new Config("dryad-temp", "main1", @"d:\temp\TestLog\");
DryadLinqContext context = new DryadLinqContext(Config.cluster);
string matchPattern = @"";
TestLog.LogInit(Config.testLogPath + "BasicAPITests.txt");
BasicAPITests.Run(context, matchPattern);
TestLog.LogInit(Config.testLogPath + "ApplyAndForkTests.txt");
ApplyAndForkTests.Run(context, matchPattern);
TestLog.LogInit(Config.testLogPath + "GroupByReduceTests.txt");
GroupByReduceTests.Run(context, matchPattern);
TestLog.LogInit(Config.testLogPath + "RangePartitionAPICoverageTests.txt");
RangePartitionAPICoverageTests.Run(context, matchPattern);
TestLog.LogInit(Config.testLogPath + "TypesInQueryTests.txt");
TypesInQueryTests.Run(context, matchPattern);
TestLog.LogInit(Config.testLogPath + "SerializationTests.txt");
SerializationTests.Run(context, matchPattern);
TestLog.LogInit(Config.testLogPath + "MiscBugFixTests.txt");
MiscBugFixTests.Run(context, matchPattern);
*/
}
}
}

27
DryadLinqTests/Properties/AssemblyInfo.cs
View File

@ -1,22 +1,3 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
@ -28,7 +9,7 @@ using System.Runtime.InteropServices;
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("DryadLinqTests")]
[assembly: AssemblyProduct("Dryad")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -39,7 +20,7 @@ using System.Runtime.InteropServices;
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("1fce146c-8597-4cc2-92f6-ebb655cd99ed")]
[assembly: Guid("49b725be-1187-4056-9890-7a0cb4c064a4")]
// Version information for an assembly consists of the following four values:
//
@ -51,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.3.0")]
[assembly: AssemblyFileVersion("0.1.3.0")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

817
DryadLinqTests/RangePartitionAPICoverageTests.cs
View File

@ -1,12 +1,823 @@
using System;
using Microsoft.Research.DryadLinq;
using Microsoft.Research.DryadLinq.Internal;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Linq.Expressions;
using System.Text.RegularExpressions;
namespace DryadLinqTests
{
public class RangePartitionAPICoverageTests
{
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" RangePartitionAPICoverageTests ");
TestLog.Message(" **********************");
var tests = new Dictionary<string, Action>()
{
{"RP_keySelector", () => RP_keySelector(context) },
{"RP_keySelector_pcount", () => RP_keySelector_pcount(context) },
{"RP_keySelector_isDescending", () => RP_keySelector_isDescending(context) },
{"RP_keySelector_rangeKeys", () => RP_keySelector_rangeKeys(context) },
{"RP_keySelector_isDescending_pcount", () => RP_keySelector_isDescending_pcount(context) },
{"RP_keySelector_keyComparer_isDescending", () => RP_keySelector_keyComparer_isDescending(context) },
{"RP_keySelector_rangeKeys_keyComparer", () => RP_keySelector_rangeKeys_keyComparer(context) },
{"RP_keySelector_keyComparer_isDescending_pcount", () => RP_keySelector_keyComparer_isDescending_pcount(context) },
{"RP_keySelector_rangeKeys_keyComparer_isDescending", () => RP_keySelector_rangeKeys_keyComparer_isDescending(context) },
{"RP_singlePartition_autoSeparators", () => RP_singlePartition_autoSeparators(context) },
{"RP_rangeSeparators_customComparer", () => RP_rangeSeparators_customComparer(context) },
{"RP_rangeSeparators_nullCustomComparer", () => RP_rangeSeparators_nullCustomComparer(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static IEnumerable<int> CountPartitions<T>(IEnumerable<T> pt)
{
yield return 1;
}
public static IEnumerable<T> MaxInPartition<T>(IEnumerable<T> pt)
{
yield return pt.Max();
}
public static IEnumerable<T> MinInPartition<T>(IEnumerable<T> pt)
{
yield return pt.Min();
}
public static IEnumerable<T2> MaxCompare<T1, T2>(IEnumerable<T1> pt, int i, IEnumerable<T2> maxes)
{
if (0 == i)
{
yield return (T2)Convert.ChangeType(1, typeof(int));
}
int priorMax = (int)Convert.ChangeType(maxes.ToArray()[i - 1], typeof(T2));
foreach (T1 item in pt)
{
int val = (int)Convert.ChangeType(item, typeof(T1));
if (val <= priorMax)
{
yield return (T2)Convert.ChangeType(1, typeof(int));
}
}
}
public static bool RP_keySelector(DryadLinqContext context)
{
string testName = "RP_keySelector";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 3, false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_pcount(DryadLinqContext context)
{
string testName = "RP_keySelector_pcount";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_pcount";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, 2).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 2, false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, 2);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_isDescending(DryadLinqContext context)
{
string testName = "RP_keySelector_isDescending";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_isDescending";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 3, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, true);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_rangeKeys(DryadLinqContext context)
{
string testName = "RP_keySelector_rangeKeys";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
IEnumerable<int>[] result2 = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_rangeKeys_1";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//increasing ranges keys.
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 4, false);
result[0] = pt2.ToList();
string outFile2 = "unittest/output/RP_keySelector_rangeKeys_2";
IQueryable<int> pt1_2 = DataGenerator.GetRangePartitionDataSet(context);
//decreasing ranges keys.
IQueryable<int> pt2_2 = pt1_2.RangePartition(x => x, new[] { 8, 5, 2 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile2), true);
var jobInfo2 = pt2_2.Submit();
jobInfo2.Wait();
passed &= TestRangePartitioned(pt2_2, 4, true);
result2[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//increasing ranges keys.
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 });
result[1] = pt2.ToList();
IQueryable<int> pt1_2 = DataGenerator.GetRangePartitionDataSet(context);
//decreasing ranges keys.
IQueryable<int> pt2_2 = pt1_2.RangePartition(x => x, new[] { 8, 5, 2 });
result2[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
Validate.Check(result2);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_isDescending_pcount(DryadLinqContext context)
{
string testName = "RP_keySelector_isDescending_pcount";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_isDescending_pcount";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, true, 2).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 2, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, true, 2);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_keyComparer_isDescending(DryadLinqContext context)
{
string testName = "RP_keySelector_keyComparer_isDescending";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_keyComparer_isDescending";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 3, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_rangeKeys_keyComparer(DryadLinqContext context)
{
string testName = "RP_keySelector_rangeKeys_keyComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_rangeKeys_keyComparer";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }, Comparer<int>.Default)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 4, false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }, Comparer<int>.Default);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_keyComparer_isDescending_pcount(DryadLinqContext context)
{
string testName = "RP_keySelector_keyComparer_isDescending_pcount";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_keyComparer_isDescending_pcount";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true, 2)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 2, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true, 2);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
return passed;
}
public static bool RP_keySelector_rangeKeys_keyComparer_isDescending(DryadLinqContext context)
{
string testName = "RP_keySelector_rangeKeys_keyComparer_isDescending";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_rangeKeys_keyComparer_isDescending";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer<int>.Default, true)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 4, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer<int>.Default, true);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_singlePartition_autoSeparators(DryadLinqContext context)
{
string testName = "RP_singlePartition_autoSeparators";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_singlePartition_autoSeparators";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.Apply(x => x) // force a merge
.RangePartition(x => x)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 1, false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.Apply(x => x) // force a merge
.RangePartition(x => x);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_rangeSeparators_customComparer(DryadLinqContext context)
{
string testName = "RP_rangeSeparators_customComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_rangeSeparators_customComparer";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//these keys should be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should also be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 4, 3, 2, 1 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should work
IQueryable<int> pt2 = pt1.RangePartition(x => x, new int[] { 6, 6, 3, 1 }, new WeirdIntComparer(), false)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 5, new WeirdIntComparer(), false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
string outFile = "unittest/output/RP_rangeSeparators_customComparer";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//these keys should be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should also be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 4, 3, 2, 1 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should work
IQueryable<int> pt2 = pt1.RangePartition(x => x, new int[] { 6, 6, 3, 1 }, new WeirdIntComparer(), false)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool TestRangePartitioned(IQueryable<int> pt, int expectedPCount, bool expectedIsDescending)
{
bool passed = true;
try
{
IEnumerable<int> ptCount = pt.ApplyPerPartition(x => CountPartitions(x));
int cPartitions = ptCount.Count();
passed &= (expectedPCount == cPartitions);
if (passed)
{
int[] ptMax = pt.ApplyPerPartition(x => MaxInPartition(x)).ToArray();
int[] ptMin = pt.ApplyPerPartition(x => MinInPartition(x)).ToArray();
// compare mins and maxs of each partition
for (int i = 1; i < cPartitions; i++)
{
if (false == expectedIsDescending)
passed &= (ptMin[i] > ptMax[i - 1]);
else
passed &= (ptMax[i] < ptMin[i - 1]);
}
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
return passed;
}
[Serializable]
public class WeirdIntComparer : IComparer<int>
{
public int Compare(int x, int y)
{
int xx = x % 2;
int yy = y % 2;
return xx.CompareTo(yy);
}
}
public static bool RP_rangeSeparators_nullCustomComparer(DryadLinqContext context)
{
string testName = "RP_rangeSeparators_nullCustomComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
var results1 = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null, false).ToArray();
var results2 = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null).ToArray();
//passing is not throwing.
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool TestRangePartitioned(IQueryable<int> pt, int expectedPCount, IComparer<int> comparer, bool expectedIsDescending)
{
bool passed = true;
try
{
IEnumerable<int> ptCount = pt.ApplyPerPartition(x => CountPartitions(x));
int cPartitions = ptCount.Count();
passed &= (expectedPCount == cPartitions);
if (passed)
{
int[] ptMax = pt.ApplyPerPartition(x => MaxInPartition(x)).ToArray();
int[] ptMin = pt.ApplyPerPartition(x => MinInPartition(x)).ToArray();
// compare mins and maxs of each partition
for (int i = 1; i < cPartitions; i++)
{
passed &= (comparer.Compare(ptMin[i], ptMax[i - 1]) == 0); // ToDo
}
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
return passed;
}
}
}

576
DryadLinqTests/SerializationTestTypes.cs Normal file
View File

@ -0,0 +1,576 @@
using Microsoft.Research.DryadLinq;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace DryadLinqTests
{
//////////////////////////////////////////////////////////////////////
//
// Decorated and undecorated types
//
// A UDT with no attribute shouldn't autoserialize without an explicit [Serializable] attribute
public class UDT_Undecorated
{
public UDT_Undecorated(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
}
public int m_field1;
public int m_field2;
}
// UDT which is marked as serializable. This should be autoserialized
[Serializable]
public class UDT_MarkedSerializable
{
public UDT_MarkedSerializable(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
}
public int m_field1;
public int m_field2;
}
// UDT that has the old style static serialization methods. This should be rejected.
public class UDT_StaticSerializer
{
public UDT_StaticSerializer(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
}
public int m_field1;
public int m_field2;
public static UDT_StaticSerializer Read(DryadLinqBinaryReader reader)
{
var val = new UDT_StaticSerializer(0);
val.m_field1 = reader.ReadInt32();
val.m_field2 = reader.ReadInt32();
return val;
}
public static void Write(DryadLinqBinaryWriter writer, UDT_StaticSerializer val)
{
writer.Write(val.m_field1);
writer.Write(val.m_field2);
}
}
//////////////////////////////////////////////////////////////////////
//
// UDTs with custom serializers
//
// UDT that has an attribute declaring itself as its custom serializer
[CustomDryadLinqSerializer(typeof(UDT_SelfCustomSerializer))]
public struct UDT_SelfCustomSerializer : IDryadLinqSerializer<UDT_SelfCustomSerializer>
{
public UDT_SelfCustomSerializer(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
}
public int m_field1;
public int m_field2;
public UDT_SelfCustomSerializer Read(DryadLinqBinaryReader reader)
{
var val = new UDT_SelfCustomSerializer(0);
val.m_field1 = reader.ReadInt32();
val.m_field2 = reader.ReadInt32();
return val;
}
public void Write(DryadLinqBinaryWriter writer, UDT_SelfCustomSerializer val)
{
writer.Write(val.m_field1);
writer.Write(val.m_field2);
}
}
// UDT that declares another type as its custom serializer
[CustomDryadLinqSerializer(typeof(CustomUDTSerializer))]
public struct UDT_ExternalCustomSerializer
{
public UDT_ExternalCustomSerializer(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
}
public int m_field1;
public int m_field2;
}
// this is the custom serializer for UDT_ExternalCustomSerializer
public class CustomUDTSerializer : IDryadLinqSerializer<UDT_ExternalCustomSerializer>
{
public UDT_ExternalCustomSerializer Read(DryadLinqBinaryReader reader)
{
var val = new UDT_ExternalCustomSerializer(0);
val.m_field1 = reader.ReadInt32();
val.m_field2 = reader.ReadInt32();
return val;
}
public void Write(DryadLinqBinaryWriter writer, UDT_ExternalCustomSerializer val)
{
writer.Write(val.m_field1);
writer.Write(val.m_field2);
}
}
// UDT with a CustomHpcSerializer attribute that points to an invalid type for the serializer
[CustomDryadLinqSerializer(typeof(int))]
public struct UDT_BadCustomSerializerType1
{
public UDT_BadCustomSerializerType1(int val)
{
m_field1 = val + 1;
}
public int m_field1;
}
// UDT with a CustomHpcSerializer attribute that points to a serializer type that targets a different UDT
[CustomDryadLinqSerializer(typeof(CustomUDTSerializer))]
public struct UDT_BadCustomSerializerType2
{
public UDT_BadCustomSerializerType2(int val)
{
m_field1 = val + 1;
}
public int m_field1;
}
//////////////////////////////////////////////////////////////////////
//
// Inheritance
//
// A UDT that has sub types. This should be rejected
[Serializable]
public class UDT_BaseType
{
public UDT_BaseType(int val)
{
m_baseTypeField = val + 42;
}
public int m_baseTypeField;
}
// A UDT that has derives from a type other than object. This should be rejected
[Serializable]
public class UDT_DerivedType : UDT_BaseType
{
public UDT_DerivedType(int val)
: base(val)
{
m_derivedTypeField = val + 84;
}
public int m_derivedTypeField;
}
//////////////////////////////////////////////////////////////////////
//
// Field types
//
// A UDT with no data. This should be rejected
[Serializable]
public class UDT_EmptyType
{
public UDT_EmptyType(int val)
{
}
}
// A UDT with a public field of type System.Object. This should be rejected
[Serializable]
public class UDT_ObjectField
{
public UDT_ObjectField(int val)
{
m_intField = val + 84;
m_objectField = null;
}
public int m_intField;
public object m_objectField;
}
// A UDT with a public field of type System.Object[]. This should be rejected
[Serializable]
public class UDT_ObjectArrayField
{
public UDT_ObjectArrayField(int val)
{
m_intField = val + 84;
m_objectArrayField = new object[1];
m_objectArrayField[0] = null;
}
public int m_intField;
public object[] m_objectArrayField;
}
// A UDT with a public field of type List<System.Object>. This should be rejected
[Serializable]
public class UDT_ObjectListField
{
public UDT_ObjectListField(int val)
{
m_intField = val + 84;
m_objectListField = new List<object>();
m_objectListField.Add(null);
}
public int m_intField;
public List<object> m_objectListField;
}
// A UDT with an object field, and a custom serializer. This should not be rejected
[CustomDryadLinqSerializer(typeof(UDT_ObjectFieldAndCustomSerializer))]
public class UDT_ObjectFieldAndCustomSerializer : IDryadLinqSerializer<UDT_ObjectFieldAndCustomSerializer>
{
private object m_objectRecord = (object)"";
// Required by CustomHpcSerializer
public UDT_ObjectFieldAndCustomSerializer() { }
public UDT_ObjectFieldAndCustomSerializer(int val)
{
m_objectRecord = (object)String.Format("{0}", val);
}
#region IHpcSerializer implementation
public UDT_ObjectFieldAndCustomSerializer Read(DryadLinqBinaryReader reader)
{
string tmp = reader.ReadString();
int val = Int32.Parse(tmp);
return new UDT_ObjectFieldAndCustomSerializer(val);
}
public void Write(DryadLinqBinaryWriter writer, UDT_ObjectFieldAndCustomSerializer record)
{
writer.Write((string)record.m_objectRecord);
}
#endregion
}
// An empty UDT with a custom serializer. This should be rejected, because even though the user has control over serialization
// we will encounter runtime problems if the custom serializer reads/writer 0 bytes. This behavior is simply to discourage empty CS code.
[CustomDryadLinqSerializer(typeof(UDT_EmptyTypeWithCustomSerializer))]
public class UDT_EmptyTypeWithCustomSerializer : IDryadLinqSerializer<UDT_EmptyTypeWithCustomSerializer>
{
// Required by CustomHpcSerializer
public UDT_EmptyTypeWithCustomSerializer() { }
public UDT_EmptyTypeWithCustomSerializer(int val)
{
}
#region IHpcSerializer implementation
public UDT_EmptyTypeWithCustomSerializer Read(DryadLinqBinaryReader reader)
{
return new UDT_EmptyTypeWithCustomSerializer(0);
}
public void Write(DryadLinqBinaryWriter writer, UDT_EmptyTypeWithCustomSerializer record)
{
}
#endregion
}
//////////////////////////////////////////////////////////////////////
//
// Visibility
//
// UDT with a field of non-public type. We cannot handle these
[Serializable]
public class UDT_FieldOfNonPublicType
{
private enum SecretCodeLevel
{
Secret,
SuperSecret,
}
public UDT_FieldOfNonPublicType(int val)
{
m_field = (SecretCodeLevel)val;
}
private SecretCodeLevel m_field;
}
// UDT with a private field of a public type. We do handle these using emitted IL code.
[Serializable]
public class UDT_PrivateFieldOfPublicType
{
public UDT_PrivateFieldOfPublicType(int val)
{
m_field = val + 1;
}
private int m_field;
}
//////////////////////////////////////////////////////////////////////
//
// Nesting
//
[Serializable]
public class UDT_Nested_InnerAndOuterSerializable
{
public UDT_Nested_InnerAndOuterSerializable(int val)
{
m_field = val + 1;
m_field2 = new NestedSerializable(val);
}
private int m_field;
private NestedSerializable m_field2;
[Serializable]
public class NestedSerializable
{
public NestedSerializable(int val)
{
m_field = val / 2.0;
}
private double m_field;
}
}
[Serializable]
public class UDT_Nested_InnerEnum_InnerAndOuterSerializable
{
public UDT_Nested_InnerEnum_InnerAndOuterSerializable(int val)
{
m_field = val + 1;
m_field2 = (NestedEnum)(val % 3);
}
private int m_field;
private NestedEnum m_field2;
//[Serializable]
public enum NestedEnum
{
Foo = 0,
Bar = 1,
Baz = 2,
}
}
[Serializable]
public class UDT_Nested_OuterSerializableInnerNotSerializable
{
public UDT_Nested_OuterSerializableInnerNotSerializable(int val)
{
m_field = val + 1;
m_field2 = new NestedNotSerializable(val);
}
private int m_field;
private NestedNotSerializable m_field2;
public class NestedNotSerializable
{
public NestedNotSerializable(int val)
{
m_field = val / 2.0;
}
private double m_field;
}
}
public class UDT_Nested_OuterNotSerializableInnerSerializable
{
public UDT_Nested_OuterNotSerializableInnerSerializable(int val)
{
m_field = val + 1;
m_field2 = new NestedSerializable(val);
}
private int m_field;
private NestedSerializable m_field2;
[Serializable]
public class NestedSerializable
{
public NestedSerializable(int val)
{
m_field = val / 2.0;
}
private double m_field;
}
}
//////////////////////////////////////////////////////////////////////
//
// Self reference
//
// First level circular type
[Serializable]
public class UDT_FirstLevelCircular
{
public UDT_FirstLevelCircular(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
m_circularRef = null;
}
public int m_field1;
public int m_field2;
public UDT_FirstLevelCircular m_circularRef;
}
// First level circular type with an array reference to self
[Serializable]
public class UDT_FirstLevelCircularArrayRef
{
public UDT_FirstLevelCircularArrayRef(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
m_circularRefArray = new UDT_FirstLevelCircular[5];
}
public int m_field1;
public int m_field2;
public UDT_FirstLevelCircular[] m_circularRefArray;
}
// Second level circular type
[Serializable]
public class UDT_SecondLevelCircular
{
public UDT_SecondLevelCircular(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
m_child = new UDT_CircularRefChild(this);
}
public int m_field1;
public int m_field2;
public UDT_CircularRefChild m_child;
}
[Serializable]
public class UDT_CircularRefChild
{
public UDT_CircularRefChild(UDT_SecondLevelCircular parent)
{
m_parent = parent;
}
public UDT_SecondLevelCircular m_parent;
}
// Circular type with custom serializer. Should not be rejected
[CustomDryadLinqSerializer(typeof(UDT_CircularTypeWithCustomSerializer))]
public class UDT_CircularTypeWithCustomSerializer : IDryadLinqSerializer<UDT_CircularTypeWithCustomSerializer>
{
public UDT_CircularTypeWithCustomSerializer() { }
public UDT_CircularTypeWithCustomSerializer(int val)
{
m_field1 = val;
// create each new object with log2(val) self references hanging off of m_next
if (val == 0)
m_next = null;
else
m_next = new UDT_CircularTypeWithCustomSerializer(val / 2);
}
public int m_field1;
public UDT_CircularTypeWithCustomSerializer m_next;
//
// sample recursive custom serializer
//
public UDT_CircularTypeWithCustomSerializer Read(DryadLinqBinaryReader reader)
{
UDT_CircularTypeWithCustomSerializer obj = new UDT_CircularTypeWithCustomSerializer();
bool bHasValidNext = reader.ReadBool();
obj.m_field1 = reader.ReadInt32();
if (bHasValidNext)
{
obj.m_next = this.Read(reader); // recursively read the next
}
else
{
obj.m_next = null; // terminate recursion
}
return obj;
}
public void Write(DryadLinqBinaryWriter writer, UDT_CircularTypeWithCustomSerializer x)
{
if (x.m_next != null)
{
writer.Write(true); // bHasValidNext for the reader side
writer.Write(x.m_field1);
this.Write(writer, x.m_next); // write out recursively
}
else
{
writer.Write(false); // bHasValidNext = false for the reader side, makes sure recursive reads stop
writer.Write(x.m_field1);
}
}
}
// This type itself isn't circular, but contains a field of a circular type that is custom serialized. Autoserialization should work for this guy
[Serializable]
public class UDT_TypeContainingCustomSerializedCircularType
{
public UDT_TypeContainingCustomSerializedCircularType(int val)
{
m_field1 = val;
m_circularType = new UDT_CircularTypeWithCustomSerializer(val);
}
public int m_field1;
public UDT_CircularTypeWithCustomSerializer m_circularType;
}
}

361
DryadLinqTests/SerializationTests.cs Normal file
View File

@ -0,0 +1,361 @@
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Text.RegularExpressions;
namespace DryadLinqTests
{
public class SerializationTests
{
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" SerializationTests ");
TestLog.Message(" **********************");
context.LocalDebug = false;
IQueryable<int> source = DataGenerator.GetSimpleFileSets(context);
var tests = new Dictionary<string, Action>()
{
{"UDT_Undecorated", () =>
SerializationTests.TestUDT<UDT_Undecorated>(context, source, "Basic UDT with no attribute. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
{"UDT_MarkedSerializable", () =>
SerializationTests.TestUDT<UDT_MarkedSerializable>(context, source, "Basic UDT which is marked as serializable.", null, 0)},
{"UDT_StaticSerializer", () =>
SerializationTests.TestUDT<UDT_StaticSerializer>(context, source, "Basic UDT with old stype static serializers. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
{"UDT_SelfCustomSerializer", () =>
SerializationTests.TestUDT<UDT_SelfCustomSerializer>(context, source, "UDT implementing custom serializer for itself.", null, 0)},
{"UDT_ExternalCustomSerializer", () =>
SerializationTests.TestUDT<UDT_ExternalCustomSerializer>(context, source, "UDT with a custom serializer", null, 0)},
{"UDT_BadCustomSerializerType1", () =>
SerializationTests.TestUDT<UDT_BadCustomSerializerType1>(context, source, "UDT decorated with an invalid custom serializer type.", typeof(DryadLinqException), 0)},
{"UDT_BadCustomSerializerType2", () =>
SerializationTests.TestUDT<UDT_BadCustomSerializerType2>(context, source, "UDT decorated with another UDT's custom serializer.", typeof(DryadLinqException), 0)},
{"UDT_BaseType", () =>
SerializationTests.TestUDT<UDT_BaseType>(context, source, "UDT with subtypes. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleSubtypes"))},
{"UDT_DerivedType", () =>
SerializationTests.TestUDT<UDT_DerivedType>(context, source, "UDT with a base type. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleDerivedtypes"))},
{"UDT_FieldOfNonPublicType", () =>
SerializationTests.TestUDT<UDT_FieldOfNonPublicType>(context, source, "UDT with a field of a non-public type. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("UDTHasFieldOfNonPublicType"))},
{"UDT_Nested_OuterSerializableInnerNotSerializable", () =>
SerializationTests.TestUDT<UDT_Nested_OuterSerializableInnerNotSerializable>(context, source, "Nested UDT, outer type Serializable, inner type not Serializable. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable"))},
{"UDT_Nested_OuterNotSerializableInnerSerializable", () =>
SerializationTests.TestUDT<UDT_Nested_OuterNotSerializableInnerSerializable>(context, source, "Nested UDT, outer type not Serializable, inner type Serializable. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable"))},
{"UDT_Nested_InnerAndOuterSerializable", () =>
SerializationTests.TestUDT<UDT_Nested_InnerAndOuterSerializable>(context, source, "Nested UDT, both outer and innter types marked Serializable.", null, 0)},
{"UDT_Nested_InnerEnum_InnerAndOuterSerializable", () =>
SerializationTests.TestUDT<UDT_Nested_InnerEnum_InnerAndOuterSerializable>(context, source, "Nested UDT, inner enum, both outer and innter types marked Serializable.", null, 0)},
{"int?", () =>
SerializationTests.TestUDT<int?>(context, source, "Nullable type as the UDT. Should be autoserialized.", null, 0)},
{"UDT_FirstLevelCircular", () =>
SerializationTests.TestUDT<UDT_FirstLevelCircular>(context, source, "Circular UDT. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
{"UDT_FirstLevelCircularArrayRef", () =>
SerializationTests.TestUDT<UDT_FirstLevelCircularArrayRef>(context, source, "Circular UDT with array reference to self. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
{"UDT_SecondLevelCircular", () =>
SerializationTests.TestUDT<UDT_SecondLevelCircular>(context, source, "Circular UDT with indirect reference to self. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
{"UDT_CircularTypeWithCustomSerializer", () =>
SerializationTests.TestUDT<UDT_CircularTypeWithCustomSerializer>(context, source, "Circular UDT with a custom serializer. Should not be rejected.", null, 0)},
{"UDT_TypeContainingCustomSerializedCircularType", () =>
SerializationTests.TestUDT<UDT_TypeContainingCustomSerializedCircularType>(context, source, "An UDT containing a field of a circular UDT with a custom serializer. Should be autoserialized.", null, 0)},
{"UDT_ObjectField", () =>
SerializationTests.TestUDT<UDT_ObjectField>(context, source, "UDT with a field of type System.Object. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"UDT_ObjectArrayField", () =>
SerializationTests.TestUDT<UDT_ObjectArrayField>(context, source, "UDT with a field of type System.Object[]. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"UDT_ObjectListField", () =>
SerializationTests.TestUDT<UDT_ObjectListField>(context, source, "UDT with a field of type List<object>. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"object", () =>
SerializationTests.TestUDT<object>(context, source, "UDT = System.Object. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"object[]", () =>
SerializationTests.TestUDT<object[]>(context, source, "UDT = System.Object[]. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"List<object>", () =>
SerializationTests.TestUDT<List<object>>(context, source, "UDT = List<object>. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"UDT_EmptyType", () =>
SerializationTests.TestUDT<UDT_EmptyType>(context, source, "Empty UDT. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers"))},
{"UDT_ObjectFieldAndCustomSerializer", () =>
SerializationTests.TestUDT<UDT_ObjectFieldAndCustomSerializer>(context, source, "UDT with an Object field, and a custom serializer. Should not be rejected.", null, 0)},
{"UDT_EmptyTypeWithCustomSerializer", () =>
SerializationTests.TestUDT<UDT_EmptyTypeWithCustomSerializer>(context, source, "Empty UDT with a custom serializer. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers"))},
{"UDT_PrivateFieldOfPublicType", () =>
SerializationTests.TestUDT<UDT_PrivateFieldOfPublicType>(context, source, "UDT with a private field of a public type. Should be autoserialized.", null, 0)},
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
// the generic entry method used by most tests in this file
public static bool TestUDT<TRecord>(DryadLinqContext context, IQueryable<int> source, string testMessage, Type expectedExceptionType, int expectedErrorCode)
{
string testName = testMessage;
TestLog.TestStart(testName);
bool passed = true;
// first make sure SelectFunc knows how to handle the type.
try
{
object o = SelectFunc<TRecord>(0);
}
catch
{
TestLog.Message(String.Format("SelectFunc doesn't support type {0}", typeof(TRecord).Name));
passed &= false;
goto Done;
}
try
{
var results = source.Select(x => SelectFunc<TRecord>(x));
int count = 0;
foreach (var r in results) count++;
if (count == 0)
{
TestLog.Message("...FAILED! No elements returned from query");
passed &= false;
goto Done;
}
}
catch (Exception exp)
{
if (expectedExceptionType == null)
{
TestLog.Message(String.Format("...FAILED! Caught {0} while none was expected: ", exp.GetType().Name));
passed &= false;
goto Done;
}
else if (expectedExceptionType != exp.GetType())
{
TestLog.Message(String.Format("...FAILED! Caught {0} while while {1} was expected: {0}", exp.GetType().Name, expectedExceptionType.Name));
passed &= false;
goto Done;
}
else if (exp is DryadLinqException && expectedErrorCode != 0 && ((DryadLinqException)exp).ErrorCode != expectedErrorCode)
{
TestLog.Message("...FAILED! Caught DryadLinqException but fault code is wrong.");
passed &= false;
goto Done;
}
TestLog.Message("....PASSED (caught correct exception)");
goto Done;
}
if (expectedExceptionType == null)
{
TestLog.Message(".....PASSED");
}
else
{
TestLog.Message("...FAILED! No exception was thrown from query.");
passed &= false;
}
Done:
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static TRecord SelectFunc<TRecord>(int x)
{
if (typeof(TRecord) == typeof(object))
{
return (TRecord)(new object());
}
if (typeof(TRecord) == typeof(object[]))
{
return (TRecord)((object)new object[0]);
}
if (typeof(TRecord) == typeof(List<object>))
{
return (TRecord)((object)new List<object>());
}
if (typeof(TRecord) == typeof(UDT_Undecorated))
{
return (TRecord)((object)new UDT_Undecorated(x));
}
if (typeof(TRecord) == typeof(UDT_SelfCustomSerializer))
{
return (TRecord)((object)new UDT_SelfCustomSerializer(x));
}
if (typeof(TRecord) == typeof(UDT_MarkedSerializable))
{
return (TRecord)((object)new UDT_MarkedSerializable(x));
}
if (typeof(TRecord) == typeof(UDT_StaticSerializer))
{
return (TRecord)((object)new UDT_StaticSerializer(x));
}
if (typeof(TRecord) == typeof(UDT_ExternalCustomSerializer))
{
return (TRecord)((object)new UDT_ExternalCustomSerializer(x));
}
if (typeof(TRecord) == typeof(UDT_BadCustomSerializerType1))
{
return (TRecord)((object)new UDT_BadCustomSerializerType1(x));
}
if (typeof(TRecord) == typeof(UDT_BadCustomSerializerType2))
{
return (TRecord)((object)new UDT_BadCustomSerializerType2(x));
}
if (typeof(TRecord) == typeof(UDT_BaseType))
{
return (TRecord)((object)new UDT_BaseType(x));
}
if (typeof(TRecord) == typeof(UDT_DerivedType))
{
return (TRecord)((object)new UDT_DerivedType(x));
}
if (typeof(TRecord) == typeof(UDT_DerivedType))
{
return (TRecord)((object)new UDT_DerivedType(x));
}
if (typeof(TRecord) == typeof(UDT_FieldOfNonPublicType))
{
return (TRecord)((object)new UDT_FieldOfNonPublicType(x));
}
if (typeof(TRecord) == typeof(UDT_PrivateFieldOfPublicType))
{
return (TRecord)((object)new UDT_PrivateFieldOfPublicType(x));
}
if (typeof(TRecord) == typeof(UDT_Nested_OuterSerializableInnerNotSerializable))
{
return (TRecord)((object)new UDT_Nested_OuterSerializableInnerNotSerializable(x));
}
if (typeof(TRecord) == typeof(UDT_Nested_OuterNotSerializableInnerSerializable))
{
return (TRecord)((object)new UDT_Nested_OuterNotSerializableInnerSerializable(x));
}
if (typeof(TRecord) == typeof(UDT_Nested_InnerAndOuterSerializable))
{
return (TRecord)((object)new UDT_Nested_InnerAndOuterSerializable(x));
}
if (typeof(TRecord) == typeof(UDT_Nested_InnerEnum_InnerAndOuterSerializable))
{
return (TRecord)((object)new UDT_Nested_InnerEnum_InnerAndOuterSerializable(x));
}
if (typeof(TRecord) == typeof(int?))
{
return (TRecord)((object)((int?)x));
}
if (typeof(TRecord) == typeof(UDT_FirstLevelCircular))
{
return (TRecord)((object)new UDT_FirstLevelCircular(x));
}
if (typeof(TRecord) == typeof(UDT_FirstLevelCircularArrayRef))
{
return (TRecord)((object)new UDT_FirstLevelCircularArrayRef(x));
}
if (typeof(TRecord) == typeof(UDT_SecondLevelCircular))
{
return (TRecord)((object)new UDT_SecondLevelCircular(x));
}
if (typeof(TRecord) == typeof(UDT_CircularTypeWithCustomSerializer))
{
return (TRecord)((object)new UDT_CircularTypeWithCustomSerializer(x));
}
if (typeof(TRecord) == typeof(UDT_TypeContainingCustomSerializedCircularType))
{
return (TRecord)((object)new UDT_TypeContainingCustomSerializedCircularType(x));
}
if (typeof(TRecord) == typeof(UDT_ObjectField))
{
return (TRecord)((object)new UDT_ObjectField(x));
}
if (typeof(TRecord) == typeof(UDT_ObjectArrayField))
{
return (TRecord)((object)new UDT_ObjectArrayField(x));
}
if (typeof(TRecord) == typeof(UDT_ObjectListField))
{
return (TRecord)((object)new UDT_ObjectListField(x));
}
if (typeof(TRecord) == typeof(UDT_ObjectFieldAndCustomSerializer))
{
return (TRecord)((object)new UDT_ObjectFieldAndCustomSerializer(x));
}
if (typeof(TRecord) == typeof(UDT_EmptyType))
{
return (TRecord)((object)new UDT_EmptyType(x));
}
if (typeof(TRecord) == typeof(UDT_EmptyTypeWithCustomSerializer))
{
return (TRecord)((object)new UDT_EmptyTypeWithCustomSerializer(x));
}
throw new InvalidOperationException("Unrecognized TRecord");
}
}
}

89
DryadLinqTests/SimpleTests.cs Normal file
View File

@ -0,0 +1,89 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
namespace DryadLinqTests
{
public class C1
{
public int x;
public float y;
public C1(int a, float b)
{
this.x = a;
this.y = b;
}
}
public class SimpleTests
{
public static void Run(string[] args)
{
Test1(args);
//Test2(args);
//Test3(args);
//Test4(args);
//Test5(args);
}
public static void Test1(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var lines = input.Where(x => x.Line.Contains("white"));
foreach (var x in lines) Console.WriteLine(x);
//var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
//result.SubmitAndWait();
}
public static void Test2(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var q1 = input.Where(x => x.Line.Contains("white"));
var q2 = input.Where(x => x.Line.Contains("the"));
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
DryadLinqQueryable.SubmitAndWait(res1, res2);
}
public static void Test3(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
result.SubmitAndWait();
}
public static void Test4(string[] args)
{
DryadLinqContext context = new DryadLinqContext("svc-d2-01");
var input = context.FromStore<LineRecord>("hdfs://svc-d2-01:8033/user/misard/foo.txt");
var lines = input.Where(x => x.Line.Contains("white"));
var result = lines.ToStore("hdfs://svc-d2-01:8033/user/yuanbyu/foo.txt", true);
result.SubmitAndWait();
}
public static void Test5(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1);
Uri dataUri = AzureUtils.ToAzureUri("msrsvc", "test", "testwrite");
IEnumerable<LineRecord> lines = DataProvider.ReadData<LineRecord>(context, dataUri);
foreach (var x in lines)
{
Console.WriteLine(x);
}
}
}
}

99
DryadLinqTests/TestLog.cs Normal file
View File

@ -0,0 +1,99 @@
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
namespace DryadLinqTests
{
public class TestResult
{
public DryadLinqContext Context { get; private set; }
public string TestName { get; private set; }
public bool Passed { get; private set; }
//public string Error { get; private set; }
public TestResult(string testName, DryadLinqContext context, bool passed)
{
TestName = testName;
Passed = passed;
Context = context;
}
}
public class TestLog
{
static internal int nTestsRun;
static internal int nTestsPassed;
static internal string fileName = "";
internal static void LogInit(string path)
{
fileName = path;
// create empty file
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName))
{
}
}
// NOTE: QA test harnesses may rely on the formatting
internal static void TestStart(string testName)
{
Message(" --- Starting: [" + testName + "] --- ");
}
// NOTE: QA tests may rely on the formatting
internal static void LogResult(TestResult result)
{
nTestsRun++;
if (result.Passed)
{
nTestsPassed++;
}
Message(" * " + (result.Passed ? "Pass" : "FAIL") );
Message(" --- Completed: [" + result.TestName + "] --- ");
NewLine();
}
internal static void LogResult(bool result)
{
nTestsRun++;
if (result)
{
nTestsPassed++;
}
NewLine();
}
internal static void NewLine()
{
if (fileName.Length > 0)
{
// append data
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName, true))
{
file.WriteLine();
}
}
Console.WriteLine();
}
internal static void Message(string msg)
{
if (fileName.Length > 0)
{
// append data
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName, true))
{
file.WriteLine(msg);
}
}
Console.WriteLine(msg);
}
}
}

496
DryadLinqTests/TypesInQueryTests.cs Normal file
View File

@ -0,0 +1,496 @@
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Text.RegularExpressions;
namespace DryadLinqTests
{
public class TypesInQueryTests
{
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" TypesInQueryTests ");
TestLog.Message(" **********************");
var tests = new Dictionary<string, Action>()
{
{"NonSealedTypeRecords", () => NonSealedTypeRecords(context) },
{"DerivedTypeRecords", () => DerivedTypeRecords(context) },
{"ObjectRecords", () => ObjectRecords(context) },
{"BadRecordsNotSerialized", () => BadRecordsNotSerialized(context) },
{"GroupByWithAnonymousTypes_Bug15675", () => GroupByWithAnonymousTypes_Bug15675(context) },
{"GroupByWithAnonymousTypes_Pipeline", () => GroupByWithAnonymousTypes_Pipeline(context) },
{"GroupByWithAnonymousTypes_MultipleAnonymousTypes", () => GroupByWithAnonymousTypes_MultipleAnonymousTypes(context) },
{"GroupByWithAnonymousTypes_GenericWithAnonTypeParam", () => GroupByWithAnonymousTypes_GenericWithAnonTypeParam(context) },
{"GroupByWithAnonymousTypes_ArrayOfAnon", () => GroupByWithAnonymousTypes_ArrayOfAnon(context) },
{"GroupByWithAnonymousTypes_NestedAnonTypes", () => GroupByWithAnonymousTypes_NestedAnonTypes(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static bool NonSealedTypeRecords(DryadLinqContext context)
{
string testName = "NonSealedTypeRecords";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<NonSealedClass>[] result = new IEnumerable<NonSealedClass>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => new NonSealedClass()).ToArray();
result[0] = output;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => new NonSealedClass()).ToArray();
result[1] = output;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool DerivedTypeRecords(DryadLinqContext context)
{
string testName = "DerivedTypeRecords";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<ParentClass>[] result1 = new IEnumerable<ParentClass>[2];
IEnumerable<ChildClass>[] result2 = new IEnumerable<ChildClass>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output1 = pt1.Select(x => new ParentClass()).ToArray();
var output2 = pt1.Select(x => new ChildClass()).ToArray();
result1[0] = output1;
result2[0] = output2;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output1 = pt1.Select(x => new ParentClass()).ToArray();
var output2 = pt1.Select(x => new ChildClass()).ToArray();
result1[1] = output1;
result2[1] = output2;
}
// compare result
try
{
Validate.Check(result1);
Validate.Check(result2);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool ObjectRecords(DryadLinqContext context)
{
string testName = "ObjectRecords";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<object>[] result = new IEnumerable<object>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => new object()).ToArray();
result[0] = output;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => new object()).ToArray();
result[1] = output;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool BadRecordsNotSerialized(DryadLinqContext context)
{
string testName = "BadRecordsNotSerialized";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<string>[] result = new IEnumerable<string>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => (x % 2 == 0 ? new ChildClass() : new ParentClass())).Select(x => x is ChildClass ? "child" : "parent").ToArray();
result[0] = output;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => (x % 2 == 0 ? new ChildClass() : new ParentClass())).Select(x => x is ChildClass ? "child" : "parent").ToArray();
result[1] = output;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
//Bug 15675 -- GroupBy creates a DryadOrderByNode which is confused by the anonymous type.
// specifically, the call to DryadSort incorrectly referenced both the anonymous type and the code-gen proxy for the anonymous type.
// -- the fix was to clean up how & when we process the types involved in the query so that only the proxy would be referenced.
public static bool GroupByWithAnonymousTypes_Bug15675(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_Bug15675";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new { Num = i % 10 })
.GroupBy(x => x.Num, x => x.Num)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new { Num = i % 10 })
.GroupBy(x => x.Num, x => x.Num)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool GroupByWithAnonymousTypes_Pipeline(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_Pipeline";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new { Num = i % 10 })
.Where(x => true)
.GroupBy(x => x.Num, x => x.Num)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new { Num = i % 10 })
.Where(x => true)
.GroupBy(x => x.Num, x => x.Num)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool GroupByWithAnonymousTypes_MultipleAnonymousTypes(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_MultipleAnonymousTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new { Num = i % 10 })
.Where(x => true)
.Select(i => new { Num2 = i.Num })
.GroupBy(x => x.Num2, x => x.Num2)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new { Num = i % 10 })
.Where(x => true)
.Select(i => new { Num2 = i.Num })
.GroupBy(x => x.Num2, x => x.Num2)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool GroupByWithAnonymousTypes_GenericWithAnonTypeParam(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_GenericWithAnonTypeParam";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => MakeNewMyGenericType(new { Num = i % 10 }))
.GroupBy(x => x.Field.Num, x => x.Field.Num)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => MakeNewMyGenericType(new { Num = i % 10 }))
.GroupBy(x => x.Field.Num, x => x.Field.Num)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool GroupByWithAnonymousTypes_ArrayOfAnon(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_ArrayOfAnon";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new[] { new { Num = i % 10 } })
.GroupBy(x => x[0].Num, x => x[0].Num)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new[] { new { Num = i % 10 } })
.GroupBy(x => x[0].Num, x => x[0].Num)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool GroupByWithAnonymousTypes_NestedAnonTypes(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_NestedAnonTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new { Num = new { NumInner = new int?(i % 10) } }) // nullable-fields present particular challenge.
.GroupBy(x => x.Num.NumInner, x => x.Num.NumInner)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new { Num = new { NumInner = new int?(i % 10) } }) // nullable-fields present particular challenge.
.GroupBy(x => x.Num.NumInner, x => x.Num.NumInner)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
// This is a helper method to allow 'naming the anonymous type'
// -- there is no name to give 'anonymous' inline to the original query,
// but a generic-method only has to call it by pseudonym T, which is a useful workaround.
public static MyGenericType<T> MakeNewMyGenericType<T>(T item)
{
return new MyGenericType<T>(item);
}
}
[Serializable]
public class MyGenericType<T>
{
public T Field;
public MyGenericType(T data)
{
Field = data;
}
}
[Serializable]
public class NonSealedClass
{
public int X;
}
[Serializable]
public class ParentClass
{
public int X;
}
[Serializable]
public class ChildClass : ParentClass
{
public int Y;
}
}

254
DryadLinqTests/Utils.cs
View File

@ -1,20 +1,40 @@
using Microsoft.Research.DryadLinq;
using Microsoft.Research.DryadLinq;
using Microsoft.Research.DryadLinq.Internal;
using Microsoft.Research.Peloponnese.Storage;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
namespace DryadLinqTests
{
public static class Config
public class Config
{
public static string accountName = @"MyAccountName";
public static string storageKey = @"MyStorageKey";
public static string containerName = @"MyContainerName";
public static string cluster = "MyCcluster";
public static string accountName = @"";
public static string storageKey = @"";
public static string containerName = @"";
public static string cluster = @"";
public static string testLogPath = @"";
public Config(string clusterName, string container, string logPath)
{
cluster = clusterName;
AzureSubscriptions azs = new AzureSubscriptions();
Task<AzureCluster> clusterTask = azs.GetClusterAsync(clusterName);
clusterTask.Wait();
accountName = clusterTask.Result.StorageAccount;
storageKey = clusterTask.Result.StorageKey;
containerName = container;
testLogPath = logPath;
}
}
public class DataGenerator
@ -23,6 +43,16 @@ namespace DryadLinqTests
{
}
public static IQueryable<int> CreateSimpleFileSetsEx()
{
int[] input = { 0, 1, 2 };
IEnumerable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
IEnumerable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
IEnumerable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 4, 4));
return rangePartition.AsQueryable();
}
public static IEnumerable<IEnumerable<int>> CreateSimpleFileSets()
{
IEnumerable<IEnumerable<int>> data = new int[][]
@ -46,9 +76,100 @@ namespace DryadLinqTests
return data;
}
public static IEnumerable<IEnumerable<int>> CreateRangePartitionDataSet()
{
// we need a lot of data to ensure sampler will get some data.
// A few thousand should suffice.
IEnumerable<IEnumerable<int>> data = new int[][]
{
Enumerable.Range(1,1000).ToArray(),
Enumerable.Range(20000,2000).ToArray(),
Enumerable.Range(40000,5000).ToArray(),
};
return data;
}
public static IQueryable<int> GetSimpleFileSets(DryadLinqContext context)
{
//IEnumerable<IEnumerable<int>> data = new int[][]
// {
// new[] { 0, 1, 2, 3 },
// new[] { 4, 5, 6, 7 },
// new[] { 8, 9, 10, 11},
// };
//IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
// "unittest/inputdata/SimpleFile.txt"));
IQueryable<int> input = context.FromEnumerable(new int[1]);
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
IQueryable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 4, 4));
//IQueryable<int> store = rangePartition.ToStore(@"unittest/inputdata/SimpleFile.txt");
return rangePartition;
}
public static IQueryable<int> GetGroupByReduceDataSet(DryadLinqContext context)
{
//IEnumerable<IEnumerable<int>> data = new int[][] {
// Enumerable.Range(1,100).ToArray(),
// Enumerable.Range(101,100).ToArray(),
// };
IQueryable<int> input = context.FromEnumerable(new int[1]);
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 2)); // {0, 1}
IQueryable<int> partitions = range.HashPartition(x => x, 2); // create 2 partitions
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 100 + 1, 100));
return rangePartition;
}
public static IQueryable<int> GetRangePartitionDataSet(DryadLinqContext context)
{
// we need a lot of data to ensure sampler will get some data.
// A few thousand should suffice.
//IEnumerable<IEnumerable<int>> data = new int[][] {
// Enumerable.Range(1,1000).ToArray(),
// Enumerable.Range(20000,2000).ToArray(),
// Enumerable.Range(40000,5000).ToArray(),
// };
IQueryable<int> input = context.FromEnumerable(new int[1]);
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
IQueryable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 20000 + 1, 1000));
return rangePartition;
}
}
[Serializable]
public class ReverseComparer<T> : IComparer<T>
{
IComparer<T> _originalComparer;
public ReverseComparer(IComparer<T> originalComparer)
{
_originalComparer = originalComparer ?? Comparer<T>.Default;
}
public int Compare(T x, T y)
{
return (_originalComparer.Compare(y, x)); //note reversed order of operands
}
public override bool Equals(object obj)
{
ReverseComparer<T> objTyped = obj as ReverseComparer<T>;
return objTyped != null && _originalComparer.Equals(objTyped._originalComparer);
}
public override int GetHashCode()
{
// Modify the hash code so that it differs from the hash code for the underlying comparer.
// It would also probably be good enough to just return _originalComparer.GetHashCode().
return unchecked((_originalComparer.GetHashCode() + 123457) * 10007);
}
}
public class Utils
{
public static bool DeleteFile(string accountName, string accountKey, string containerName, string fileName, bool delSubDirs)
@ -96,15 +217,13 @@ namespace DryadLinqTests
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
container.SetPermissions(containerPermissions);
CloudBlockBlob remoteFile = container.GetBlockBlobReference(fileName);
if (!remoteFile.Exists())
return false;
IEnumerable<IListBlobItem> files = container.ListBlobs(fileName, true);
return (files.Count() > 0);
}
catch (Exception)
{
return false;
}
return true;
}
internal static DryadLinqContext MakeBasicConfig(string cluster) //???
@ -153,5 +272,120 @@ namespace DryadLinqTests
}
return context;
}
internal static DryadLinqRecordReader<TRecord> MakeDryadRecordReader<TRecord>(DryadLinqContext context, string readPath)
{
DryadLinqFactory<TRecord> factory = (DryadLinqFactory<TRecord>)DryadLinqCodeGen.GetFactory(context, typeof(TRecord));
NativeBlockStream nativeStream = ReflectionHelper.CreateDryadLinqFileStream(readPath, FileMode.Open, FileAccess.Read);
// ??? NativeBlockStream nativeStream = ReflectionHelper.CreateDryadLinqFileStream(readPath, FileMode.Open, FileAccess.Read, DscCompressionScheme.None);
DryadLinqRecordReader<TRecord> reader = factory.MakeReader(nativeStream);
return reader;
}
}
public class Validate
{
public static void
Check<T>(
IEnumerable<T>[] ss,
IComparer<T> comparer = null,
bool sort = true,
bool verbose = false,
IComparer<T> sortcomparer = null
)
{
if (ss.Length == 0) return;
if (comparer == null)
{
comparer = Comparer<T>.Default;
if (comparer == null)
{
throw new ArgumentNullException("Can't not be null.");
}
}
if (sortcomparer == null)
sortcomparer = comparer;
T[][] aa = new T[ss.Length][];
for (int i = 0; i < aa.Length; i++)
{
aa[i] = ss[i].ToArray();
if (sort) Array.Sort(aa[i], sortcomparer);
}
int len = aa[0].Length;
for (int i = 1; i < aa.Length; i++)
{
if (aa[i].Length != len)
{
throw new Exception("Wrong number of elements.");
}
}
for (int i = 0; i < len; i++)
{
T elem = aa[0][i];
for (int j = 1; j < aa.Length; j++)
{
if (verbose)
{
//TestOutput.WriteLine("Comparing {0} to {1}", elem.ToString(), aa[j][i].ToString());
}
if (comparer.Compare(elem, aa[j][i]) != 0)
{
throw new Exception("Elements failed to match: " + elem + " != " + aa[j][i]);
}
}
}
}
internal static bool outFileExists(string outFile)
{
try
{
return Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (Exception)
{
return false;
}
}
}
public static class ReflectionHelper
{
/// <summary>
/// Create DryadLinqFileStream object via reflection
/// </summary>
///<param name="parameters"></param>
/// <returns></returns>
public static NativeBlockStream CreateDryadLinqFileStream(params object[] parameters)
{
return Assembly.LoadWithPartialName("Microsoft.Hpc.Linq").GetType("Microsoft.Hpc.Linq.Internal.HpcLinqFileStream") //???
.GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, parameters.Select(p => p.GetType()).ToArray(), null)
.Invoke(parameters) as NativeBlockStream;
}
private static Type s_errorCodeType = null;
public static int GetDryadLinqErrorCode(string name)
{
if (s_errorCodeType == null)
{
Assembly asm = Assembly.Load("Microsoft.Research.DryadLinq");
Type[] types = asm.GetTypes();
foreach (var t in types)
{
if (t.Name == "DryadLinqErrorCode")
{
s_errorCodeType = t;
break;
}
}
}
var finfo = s_errorCodeType.GetField(name);
return (int)finfo.GetValue(null);
}
}
}

267
DryadLinqTests/Validate.cs
View File

@ -1,267 +0,0 @@
///-------------------------------------------------------------------------------------------------
// file: Validate.cs
//
// summary: Implements the validate class
///-------------------------------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
namespace BenchmarkFramework {
///-------------------------------------------------------------------------------------------------
/// <summary> Validation utils </summary>
///-------------------------------------------------------------------------------------------------
public class Validate {
public static void
Check<T>(
IEnumerable<T>[] ss,
IComparer<T> comparer = null,
bool sort = true,
bool verbose = false,
IComparer<T> sortcomparer = null
) {
if(ss.Length == 0) return;
if(comparer == null) {
comparer = Comparer<T>.Default;
if(comparer == null) {
throw new ArgumentNullException("Can't not be null.");
}
}
if(sortcomparer == null)
sortcomparer = comparer;
T[][] aa = new T[ss.Length][];
for(int i = 0; i < aa.Length; i++) {
aa[i] = ss[i].ToArray();
if(sort) Array.Sort(aa[i], sortcomparer);
}
int len = aa[0].Length;
for(int i = 1; i < aa.Length; i++) {
if(aa[i].Length != len) {
throw new Exception("Wrong number of elements.");
}
}
for(int i = 0; i < len; i++) {
T elem = aa[0][i];
for(int j = 1; j < aa.Length; j++) {
if(verbose) {
//TestOutput.WriteLine("Comparing {0} to {1}", elem.ToString(), aa[j][i].ToString());
}
if(comparer.Compare(elem, aa[j][i]) != 0) {
throw new Exception("Elements failed to match: " + elem + " != " + aa[j][i]);
}
}
}
}
//public static void
//GroupCheck<K, V>(
// IEnumerable<IGrouping<K, V>>[] ss,
// IComparer<K> kComparer = null,
// IComparer<V> vComparer = null
// ) {
// if(ss.Length == 0) return;
// if(kComparer == null) {
// kComparer = Comparer<K>.Default;
// if(kComparer == null) {
// throw new ArgumentNullException("Can't not be null.");
// }
// }
// if(vComparer == null) {
// vComparer = Comparer<V>.Default;
// if(vComparer == null) {
// throw new ArgumentNullException("Can't not be null.");
// }
// }
// IGrouping<K, V>[][] aa = new IGrouping<K, V>[ss.Length][];
// for(int i = 0; i < aa.Length; i++) {
// aa[i] = ss[i].ToArray();
// K[] keys = aa[i].Select(x => x.Key).ToArray();
// Array.Sort(keys, aa[i], kComparer);
// }
// int len = aa[0].Length;
// for(int i = 1; i < aa.Length; i++) {
// if(aa[i].Length != len) {
// throw new Exception("Wrong number of elements.");
// }
// }
// for(int i = 0; i < len; i++) {
// IEnumerable<V> elem = aa[0][i];
// for(int j = 1; j < aa.Length; j++) {
// Check(new IEnumerable<V>[] { elem, aa[j][i] }, vComparer);
// }
// }
//}
}
/*
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantDoubleComparer : IComparer<double> {
Double EPSILON;
public TolerantDoubleComparer(Double _epsilon = 0.000001f) {
EPSILON = _epsilon;
}
public int Compare(Double a, Double b) {
Double delta = a - b;
if(Math.Abs(delta) <= EPSILON)
return 0;
return delta < 0.0f ? -1 : 1;
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantFloatComparer : IComparer<float> {
float EPSILON;
public TolerantFloatComparer(float _epsilon = 0.000001f) {
EPSILON = _epsilon;
}
public int Compare(float a, float b) {
float delta = a - b;
if(Math.Abs(delta) <= EPSILON)
return 0;
return delta < 0.0f ? -1 : 1;
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantVectorComparer : IComparer<Vector> {
float EPSILON;
public TolerantVectorComparer(float _epsilon = 0.0001f) {
EPSILON = _epsilon;
}
public int Compare(Vector a, Vector b) {
for(int i = 0; i < a.m_elems.Length; i++) {
float delta = a.m_elems[i] - b.m_elems[i];
if(Math.Abs(delta) > EPSILON)
return delta < 0.0f ? -1 : 1;
}
return 0;
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Interface for epsilon comparable single. </summary>
///
/// <remarks> Crossbac, 1/16/2014. </remarks>
///
/// <typeparam name="T"> Generic type parameter. </typeparam>
///-------------------------------------------------------------------------------------------------
public interface IEpsilonComparableSingle<T> {
int EpsilonCompare(T a, T b, float epsilon);
}
///-------------------------------------------------------------------------------------------------
/// <summary> Interface for epsilon comparable double. </summary>
///
/// <remarks> Crossbac, 1/16/2014. </remarks>
///
/// <typeparam name="T"> Generic type parameter. </typeparam>
///-------------------------------------------------------------------------------------------------
public interface IEpsilonComparableDouble<T> {
int EpsilonCompare(T a, T b, double epsilon);
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class EpsilonComparer<T> : IComparer<T> where T : IEpsilonComparableSingle<T> {
float EPSILON;
public EpsilonComparer(float _epsilon = 0.0001f) {
EPSILON = _epsilon;
}
public int Compare(T a, T b) {
return a.EpsilonCompare(a, b, EPSILON);
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer for images. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantImageComparer : IComparer<Image> {
float EPSILON;
public TolerantImageComparer(float _epsilon = 0.0001f) {
EPSILON = _epsilon;
}
public int Compare(Image a, Image b) {
for(int i = 0; i < a.m_elems.Length; i++) {
float delta = a.m_elems[i] - b.m_elems[i];
if(Math.Abs(delta) > EPSILON)
return delta < 0.0f ? -1 : 1;
}
return 0;
}
}
/// <summary>
/// Compare two instances of Pair<int,int>.
/// Implement for concrete type as Pair<T1,T2> cannot straighforwardly implement IComparable.
/// </summary>
///
/// <remarks> jcurrey, 3/11/2013. </remarks>
public class PairIntIntComparer : IComparer<Pair<int, int>> {
public int Compare(Pair<int, int> a, Pair<int, int> b) {
int keyComparison = a.Key.CompareTo(b.Key);
if(keyComparison == 0) {
return a.Value.CompareTo(b.Value);
} else {
return keyComparison;
}
}
}
*/
}

59
DryadLinqTests/WordCount.cs Normal file
View File

@ -0,0 +1,59 @@
// comment the following line to run on Azure
#define local
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
namespace DryadLinqTests
{
public class WordCount
{
public static void WordCountExample()
{
#if local
// This overload runs the computation on your local computer using a single worker
var config = new DryadLinqContext(1);
var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
// You can create inputs from any IEnumerable source using this method
var input = config.FromEnumerable(lines);
#else
string clusterName = "Replace with your HDInsight 3.0 cluster name";
// to use the davinci.txt example input below, select your cluster's default
// storage account and container, which automatically includes the sample text
string accountName = "Replace with a storage account name";
string containerName = "Replace with a storage container name";
// This overload creates an Azure-based computation
var config = new DryadLinqContext(clusterName);
config.JobFriendlyName = "DryadLINQ Sample Wordcount";
// plain text files should be read as type LineRecord
var input = config.FromStore<LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
"example/data/gutenberg/davinci.txt"));
#endif
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
#if local
// any collection computed by the query can be materialized back at the client,
// not just the 'output' collection. For large collections this is expensive!
foreach (LineRecord line in toOutput)
{
Console.WriteLine(line.Line);
}
#else
// the 'true' parameter to ToStore means the output will be over-written if you run
// the job more than once
var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
"wc-out.txt"), true).SubmitAndWait();
#endif
}
}
}

24
DryadLinqTests/packages.config
View File

@ -1,21 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Dryad" version="0.1.4-beta001" targetFramework="net45" />
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages>

8
DryadVertex/VertexHost/system/channel/channel.vcxproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -149,13 +149,13 @@
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
</ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
</Project>

2
DryadVertex/VertexHost/system/channel/packages.config
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="Native" />
</packages>

5
DryadVertex/VertexHost/system/dprocess/src/vertexfactory.cpp
View File

@ -277,7 +277,6 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
DryadVertexProgramRef* pProgram)
{
DryadVertexProgramBase* program;
//
// If factory is not supplied, try to get it from factory registry.
// If still unable, fail with error.
@ -294,6 +293,7 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
"Factory Registry called with no arguments");
return DryadError_VertexInitialization;
}
//
// Get vertex factory. If one cannot be found, report initialization error
@ -301,6 +301,7 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
factory = LookUpFactory(argumentList[0]);
if (factory == NULL)
{
DrLogW("Factory Registry called with unknown factory UID %s.", argumentList[0].GetString());
DrStr128 errorString;
errorString.SetF("Factory Registry called with unknown factory UID %s",
argumentList[0].GetString());
@ -309,7 +310,7 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
errorString);
return DryadError_VertexInitialization;
}
//
// report new vertex creation
//

6
DryadVertex/VertexHost/system/managedchannel/Properties/AssemblyInfo.cs
View File

@ -28,7 +28,7 @@ using System.Runtime.InteropServices;
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("managedchannel")]
[assembly: AssemblyProduct("dryad")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.3.0")]
[assembly: AssemblyFileVersion("0.1.3.0")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

2
DryadVertex/VertexHost/system/managedchannel/app.config
View File

@ -4,7 +4,7 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />

58
DryadVertex/VertexHost/system/managedchannel/managedchannel.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -47,81 +47,81 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
<HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@ -160,10 +160,10 @@
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

24
DryadVertex/VertexHost/system/managedchannel/packages.config
View File

@ -1,21 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages>

17
DryadVertex/VertexHost/vertex/managedwrappervertex/ManagedWrapperVertex.cpp
View File

@ -28,7 +28,6 @@ limitations under the License.
#include <mscoree.h>
#include <wrappernativeinfo.h>
#pragma managed
#pragma warning(disable:4947) // so that we can use Assembly::LoadWithPartialName()
@ -202,14 +201,13 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
DrLogI("ManagedWrapperVertex: Calling %s.%s", GetArgument(2), GetArgument(3));
DrLogging::FlushLog();
DrStr128 errorMsg;
DrError error;
{
//
// Instead of invoking the vertex entry point directly from here, we delegate it to the bridge method in the Microsoft.Hpc.Linq assembly, specifically:
// static int Microsoft.Hpc.Linq.Internal.VertexEnv.VertexBridge(string vertexBridgeArgs)
// Instead of invoking the vertex entry point directly from here, we delegate it to the bridge method in the Microsoft.Research.DryadLinq assembly, specifically:
// static int Microsoft.Research.DryadLinq.Internal.VertexEnv.VertexBridge(string vertexBridgeArgs)
//
// This indirect method of invoking the vertex entry point is used so that any type load / assembly load problems coming from user code
// can be caught and reported with full details using the same mechanism that other vertex failures go through (exception dumped into vertexexception.txt etc.)
@ -217,8 +215,10 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
// The format of vertexBridgeArgs is simply a comma separated string packing vertex assembly, class, method name, and the *actual* vertex method args (==the native channel string)
// L"<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
//
System::String ^bridgeAssemblyPartialName = gcnew System::String(L"Microsoft.Research.DryadLinq");
System::String ^bridgeClassName = gcnew System::String(L"Microsoft.Research.DryadLinq.Internal.VertexEnv");
System::String^ classFullName = gcnew System::String(GetArgument(2));
System::String^ assemblyName = classFullName->Substring(0, classFullName->LastIndexOf('.'));
System::String ^bridgeAssemblyPartialName = gcnew System::String(assemblyName);
System::String ^bridgeClassName = gcnew System::String(assemblyName + ".Internal.VertexEnv");
System::String ^bridgeMethodName = gcnew System::String(L"VertexBridge");
//
@ -235,7 +235,6 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
vertexMethodArgs->Append(gcnew System::String(arg.GetString()));
}
//
// Get assembly path, class name, and method name, and construct the vertex bridge args with the following format:
// "<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
@ -243,7 +242,7 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
System::Text::StringBuilder ^vertexBridgeArg = gcnew System::Text::StringBuilder();
vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"c:\\HpcTemp\\user\\jobID\\Microsoft.Hpc.Linq0.dll";
vertexBridgeArg->Append(",");
vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Hpc.Linq.HpcLinq__Vertex";
vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Research.DryadLinq.DryadLinq__Vertex";
vertexBridgeArg->Append(",");
vertexBridgeArg->Append(gcnew System::String(GetArgument(3))); // vertex entry method name L"Select__1";
vertexBridgeArg->Append(",");
@ -302,7 +301,7 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
error = (DrError)hr;
errorMsg.Set("Error returned from managed runtime invocation, ");
errorMsg.Append(DRERRORSTRING(error));
errorMsg.Append("\n");
errorMsg.Append("\n");
DrLogE( "Error returned from managed runtime invocation. %s (%d)", DRERRORSTRING(error), error);
//

8
DryadVertex/VertexHost/vertex/vertexHost/app.config
View File

@ -18,10 +18,10 @@
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

6
GraphManager/kernel/DrCluster.cpp
View File

@ -570,7 +570,7 @@ DrString DrClusterInternal::TranslateFileToURI(DrString leafName, DrString direc
}
void DrClusterInternal::ScheduleProcess(DrAffinityListRef affinities,
DrString name, DrString commandLine,
DrString name, DrString commandLineArgs,
DrProcessTemplatePtr processTemplate,
DrPSRListenerPtr listener)
{
@ -603,8 +603,8 @@ void DrClusterInternal::ScheduleProcess(DrAffinityListRef affinities,
DrLogI("Starting schedule process for %s.%s",
processTemplate->GetProcessClass().GetChars(), name.GetChars());
IProcess^ rawProcess = m_cluster->NewProcess(process, commandLine.GetString());
IProcess^ rawProcess = m_cluster->NewProcess(process, processTemplate->GetCommandLineBase().GetString(), commandLineArgs.GetString());
process->SetProcess(rawProcess);
DrLogI("Assigned GUID %s to process for %s.%s",

3
GraphManager/kernel/DrCluster.h
View File

@ -45,9 +45,10 @@ public:
DrResourcePtr srcResource, DrResourcePtr dstResource, int compressionMode) = 0;
virtual void ScheduleProcess(DrAffinityListRef affinities,
DrString name, DrString commandLine,
DrString name, DrString commandLineArgs,
DrProcessTemplatePtr processTemplate,
DrPSRListenerPtr listener) = 0;
virtual void CancelScheduleProcess(DrProcessHandlePtr process) = 0;
virtual void WaitForStateChange(DrProcessHandlePtr process, DrPSRListenerPtr listener) = 0;

3
GraphManager/kernel/DrClusterInternal.h
View File

@ -165,9 +165,10 @@ public:
DrResourcePtr srcResource, DrResourcePtr dstResource, int compressionMode) DROVERRIDE;
virtual void ScheduleProcess(DrAffinityListRef affinities,
DrString name, DrString commandLine,
DrString name, DrString commandLineArgs,
DrProcessTemplatePtr processTemplate,
DrPSRListenerPtr listener) DROVERRIDE;
virtual void CancelScheduleProcess(DrProcessHandlePtr process) DROVERRIDE;
virtual void WaitForStateChange(DrProcessHandlePtr process, DrPSRListenerPtr listener) DROVERRIDE;

3
GraphManager/vertex/DrCohort.cpp
View File

@ -470,8 +470,7 @@ void DrCohort::StartProcess(DrGraphPtr graph, int version)
processName.SetF("%s v.%d", m_description.GetChars(), version);
DrString commandLine;
commandLine.SetF("%s --noredirect --startfrompn %d",
m_processTemplate->GetCommandLineBase().GetChars(), m_list->Size());
commandLine.SetF("--noredirect --startfrompn %d", m_list->Size());
for (i=0; i<m_list->Size(); ++i)
{

12
LinqToDryad/DataProvider.cs
View File

@ -76,7 +76,7 @@ namespace Microsoft.Research.DryadLinq
/// The scheme of this data provider.
/// </summary>
public abstract string Scheme { get; }
/// <summary>
/// Gets the metadata of a specified dataset.
/// </summary>
@ -197,14 +197,13 @@ namespace Microsoft.Research.DryadLinq
/// <typeparam name="T">The record type of the dataset.</typeparam>
/// <param name="context">An instance of <see cref="DryadLinqContext"/></param>
/// <param name="dataSetUri">The URI of the dataset</param>
/// <returns>A query object representing the dsc file set data.</returns>
/// <returns>A query object representing the specified dataset.</returns>
internal static DryadLinqQuery<T> GetPartitionedTable<T>(DryadLinqContext context, Uri dataSetUri)
{
string scheme = DataPath.GetScheme(dataSetUri);
DataProvider dataProvider = DataProvider.GetDataProvider(scheme);
DryadLinqProvider queryProvider = new DryadLinqProvider(context);
dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri);
return new DryadLinqQuery<T>(null, queryProvider, dataProvider, dataSetUri);
return new DryadLinqQuery<T>(context, dataProvider, dataSetUri);
}
/// <summary>
@ -244,7 +243,9 @@ namespace Microsoft.Research.DryadLinq
DataProvider dataProvider = DataProvider.GetDataProvider(scheme);
dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri);
dataProvider.Ingress(context, source, dataSetUri, metaData, outputScheme, isTemp);
return DataProvider.GetPartitionedTable<T>(context, dataSetUri);
DryadLinqQuery<T> res = DataProvider.GetPartitionedTable<T>(context, dataSetUri);
res.CheckAndInitialize(); // must initialize
return res;
}
}
@ -589,7 +590,6 @@ namespace Microsoft.Research.DryadLinq
try
{
AzureCollectionPartition partition = new AzureCollectionPartition(dataSetUri);
if (!partition.IsCollectionExists())
{
throw new DryadLinqException("Input collection " + dataSetUri + " does not exist");

65
LinqToDryad/DryadLinqContext.cs
View File

@ -143,8 +143,13 @@ namespace Microsoft.Research.DryadLinq
_clusterClient = null;
}
public PlatformKind Kind { get { return PlatformKind.YARN_NATIVE; } }
public IDfsClient DfsClient {
public PlatformKind Kind
{
get { return PlatformKind.YARN_NATIVE; }
}
public IDfsClient DfsClient
{
get
{
if (_dfsClient == null)
@ -195,7 +200,8 @@ namespace Microsoft.Research.DryadLinq
// start fetching details about the subscriptions, available clusters, etc.
_azureSubscriptions = new AzureSubscriptions();
_cluster = _azureSubscriptions.GetClusterAsync(clusterName);
_dfsClient = _cluster.ContinueWith(c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, "staging"));
_dfsClient = _cluster.ContinueWith(
c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, "staging"));
}
/// <summary>
@ -215,8 +221,10 @@ namespace Microsoft.Research.DryadLinq
_azureSubscriptions.AddAccount(storageAccount, storageKey);
}
_cluster = _azureSubscriptions.GetClusterAsync(clusterName)
.ContinueWith(t => { t.Result.SetStorageAccount(storageAccount, storageKey); return t.Result; });
_dfsClient = _cluster.ContinueWith(c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, storageContainer));
.ContinueWith(t => { t.Result.SetStorageAccount(storageAccount, storageKey);
return t.Result; });
_dfsClient = _cluster.ContinueWith(
c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, storageContainer));
}
/// <summary>
@ -248,7 +256,8 @@ namespace Microsoft.Research.DryadLinq
_azureSubscriptions = new AzureSubscriptions();
_azureSubscriptions.AddSubscription(subscriptionId, certificate);
_cluster = _azureSubscriptions.GetClusterAsync(clusterName);
_dfsClient = _cluster.ContinueWith(c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, "staging"));
_dfsClient = _cluster.ContinueWith(
c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, "staging"));
}
/// <summary>
@ -272,7 +281,8 @@ namespace Microsoft.Research.DryadLinq
}
_azureSubscriptions.AddCluster(clusterName, storageAccount, storageKey, subscriptionId, certificateThumbprint);
_cluster = _azureSubscriptions.GetClusterAsync(clusterName);
_dfsClient = _cluster.ContinueWith(c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, storageContainer));
_dfsClient = _cluster.ContinueWith(
c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, storageContainer));
}
/// <summary>
@ -320,7 +330,8 @@ namespace Microsoft.Research.DryadLinq
public Uri MakeDefaultUri(string path)
{
return AzureUtils.ToAzureUri(_dfsClient.Result.AccountName, _dfsClient.Result.ContainerName, path, null, _dfsClient.Result.AccountKey);
return AzureUtils.ToAzureUri(
_dfsClient.Result.AccountName, _dfsClient.Result.ContainerName, path, null, _dfsClient.Result.AccountKey);
}
}
@ -363,7 +374,6 @@ namespace Microsoft.Research.DryadLinq
private string _nodeGroup;
private int? _jobRuntimeLimit;
private bool _localDebug = false;
private bool _localExecution = false;
private string _jobUsername = null;
private string _jobPassword = null;
private QueryTraceLevel _runtimeTraceLevel = QueryTraceLevel.Error;
@ -371,11 +381,11 @@ namespace Microsoft.Research.DryadLinq
private bool _enableSpeculativeDuplication = true;
private bool _selectOrderPreserving = false;
private bool _matchClientNetFrameworkVersion = true;
private bool _multiThreading = true;
private string _partitionUncPath = null;
private string _storageSetScheme = null;
private DryadLinqStringDictionary _jobEnvironmentVariables = new DryadLinqStringDictionary();
private DryadLinqStringList _resourcesToAdd = new DryadLinqStringList();
private DryadLinqStringList _resourcesToRemove = new DryadLinqStringList();
@ -414,7 +424,6 @@ namespace Microsoft.Research.DryadLinq
{
this.CommonInit();
this._platformKind = PlatformKind.LOCAL;
this._localExecution = true;
this._headNode = "LocalExecution";
this._storageSetScheme = storageSetScheme;
if (String.IsNullOrEmpty(this._storageSetScheme))
@ -567,7 +576,7 @@ namespace Microsoft.Research.DryadLinq
/// <summary>
/// Gets or sets the partition UNC path used when constructing a partitioned table.
/// </summary>
/// </summary>
public string PartitionUncPath
{
get { return _partitionUncPath; }
@ -682,23 +691,6 @@ namespace Microsoft.Research.DryadLinq
set { _localDebug = value; }
}
/// <summary>
/// Gets or sets the value specifying whether to use Local execution mode.
/// </summary>
/// <remarks>
/// <para>
/// If true, the DryadLINQ Query will execute by forking processes on the local
/// computer instead of using a cluster. LocalExecution mode accesses HDFS as usual for
/// input and output data.
/// </para>
/// <para>The default is false.</para>
/// </remarks>
public bool LocalExecution
{
get { return _localExecution; }
set { _localExecution = value; }
}
/// <summary>
/// Gets and sets the value specifying whether a vertex should break into the debugger
/// </summary>
@ -881,13 +873,13 @@ namespace Microsoft.Research.DryadLinq
switch (this.ExecutorKind)
{
case ExecutorKind.DRYAD:
{
return new DryadLinqJobExecutor(this);
}
{
return new DryadLinqJobExecutor(this);
}
default:
{
throw new Exception("No implementation for scheduler: " + this.ExecutorKind.ToString());
}
{
throw new Exception("No implementation for scheduler: " + this.ExecutorKind.ToString());
}
}
}
@ -922,7 +914,7 @@ namespace Microsoft.Research.DryadLinq
{
ThrowIfDisposed();
DryadLinqQuery<T> q = DataProvider.GetPartitionedTable<T>(this, dataSetUri);
q.CheckAndInitialize(); // force the data-info checks.
q.CheckAndInitialize(); // Must initialize!
return q;
}
@ -1018,7 +1010,6 @@ namespace Microsoft.Research.DryadLinq
this.JobRuntimeLimit == context.JobRuntimeLimit &&
this.EnableSpeculativeDuplication == context.EnableSpeculativeDuplication &&
this.LocalDebug == context.LocalDebug &&
this.LocalExecution == context.LocalExecution &&
this.PlatformKind == context.PlatformKind &&
this.JobUsername == context.JobUsername &&
this.JobPassword == context.JobPassword &&

1
LinqToDryad/DryadLinqFaultCodes.cs
View File

@ -24,6 +24,7 @@ using System.Text;
using System.Linq.Expressions;
using System.Runtime.Serialization;
using System.Reflection;
using System.Runtime.CompilerServices;
namespace Microsoft.Research.DryadLinq
{

4
LinqToDryad/DryadLinqJobExecutor.cs
View File

@ -57,7 +57,7 @@ namespace Microsoft.Research.DryadLinq
// use a new job submission object for each query
this.m_context = context;
this.m_currentStatus = JobStatus.NotSubmitted;
if (context.LocalExecution)
if (context.PlatformKind == PlatformKind.LOCAL)
{
this.m_jobSubmission = new LocalJobSubmission(context);
}
@ -86,8 +86,6 @@ namespace Microsoft.Research.DryadLinq
/// <param name="file">Pathname to file to add as a resource.</param>
private void AddResource(IDryadLinqJobSubmission jobSubmission, string file)
{
// extract basename
string basename = Path.GetFileName(file);
this.m_jobSubmission.AddLocalFile(file);
}

93
LinqToDryad/DryadLinqQuery.cs
View File

@ -221,6 +221,7 @@ namespace Microsoft.Research.DryadLinq
{
this.m_queryProvider = queryProvider;
this.m_dataProvider = dataProvider;
this.m_isTemp = false;
this.m_queryExecutor = null;
}
@ -259,7 +260,7 @@ namespace Microsoft.Research.DryadLinq
public DryadLinqContext Context
{
get { return m_queryProvider.Context; }
get { return this.m_queryProvider.Context; }
}
internal bool IsTemp
@ -277,11 +278,11 @@ namespace Microsoft.Research.DryadLinq
{
if (otherQuery.m_queryProvider == null)
{
otherQuery.m_queryProvider = this.m_queryProvider;
otherQuery.m_queryProvider = this.m_queryProvider;
}
if (otherQuery.m_dataProvider == null)
{
otherQuery.m_dataProvider = this.m_dataProvider;
otherQuery.m_dataProvider = this.m_dataProvider;
}
otherQuery.m_isTemp = this.m_isTemp;
otherQuery.m_queryExecutor = this.m_queryExecutor;
@ -303,24 +304,27 @@ namespace Microsoft.Research.DryadLinq
private Expression m_queryExpression;
private Uri m_dataSourceUri;
private DataSetInfo m_dataSetInfo;
private bool m_isDynamic;
private DryadLinqQueryEnumerable<T> m_tableEnumerable;
private bool m_isDynamic;
private bool m_initialized;
// Used by IQueryProvider. e.g., IQueryable<>.Select() and IQueryable<>.ToStore()
internal DryadLinqQuery(DryadLinqProviderBase provider, Expression expression)
: base(provider, null)
{
this.m_queryExpression = expression;
this.m_isDynamic = false;
this.m_dataSourceUri = null;
this.m_dataSetInfo = null;
this.m_tableEnumerable = null;
this.m_isDynamic = false;
this.m_initialized = false;
}
// Used by DryadLinqContext.LoadFrom(uri)
internal DryadLinqQuery(Expression queryExpression,
DryadLinqProvider queryProvider,
// Used by DryadLinqContext.FromStore(uri)
internal DryadLinqQuery(DryadLinqContext context,
DataProvider dataProvider,
Uri dataSetUri)
: base(queryProvider, dataProvider)
: base(null, dataProvider)
{
if (!DataPath.IsValidDataPath(dataSetUri))
{
@ -328,10 +332,26 @@ namespace Microsoft.Research.DryadLinq
String.Format(SR.UnrecognizedDataSource, dataSetUri.AbsoluteUri));
}
this.m_queryExpression = queryExpression;
this.m_dataSourceUri = dataSetUri;
this.m_dataSetInfo = null;
this.m_isDynamic = false;
this.m_tableEnumerable = null;
this.m_initialized = false;
this.m_tableEnumerable
= new DryadLinqQueryEnumerable<T>(context, this.DataProvider, this.m_dataSourceUri);
// YY: query expression and provider are at least set consistently
if (context.LocalDebug)
{
this.m_queryExpression = Expression.Constant(this.m_tableEnumerable.AsQueryable());
IQueryProvider linqToObjectProvider = this.m_tableEnumerable.AsQueryable().Provider;
this.m_queryProvider = new DryadLinqLocalProvider(linqToObjectProvider, context);
}
else
{
this.m_queryExpression = Expression.Constant(this);
this.m_queryProvider = new DryadLinqProvider(context);
}
}
internal void Clone(DryadLinqQuery<T> otherQuery)
@ -341,8 +361,9 @@ namespace Microsoft.Research.DryadLinq
otherQuery.m_queryExpression = this.m_queryExpression;
otherQuery.m_dataSourceUri = this.m_dataSourceUri;
otherQuery.m_dataSetInfo = this.m_dataSetInfo;
otherQuery.m_isDynamic = this.m_isDynamic;
otherQuery.m_tableEnumerable = this.m_tableEnumerable;
otherQuery.m_isDynamic = this.m_isDynamic;
otherQuery.m_initialized = this.m_initialized;
}
// returns true for DLQ that are pointing directly at plain data.
@ -479,25 +500,23 @@ namespace Microsoft.Research.DryadLinq
internal void Initialize()
{
if (this.IsPlainData && this.m_tableEnumerable == null)
if (this.IsPlainData && !this.m_initialized)
{
DryadLinqStreamInfo
streamInfo = this.DataProvider.GetStreamInfo(this.Context, this.m_dataSourceUri);
DryadLinqStreamInfo streamInfo = this.DataProvider.GetStreamInfo(this.Context, this.m_dataSourceUri);
Int32 parCount = streamInfo.PartitionCount;
Int64 estSize = streamInfo.DataSize;
this.m_isDynamic = false;
// Finally load any stored metadata to check settings, extract compression-setting
// and initialize the DataInfo for this Query. It is uri.. have to convert to stream-name.
DryadLinqMetaData meta = DryadLinqMetaData.Get(Context, this.m_dataSourceUri);
DryadLinqMetaData meta = DryadLinqMetaData.Get(this.Context, this.m_dataSourceUri);
if (meta != null)
{
//check the record-type matches meta-data. (disabled until final API is determined)
//if (meta.ElemType != typeof(T))
//{
// throw new DisributedLinqException(DryadLinqErrorCode.MetadataRecordType,
// String.Format(SR.MetadataRecordType,
// typeof(T), meta.ElemType));
// throw new DryadLinqException(DryadLinqErrorCode.MetadataRecordType,
// String.Format(SR.MetadataRecordType,
// typeof(T), meta.ElemType));
//}
//check the serialization flags match meta-data.
@ -520,25 +539,7 @@ namespace Microsoft.Research.DryadLinq
DistinctInfo dinfo = DataSetInfo.NoDistinct;
this.m_dataSetInfo = new DataSetInfo(pinfo, oinfo, dinfo);
this.m_tableEnumerable
= new DryadLinqQueryEnumerable<T>(this.Context, this.DataProvider, this.m_dataSourceUri);
// YY: query expression and provider are at least set consistently
if (Context.LocalDebug)
{
this.m_queryExpression = Expression.Constant(this.m_tableEnumerable.AsQueryable());
IQueryProvider linqToObjectProvider = this.m_tableEnumerable.AsQueryable().Provider;
this.m_queryProvider = new DryadLinqLocalProvider(linqToObjectProvider, Context);
}
else
{
this.m_queryExpression = Expression.Constant(this);
if (this.m_queryProvider == null)
{
// Only set if not provided
this.m_queryProvider = new DryadLinqProvider(this.Context);
}
}
this.m_initialized = true;
}
}
@ -566,11 +567,23 @@ namespace Microsoft.Research.DryadLinq
}
else
{
DryadLinqQueryable.SubmitAndWait(this);
this.ToTemporary();
return this.m_backingData.GetEnumerator();
}
}
private void ToTemporary()
{
// Execute this query and store the result in a temp location
Uri tableUri = this.Context.MakeTemporaryStreamUri();
DryadLinqQueryGen dryadGen = new DryadLinqQueryGen(
this.Context, this.GetVertexCodeGen(), this.m_queryExpression, tableUri, true);
DryadLinqQuery[] tables = dryadGen.Execute();
tables[0].IsTemp = true;
this.BackingData = tables[0];
}
// Generate the query plan as an XML file and return the file name.
// returns the queryPlan xml path.
internal string ToDryadLinqProgram()

1
LinqToDryad/DryadLinqQueryGen.cs
View File

@ -850,7 +850,6 @@ namespace Microsoft.Research.DryadLinq
queryDoc.DocumentElement.AppendChild(elem);
// Add the visualization element
//@@TODO[p2]: remove this element from the queryXML.
elem = queryDoc.CreateElement("Visualization");
elem.InnerText = "none";
queryDoc.DocumentElement.AppendChild(elem);

6
LinqToDryad/DryadLinqQueryNode.cs
View File

@ -132,9 +132,9 @@ namespace Microsoft.Research.DryadLinq
protected internal List<Pair<ParameterExpression, DLinqQueryNode>> m_referencedQueries;
internal DLinqQueryNode(QueryNodeType nodeType,
DryadLinqQueryGen queryGen,
Expression queryExpr,
params DLinqQueryNode[] children)
DryadLinqQueryGen queryGen,
Expression queryExpr,
params DLinqQueryNode[] children)
{
this.m_nodeType = nodeType;
this.m_queryGen = queryGen;

5
LinqToDryad/DryadLinqQueryable.cs
View File

@ -4232,7 +4232,10 @@ namespace Microsoft.Research.DryadLinq
BindingFlags.Static | BindingFlags.NonPublic);
Type elemType = mcExpr.Type.GetGenericArguments()[0];
minfo = minfo.MakeGenericMethod(elemType);
mcExpr = Expression.Call(minfo, mcExpr, Expression.Constant(tableUri));
mcExpr = Expression.Call(minfo,
mcExpr,
Expression.Constant(tableUri, typeof(Uri)),
Expression.Constant(isTemp, typeof(bool)));
}
qList.Add(mcExpr);
isTempList.Add(isTemp);

58
LinqToDryad/LinqToDryad.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -58,68 +58,68 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
@ -128,13 +128,13 @@
<Reference Include="System.Drawing" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Runtime.Serialization" />
@ -249,10 +249,10 @@
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

19
LinqToDryad/LocalJobSubmission.cs
View File

@ -41,8 +41,8 @@ namespace Microsoft.Research.DryadLinq
public LocalJobSubmission(DryadLinqContext context) : base(context)
{
m_status = JobStatus.NotSubmitted;
m_error = null;
this.m_status = JobStatus.NotSubmitted;
this.m_error = null;
}
public override string ErrorMsg
@ -51,7 +51,7 @@ namespace Microsoft.Research.DryadLinq
{
lock (this)
{
return m_error;
return this.m_error;
}
}
}
@ -60,7 +60,7 @@ namespace Microsoft.Research.DryadLinq
{
lock (this)
{
return m_status;
return this.m_status;
}
}
@ -88,8 +88,8 @@ namespace Microsoft.Research.DryadLinq
environment.Add("PELOPONNESE_ADDITIONAL_CLASSPATH", jarPath);
// add the query plan to the JM directory so that job analysis tools can find it later
string queryPlanDirectory = Path.GetDirectoryName(QueryPlan);
string queryPlanFile = Path.GetFileName(QueryPlan);
string queryPlanDirectory = Path.GetDirectoryName(this.QueryPlan);
string queryPlanFile = Path.GetFileName(this.QueryPlan);
HashSet<string> queryPlanSet = new HashSet<string>();
queryPlanSet.Add(queryPlanFile);
List<XElement> resources = new List<XElement>();
@ -110,6 +110,9 @@ namespace Microsoft.Research.DryadLinq
protected override XElement MakeWorkerConfig(string configPath, XElement peloponneseResource)
{
Dictionary<string, string> environment = new Dictionary<string, string>(this.Context.JobEnvironmentVariables);
// add job-local resources to each worker directory, leaving out the standard Dryad files
var resources = new List<XElement>();
foreach (var rg in LocalResources)
@ -126,7 +129,7 @@ namespace Microsoft.Research.DryadLinq
return ConfigHelpers.MakeProcessGroup(
"Worker", "local", 2, numWorkerProcesses, false,
psPath, psArgs, null, "processservice-stdout.txt", "processservice-stderr.txt",
resources, null);
resources, environment);
}
private string MakeProcessServiceConfig()
@ -242,7 +245,7 @@ namespace Microsoft.Research.DryadLinq
Console.WriteLine(m_error);
return;
}
this.m_workingDirectory = wd;
}

10
LinqToDryad/Properties/AssemblyInfo.cs
View File

@ -1,4 +1,4 @@
using System.Reflection;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
@ -9,8 +9,8 @@ using System.Runtime.InteropServices;
[assembly: AssemblyDescription("DryadLinq client assembly")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft")]
[assembly: AssemblyProduct("Microsoft.Research.DryadLinq.Properties")]
[assembly: AssemblyCopyright("Copyright © 2014")]
[assembly: AssemblyProduct("Dryad")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -29,5 +29,5 @@ using System.Runtime.InteropServices;
// Build Number
// Revision
//
[assembly: AssemblyVersion("0.1.3.0")]
[assembly: AssemblyFileVersion("0.1.3.0")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

73
LinqToDryad/TypeSystem.cs
View File

@ -412,55 +412,62 @@ namespace Microsoft.Research.DryadLinq
HashSet<Assembly> assemblies = TypeSystem.GetAllAssemblies();
foreach (Assembly asm in assemblies)
{
foreach (Type type in asm.GetTypes())
try
{
if (!type.IsInterface)
foreach (Type type in asm.GetTypes())
{
Type[] baseTypes = type.GetInterfaces();
if (type.BaseType != null && type.BaseType != typeof(object))
if (!type.IsInterface)
{
Type[] newBaseTypes = new Type[baseTypes.Length + 1];
Array.Copy(baseTypes, newBaseTypes, baseTypes.Length);
newBaseTypes[baseTypes.Length] = type.BaseType;
baseTypes = newBaseTypes;
}
for (int i = 0; i < baseTypes.Length; i++)
{
Type baseType = baseTypes[i];
if (baseType.IsGenericType)
Type[] baseTypes = type.GetInterfaces();
if (type.BaseType != null && type.BaseType != typeof(object))
{
baseType = baseType.GetGenericTypeDefinition();
baseTypes[i] = baseType;
Type[] newBaseTypes = new Type[baseTypes.Length + 1];
Array.Copy(baseTypes, newBaseTypes, baseTypes.Length);
newBaseTypes[baseTypes.Length] = type.BaseType;
baseTypes = newBaseTypes;
}
bool isNew = true;
for (int j = 0; j < i; j++)
for (int i = 0; i < baseTypes.Length; i++)
{
if (baseTypes[j] == baseType)
Type baseType = baseTypes[i];
if (baseType.IsGenericType)
{
isNew = false;
break;
baseType = baseType.GetGenericTypeDefinition();
baseTypes[i] = baseType;
}
}
if (isNew)
{
Type[] deriveds = null;
if (typeMap.TryGetValue(baseType, out deriveds))
bool isNew = true;
for (int j = 0; j < i; j++)
{
Type[] newDeriveds = new Type[deriveds.Length + 1];
Array.Copy(deriveds, newDeriveds, deriveds.Length);
newDeriveds[deriveds.Length] = type;
deriveds = newDeriveds;
if (baseTypes[j] == baseType)
{
isNew = false;
break;
}
}
else
if (isNew)
{
deriveds = new Type[1] { type };
Type[] deriveds = null;
if (typeMap.TryGetValue(baseType, out deriveds))
{
Type[] newDeriveds = new Type[deriveds.Length + 1];
Array.Copy(deriveds, newDeriveds, deriveds.Length);
newDeriveds[deriveds.Length] = type;
deriveds = newDeriveds;
}
else
{
deriveds = new Type[1] { type };
}
typeMap[baseType] = deriveds;
}
typeMap[baseType] = deriveds;
}
}
}
}
catch (Exception)
{
// Console.WriteLine("Warning: Could not retrieve the types in: " + asm.FullName);
}
}
s_typeMap = typeMap;
return s_typeMap;

32
LinqToDryad/YarnJobSubmission.cs
View File

@ -61,10 +61,10 @@ namespace Microsoft.Research.DryadLinq
string logDirParam = Microsoft.Research.Peloponnese.Storage.AzureUtils.CmdLineEncode(jobDirectoryTemplate);
string[] jmArgs = {"--dfs=" + logDirParam, "VertexHost.exe", qpPath }; // +" --break";
return ConfigHelpers.MakeProcessGroup(
"jm", "local", 1, 1, true,
"jm", "local", 1, 1, true,
jmPath, jmArgs, "LOG_DIRS", "graphmanager-stdout.txt",
"graphmanager-stderr.txt",
null, null);
null, null);
}
protected override XElement MakeWorkerConfig(string configPath, XElement peloponneseResource)
@ -117,9 +117,9 @@ namespace Microsoft.Research.DryadLinq
string[] psArgs = { Path.GetFileName(configPath) };
int maxNodes = (Context.JobMaxNodes == null) ? -1 : Context.JobMaxNodes.Value;
return ConfigHelpers.MakeProcessGroup(
"Worker", "yarn", -1, maxNodes, false,
psPath, psArgs, "LOG_DIRS", "processservice-stdout.txt", "processservice-stderr.txt",
resources, null);
"Worker", "yarn", -1, maxNodes, false,
psPath, psArgs, "LOG_DIRS", "processservice-stdout.txt", "processservice-stderr.txt",
resources, null);
}
private string MakeProcessServiceConfig()
@ -220,17 +220,17 @@ namespace Microsoft.Research.DryadLinq
{
get
{
if (m_job == null)
if (this.m_job == null)
{
return null;
}
return m_job.ErrorMsg;
return this.m_job.ErrorMsg;
}
}
public override JobStatus GetStatus()
{
if (m_job == null)
if (this.m_job == null)
{
return JobStatus.NotSubmitted;
}
@ -261,7 +261,13 @@ namespace Microsoft.Research.DryadLinq
}
}
public string JobDirectory { get { return Context.Cluster.Client(Context).JobDirectoryTemplate.Replace("_BASELOCATION_", "dryad-jobs"); } }
private string JobDirectory
{
get
{
return Context.Cluster.Client(Context).JobDirectoryTemplate.Replace("_BASELOCATION_", "dryad-jobs");
}
}
public override void SubmitJob()
{
@ -282,7 +288,7 @@ namespace Microsoft.Research.DryadLinq
try
{
m_job = Context.Cluster.Client(Context).Submit(config, JobDirectory);
this.m_job = Context.Cluster.Client(Context).Submit(config, JobDirectory);
}
catch (Exception e)
{
@ -292,19 +298,19 @@ namespace Microsoft.Research.DryadLinq
public override JobStatus TerminateJob()
{
m_job.Kill();
this.m_job.Kill();
return GetStatus();
}
public override string GetJobId()
{
if (m_job == null)
if (this.m_job == null)
{
return "Unknown";
}
else
{
return m_job.Id;
return this.m_job.Id;
}
}
}

2
LinqToDryad/app.config
View File

@ -4,7 +4,7 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />

24
LinqToDryad/packages.config
View File

@ -1,21 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages>

58
LocalScheduler/LocalScheduler.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -46,81 +46,81 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@ -162,10 +162,10 @@
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

10
LocalScheduler/app.config
View File

@ -4,7 +4,7 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
@ -22,6 +22,14 @@
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.0.7" newVersion="1.1.0.7" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.0.7" newVersion="1.1.0.7" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

24
LocalScheduler/packages.config
View File

@ -1,21 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages>

10
Microsoft.Research.Dryad.nuspec
View File

@ -2,7 +2,7 @@
<package xmlns="http://schemas.microsoft.com/packaging/2013/01/nuspec.xsd">
<metadata minClientVersion="2.5">
<id>Microsoft.Research.Dryad</id>
<version>0.1.3-beta</version>
<version>0.1.5-beta</version>
<title>Dryad and DryadLINQ for YARN</title>
<authors>bigdatadev@microsoft.com</authors>
<owners>bigdatadev@microsoft.com,Microsoft</owners>
@ -17,7 +17,8 @@ Dryad provides reliable, distributed computing on thousands of servers for large
<language>en-US</language>
<dependencies>
<group targetFramework=".NETFramework4.5">
<dependency id="Microsoft.Research.Peloponnese" version="0.7.2-beta" />
<dependency id="Microsoft.Research.Peloponnese" version="0.7.5-beta" />
<!--<dependency id="zlib" version="1.2.8.6" />-->
</group>
</dependencies>
<frameworkAssemblies>
@ -29,7 +30,7 @@ Dryad provides reliable, distributed computing on thousands of servers for large
<frameworkAssembly assemblyName="System.Runtime.Serialization" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.ServiceModel" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Xml" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Xml.Linq" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Xml.Linq" targetFramework=".NETFramework4.5" />
</frameworkAssemblies>
<references>
<reference file="Microsoft.Research.DryadLinq.dll" />
@ -60,7 +61,6 @@ Dryad provides reliable, distributed computing on thousands of servers for large
<file src="bin\Release\VertexHost.exe.config" target="lib\net45\VertexHost.exe.config" />
<file src="bin\signed\VertexHost.exe" target="lib\net45\VertexHost.exe" />
<file src="bin\Release\VertexHost.pdb" target="lib\net45\VertexHost.pdb" />
<file src="Microsoft.Research.Dryad.targets" target="build\Microsoft.Research.Dryad.targets" />
<file src="Install.ps1" target="tools\Install.ps1" />
<file src="Microsoft.Research.Dryad.targets" target="build\Microsoft.Research.Dryad.targets" />
</files>
</package>

8
ProcessService/HttpServer.cs
View File

@ -236,12 +236,14 @@ namespace Microsoft.Research.Dryad.ProcessService
try
{
string commandLine;
string arguments;
using (var sr = new System.IO.StreamReader(context.Request.InputStream))
{
commandLine = sr.ReadToEnd();
commandLine = sr.ReadLine();
arguments = sr.ReadLine();
}
logger.Log("Received create for process " + processId + " cmdline: " + commandLine);
logger.Log("Received create for process " + processId + " cmdline: " + commandLine + " arguments: " + arguments);
if (parent.Create(processId))
{
@ -254,7 +256,7 @@ namespace Microsoft.Research.Dryad.ProcessService
await server.ReportError(context, HttpStatusCode.Conflict, "Process " + processId + " already exists");
}
parent.Launch(processId, commandLine);
parent.Launch(processId, commandLine, arguments);
return;
}

59
ProcessService/ProcessService.cs
View File

@ -565,7 +565,42 @@ namespace Microsoft.Research.Dryad.ProcessService
return true;
}
public void Launch(int processId, string commandLine)
private void SplitCmdLine(string cmdLine, out string cmd, out string args)
{
cmd = "";
args = "";
int lastSpacePos = 0;
bool rootedPath = Path.IsPathRooted(cmdLine);
string candPath = cmdLine;
while (true)
{
if (!rootedPath)
{
candPath = Path.Combine(Environment.CurrentDirectory, candPath);
}
if (File.Exists(candPath))
{
cmd = candPath;
args = (lastSpacePos > 0) ? cmdLine.Substring(lastSpacePos) : "";
return;
}
int spacePos = cmdLine.IndexOf(' ', lastSpacePos);
if (spacePos != -1)
{
candPath = cmdLine.Substring(0, spacePos);
lastSpacePos = spacePos + 1;
}
else
{
break;
}
}
throw new ApplicationException("Couldn't split command line into command line and arguments.");
}
public void Launch(int processId, string commandLine, string arguments)
{
ProcessRecord process;
lock (processTable)
@ -588,29 +623,23 @@ namespace Microsoft.Research.Dryad.ProcessService
startInfo.RedirectStandardError = true;
startInfo.WorkingDirectory = Path.Combine(serviceWorkingDirectory, processId.ToString());
logger.Log(String.Format("Working directory: '{0}'", startInfo.WorkingDirectory));
string[] args = commandLine.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
string arg = "";
for (int i = 1; i < args.Length; i++)
{
arg += args[i] + " ";
}
// Use either FQ path or path relative to job path
if (Path.IsPathRooted(args[0]))
if (Path.IsPathRooted(commandLine))
{
startInfo.FileName = args[0];
startInfo.FileName = commandLine;
}
else
{
startInfo.FileName = Path.Combine(serviceWorkingDirectory, args[0]);
startInfo.FileName = Path.Combine(serviceWorkingDirectory, commandLine);
}
startInfo.Arguments = arguments;
logger.Log(String.Format("FileName: '{0}'", startInfo.FileName));
// Add environment variable to vertex host process
startInfo.Arguments = arg;
logger.Log(String.Format("args: '{0}'", arg));
logger.Log(String.Format("args: '{0}'", arguments));
// Add environment variable to vertex host process
Uri genericUri = new Uri(processServer.BaseURI);
Uri localUri = new Uri(genericUri.Scheme + "://localhost:" + genericUri.Port + genericUri.PathAndQuery);
string processUpdateURI = localUri.ToString() + processId.ToString();

58
ProcessService/ProcessService.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -47,81 +47,81 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@ -158,10 +158,10 @@
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

6
ProcessService/Properties/AssemblyInfo.cs
View File

@ -28,7 +28,7 @@ using System.Runtime.InteropServices;
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyProduct("ProcessService")]
[assembly: AssemblyProduct("Dryad")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.3.0")]
[assembly: AssemblyFileVersion("0.1.3.0")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

2
ProcessService/app.config
View File

@ -19,7 +19,7 @@
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />

24
ProcessService/packages.config
View File

@ -1,21 +1,21 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages>

6
XmlDoc/Content/GettingStarted/Building the Job Browser.aml
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<topic id="91822db3-8a00-4307-ad8a-595c94f449b0" revisionNumber="1">
<developerHowToDocument
xmlns="http://ddue.schemas.microsoft.com/authoring/2003/5"
@ -36,8 +36,8 @@
<step>
<content>
<para>Clone the Dryad sources from <externalLink>
<linkText>https://github.com/MicrosoftResearchSVC/Dryad.git</linkText>
<linkUri>https://github.com/MicrosoftResearchSVC/Dryad.git</linkUri>
<linkText>https://github.com/MicrosoftResearch/Dryad.git</linkText>
<linkUri>https://github.com/MicrosoftResearch/Dryad.git</linkUri>
<linkTarget>_blank</linkTarget>
</externalLink>.</para>
</content>

8
XmlDoc/Content/Resources/Running a job on HDInsight.aml
View File

@ -122,11 +122,15 @@
<linkTarget>_blank</linkTarget>
</externalLink>, and click on the <command>Hadoop YARN Status</command> shortcut link on the desktop, you can see all these
jobs running.</para>
<para>Unfortunately because of the current configuration of HDInsight clusters, all DryadLINQ logs are deleted immediately
<para>Unfortunately because of the current configuration of HDInsight clusters, all DryadLINQ logs are archived immediately
when the application exits, and you will get a "Failed redirect for container" error if you try to navigate to the logs of
a completed application. We have tried to report errors in user application code back so that they are visible in the
<link xlink:href="91822db3-8a00-4307-ad8a-595c94f449b0">DryadLINQ Job Browser</link> to avoid the need to consult
the logs.</para>
the logs. If you do need to consult the logs, remote desktop to your HDInsight cluster, then click on the
<command>Hadoop Command Line</command> link on the desktop, and then run a command similar to
<command>yarn logs -applicationId &lt;APPLICATION_ID&gt; -appOwner &lt;CLUSTER_USER_NAME&gt;</command> where you replace
&lt;APPLICATION_ID&gt; and &lt;CLUSTER_USER_NAME&gt; with values appropriate to your job and cluster configuration.
</para>
<para>
<mediaLinkInline>
<image xlink:href="Dryad on Azure Architecture"/>

4
XmlDoc/Content/Welcome.aml
View File

@ -15,7 +15,7 @@
<para>Dryad, DryadLINQ, and the JobBrowser have been released as open source under the Apache 2.0 license. The
source is hosted at the <externalLink>
<linkText>Dryad GitHub repository</linkText>
<linkUri>https://github.com/MicrosoftResearchSVC/Dryad</linkUri>
<linkUri>https://github.com/MicrosoftResearch/Dryad</linkUri>
<linkTarget>_blank</linkTarget>
</externalLink>. There are also <externalLink>
<linkText>Dryad NuGet packages</linkText>
@ -39,7 +39,7 @@
<para>See details on <link xlink:href="91822db3-8a00-4307-ad8a-595c94f449b0">Building the job browser</link>. More documentation
is listed in the <externalLink>
<linkText>DryadLINQ job browser manual</linkText>
<linkUri>https://github.com/MicrosoftResearchSVC/Dryad/wiki/Using%20the%20DryadLINQ%20job%20browser</linkUri>
<linkUri>https://github.com/MicrosoftResearch/Dryad/wiki/Using%20the%20DryadLINQ%20job%20browser</linkUri>
<linkTarget>_blank</linkTarget>
</externalLink>.</para>
</content>

4
samples/Microsoft.Research.DryadLinq.Samples.nuspec
View File

@ -2,7 +2,7 @@
<package xmlns="http://schemas.microsoft.com/packaging/2013/01/nuspec.xsd">
<metadata minClientVersion="2.5">
<id>Microsoft.Research.DryadLinq.Samples</id>
<version>0.1.3-beta</version>
<version>0.1.5-beta</version>
<title>Sample DryadLINQ Programs</title>
<authors>bigdatadev@microsoft.com</authors>
<owners>bigdatadev@microsoft.com,Microsoft</owners>
@ -14,7 +14,7 @@
<language>en-US</language>
<dependencies>
<group targetFramework=".NETFramework4.5">
<dependency id="Microsoft.Research.Dryad" version="0.1.3-beta" />
<dependency id="Microsoft.Research.Dryad" version="0.1.5-beta" />
</group>
</dependencies>
<frameworkAssemblies>