berkus
/
Dryad
mirror of https://github.com/MicrosoftResearch/Dryad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: berkus:v0.1.3
Branches Tags
berkus:master
berkus:gh-pages
berkus:Release_0.2
berkus:v0.2.1-fixed
berkus:v0.2.1
berkus:v0.2.0
berkus:v0.1.5
berkus:v0.1.3
berkus:v0.1.2
berkus:v0.1.1
berkus:v0.1.0
...
compare: berkus:master
Branches Tags
berkus:gh-pages
berkus:master
berkus:Release_0.2
berkus:v0.2.1-fixed
berkus:v0.2.1
berkus:v0.2.0
berkus:v0.1.5
berkus:v0.1.3
berkus:v0.1.2
berkus:v0.1.1
berkus:v0.1.0

11 Commits
v0.1.3 ... master

Author SHA1 Message Date
misard ad28579dd8 Merge branch 'Release_0.2' 2014-10-23 10:32:28 -07:00
misard 6980be9895 Added missing documentation file 2014-10-23 10:29:37 -07:00
misard e72fef34c6 Merge branch 'Release_0.2' 2014-10-23 10:08:51 -07:00
Michael Isard 04ecff4353 Updated README 2014-10-23 09:50:48 -07:00
misard 5e9dde8ab6 Merge branch 'Release_0.2' 2014-10-06 19:26:04 -07:00
Michael Isard 47d0131083 updated versions 2014-10-06 19:13:07 -07:00
misard b31fadd55e updated with fix for azure subscription lookup 2014-10-06 19:13:06 -07:00
misard d3643fc5fb Merge branch 'Release_0.2' 2014-09-30 16:16:12 -07:00
misard efc79b749b Release 0.2 edits 2014-09-30 16:15:56 -07:00
Michael Isard b9cda36f69 release 0.2.0 2014-09-30 15:03:43 -07:00
Dennis Fetterly 5c260ffe0b version 0.1.5 release 2014-06-10 16:11:25 -07:00
174 changed files with 12127 additions and 4810 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -19,3 +19,5 @@ x64/
build/ build/
[Bb]in/ [Bb]in/
[Oo]bj/ [Oo]bj/
/Calypso/packages
/DryadLinqTests/packages

112
ClusterInterface/ClusterInterface.csproj
View File

@ -1,6 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -9,10 +10,11 @@
<OutputType>Library</OutputType> <OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Dryad.ClusterInterface</RootNamespace> <RootNamespace>Microsoft.Research.Dryad.ClusterInterface</RootNamespace>
<AssemblyName>DryadHttpClusterInterface</AssemblyName> <AssemblyName>Microsoft.Research.Dryad.HttpClusterInterface</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<TargetFrameworkProfile /> <TargetFrameworkProfile />
<NuGetPackageImportStamp>0025dfe5</NuGetPackageImportStamp>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
@ -33,93 +35,12 @@
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" /> <Reference Include="System" />
<Reference Include="System.Core" /> <Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" /> <Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" /> <Reference Include="System.Net" />
<Reference Include="System.Net.Http" /> <Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" /> <Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" /> <Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" /> <Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" /> <Reference Include="System.Data.DataSetExtensions" />
@ -128,29 +49,34 @@
<Reference Include="System.Xml" /> <Reference Include="System.Xml" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Compile Include="..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="Interfaces.cs" /> <Compile Include="Interfaces.cs" />
<Compile Include="HttpScheduler.cs" /> <Compile Include="HttpScheduler.cs" />
<Compile Include="Process.cs" /> <Compile Include="Process.cs" />
<Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="app.config" /> <None Include="app.config">
<None Include="packages.config" /> <SubType>Designer</SubType>
</None>
<None Include="packages.config">
<SubType>Designer</SubType>
</None>
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
</Target> </Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild"> <Target Name="BeforeBuild">

4
ClusterInterface/HttpScheduler.cs
View File

@ -90,10 +90,10 @@ namespace Microsoft.Research.Dryad.ClusterInterface
return fileServer.Uri.AbsoluteUri; return fileServer.Uri.AbsoluteUri;
} }
public IProcess NewProcess(IProcessWatcher watcher, string commandLine) public IProcess NewProcess(IProcessWatcher watcher, string commandLine, string commandLineArguments)
{ {
ISchedulerProcess process = scheduler.NewProcess(); ISchedulerProcess process = scheduler.NewProcess();
return new Process(process, watcher, commandLine, logger); return new Process(process, watcher, commandLine, commandLineArguments, logger);
} }
public void ScheduleProcess(IProcess ip, List<Affinity> affinities) public void ScheduleProcess(IProcess ip, List<Affinity> affinities)

3
ClusterInterface/Interfaces.cs
View File

@ -368,8 +368,9 @@ namespace Microsoft.Research.Dryad.ClusterInterface
/// the application on the process' lifecycle</param> /// the application on the process' lifecycle</param>
/// <param name="commandLine">the command line to use to start the process on the remote /// <param name="commandLine">the command line to use to start the process on the remote
/// computer</param> /// computer</param>
/// <param name="commandLineArguments">arguments to provide to the remote process</param>
/// <returns>a handle to the new process</returns> /// <returns>a handle to the new process</returns>
IProcess NewProcess(IProcessWatcher watcher, string commandLine); IProcess NewProcess(IProcessWatcher watcher, string commandLine, string commandLineArguments);
/// <summary> /// <summary>
/// request that a Process object, return from NewProcess, be scheduled according /// request that a Process object, return from NewProcess, be scheduled according

23
ClusterInterface/Process.cs
View File

@ -56,11 +56,6 @@ namespace Microsoft.Research.Dryad.ClusterInterface
/// </summary> /// </summary>
private State state; private State state;
/// <summary>
/// string used to start the remote process
/// </summary>
private string commandLine;
/// <summary> /// <summary>
/// this is the handle that the scheduler supplies that is used to refer /// this is the handle that the scheduler supplies that is used to refer
/// to the process /// to the process
@ -114,11 +109,12 @@ namespace Microsoft.Research.Dryad.ClusterInterface
/// <summary> /// <summary>
/// construct a new object to represent the lifecycle of a process being scheduled /// construct a new object to represent the lifecycle of a process being scheduled
/// </summary> /// </summary>
public Process(ISchedulerProcess p, IProcessWatcher w, string cmd, ILogger l) public Process(ISchedulerProcess p, IProcessWatcher w, string cmd, string cmdLineArgs, ILogger l)
{ {
schedulerProcess = p; schedulerProcess = p;
state = State.Initializing; state = State.Initializing;
commandLine = cmd; CommandLine = cmd;
CommandLineArguments = cmdLineArgs;
watcher = w; watcher = w;
cancelTask = new TaskCompletionSource<bool>(); cancelTask = new TaskCompletionSource<bool>();
directory = null; directory = null;
@ -137,7 +133,12 @@ namespace Microsoft.Research.Dryad.ClusterInterface
/// <summary> /// <summary>
/// the string used to start the remote process /// the string used to start the remote process
/// </summary> /// </summary>
public string CommandLine { get { return commandLine; } } public string CommandLine { get; private set; }
/// <summary>
/// arguments provided when starting the remote process
/// </summary>
public string CommandLineArguments { get; private set; }
/// <summary> /// <summary>
/// the local directory of the process at the daemon's host computer /// the local directory of the process at the daemon's host computer
@ -198,9 +199,13 @@ namespace Microsoft.Research.Dryad.ClusterInterface
logger.Log("Process " + Id + " scheduling itself as " + directory + " on computer " + computer.Name + " at " + computer.Host); logger.Log("Process " + Id + " scheduling itself as " + directory + " on computer " + computer.Name + " at " + computer.Host);
ToMatched(computer, DateTime.Now.ToFileTimeUtc()); ToMatched(computer, DateTime.Now.ToFileTimeUtc());
StringBuilder payload = new StringBuilder();
payload.AppendLine(CommandLine);
payload.AppendLine(CommandLineArguments);
Task<string> bail = interrupt.ContinueWith((t) => ""); Task<string> bail = interrupt.ContinueWith((t) => "");
Task<string> upload = PostRequest(computer, directory + "?op=create", Encoding.UTF8.GetBytes(commandLine)); Task<string> upload = PostRequest(computer, directory + "?op=create", Encoding.UTF8.GetBytes(payload.ToString()));
Task<string> completed = await Task.WhenAny(bail, upload); Task<string> completed = await Task.WhenAny(bail, upload);
if (completed == bail) if (completed == bail)

27
ClusterInterface/Properties/AssemblyInfo.cs
View File

@ -24,32 +24,7 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following // General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information // set of attributes. Change these attribute values to modify the information
// associated with an assembly. // associated with an assembly.
[assembly: AssemblyTitle("DryadHttpClusterInterface")] [assembly: AssemblyTitle("Microsoft.Research.Dryad.HttpClusterInterface")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyProduct("DryadHttpClusterInterface")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM // The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("9d262d5f-f6c5-4f9c-a1cc-7568cb399bf9")] [assembly: Guid("9d262d5f-f6c5-4f9c-a1cc-7568cb399bf9")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.3.0")]
[assembly: AssemblyFileVersion("0.1.3.0")]

24
ClusterInterface/app.config
View File

@ -4,24 +4,40 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" /> <bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" /> <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>
</configuration> </configuration>

20
ClusterInterface/packages.config
View File

@ -1,21 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
</packages> </packages>

30
Dryad.sln
View File

@ -1,6 +1,8 @@
 
Microsoft Visual Studio Solution File, Format Version 12.00 Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2012 # Visual Studio 2013
VisualStudioVersion = 12.0.30723.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DryadVertex", "DryadVertex", "{99F5E7FE-ADD4-4E9D-BA90-5D3D3409BEB3}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DryadVertex", "DryadVertex", "{99F5E7FE-ADD4-4E9D-BA90-5D3D3409BEB3}"
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "VertexHost", "VertexHost", "{37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "VertexHost", "VertexHost", "{37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D}"
@ -27,7 +29,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dprocess", "DryadVertex\Ver
EndProject EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ManagedWrapperVertex", "DryadVertex\VertexHost\vertex\managedwrappervertex\ManagedWrapperVertex.vcxproj", "{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}" Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ManagedWrapperVertex", "DryadVertex\VertexHost\vertex\managedwrappervertex\ManagedWrapperVertex.vcxproj", "{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}"
EndProject EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHost", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}" Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHostBody", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}"
ProjectSection(ProjectDependencies) = postProject ProjectSection(ProjectDependencies) = postProject
{AA529122-F51C-48D7-A8C1-C0B24F570885} = {AA529122-F51C-48D7-A8C1-C0B24F570885} {AA529122-F51C-48D7-A8C1-C0B24F570885} = {AA529122-F51C-48D7-A8C1-C0B24F570885}
{482E0741-E244-4974-97D4-3A7167581E91} = {482E0741-E244-4974-97D4-3A7167581E91} {482E0741-E244-4974-97D4-3A7167581E91} = {482E0741-E244-4974-97D4-3A7167581E91}
@ -78,7 +80,18 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalScheduler", "LocalSche
{A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6} = {A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6} {A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6} = {A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6}
EndProjectSection EndProjectSection
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadLinqTests", "DryadLinqTests\DryadLinqTests.csproj", "{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VertexHostMain", "DryadVertex\VertexHost\vertex\VertexHostMain\VertexHostMain.csproj", "{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "NuSpec", "NuSpec", "{35121ABA-5FD2-4E1D-9C05-033D78D824F3}"
ProjectSection(SolutionItems) = preProject
Microsoft.Research.Dryad.nuspec = Microsoft.Research.Dryad.nuspec
Microsoft.Research.Dryad.targets = Microsoft.Research.Dryad.targets
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Misc", "Misc", "{D361F025-2F7F-4C1A-8FBC-4AD826CC656D}"
ProjectSection(SolutionItems) = preProject
SharedAssemblyInfo.cs = SharedAssemblyInfo.cs
EndProjectSection
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -146,10 +159,10 @@ Global
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Debug|x64.Build.0 = Debug|x64 {FA798DE2-A442-4D2B-B31A-FA9274308290}.Debug|x64.Build.0 = Debug|x64
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.ActiveCfg = Release|x64 {FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.ActiveCfg = Release|x64
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.Build.0 = Release|x64 {FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.Build.0 = Release|x64
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Debug|x64.ActiveCfg = Debug|x64 {DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Debug|x64.ActiveCfg = Debug|x64
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Debug|x64.Build.0 = Debug|x64 {DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Debug|x64.Build.0 = Debug|x64
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Release|x64.ActiveCfg = Release|x64 {DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Release|x64.ActiveCfg = Release|x64
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Release|x64.Build.0 = Release|x64 {DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Release|x64.Build.0 = Release|x64
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE
@ -162,11 +175,12 @@ Global
{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE} {016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{57663B94-E11B-431E-BE4B-E2C61112DEC5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE} {57663B94-E11B-431E-BE4B-E2C61112DEC5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{AA529122-F51C-48D7-A8C1-C0B24F570885} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE} {AA529122-F51C-48D7-A8C1-C0B24F570885} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{A0033286-9C5F-4113-BAA5-58A1274F95C5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{BDEDD3BB-C7E2-498F-A212-F99786C8E23C} = {DC277807-506B-4F7C-BECD-345079B91044} {BDEDD3BB-C7E2-498F-A212-F99786C8E23C} = {DC277807-506B-4F7C-BECD-345079B91044}
{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37} = {DC277807-506B-4F7C-BECD-345079B91044} {0CF3D1D5-9BBE-4175-979B-EC6138EF4F37} = {DC277807-506B-4F7C-BECD-345079B91044}
{AB9EA66C-5811-49A7-B002-24203AEB9083} = {DC277807-506B-4F7C-BECD-345079B91044} {AB9EA66C-5811-49A7-B002-24203AEB9083} = {DC277807-506B-4F7C-BECD-345079B91044}
{3EE0920C-0607-4569-9EC3-5C12BB6EF244} = {DC277807-506B-4F7C-BECD-345079B91044} {3EE0920C-0607-4569-9EC3-5C12BB6EF244} = {DC277807-506B-4F7C-BECD-345079B91044}
{89C5654B-02E4-478D-A7E6-50D79F638B4F} = {DC277807-506B-4F7C-BECD-345079B91044} {89C5654B-02E4-478D-A7E6-50D79F638B4F} = {DC277807-506B-4F7C-BECD-345079B91044}
{A0033286-9C5F-4113-BAA5-58A1274F95C5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352} = {DC277807-506B-4F7C-BECD-345079B91044}
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal

113
DryadLinqGraphManager/DryadLinqGraphManager.csproj
View File

@ -1,6 +1,9 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -10,7 +13,7 @@
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Dryad.GraphManager</RootNamespace> <RootNamespace>Microsoft.Research.Dryad.GraphManager</RootNamespace>
<AssemblyName>DryadLinqGraphManager</AssemblyName> <AssemblyName>Microsoft.Research.Dryad.GraphManager</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<FileUpgradeFlags> <FileUpgradeFlags>
@ -33,6 +36,7 @@
<ApplicationVersion>1.0.0.%2a</ApplicationVersion> <ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<UseApplicationTrust>false</UseApplicationTrust> <UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled> <BootstrapperEnabled>true</BootstrapperEnabled>
<NuGetPackageImportStamp>5c14de70</NuGetPackageImportStamp>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
@ -53,80 +57,28 @@
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath> <HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath> <HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath> <HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath> <HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath> <HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference> </Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL"> <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath> <HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference> </Reference>
<Reference Include="System" /> <Reference Include="System" />
<Reference Include="System.Core"> <Reference Include="System.Core">
@ -138,18 +90,10 @@
<Reference Include="System.Data.Services.Client" /> <Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" /> <Reference Include="System.Net" />
<Reference Include="System.Net.Http" /> <Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" /> <Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath> <HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference> </Reference>
<Reference Include="System.Web" /> <Reference Include="System.Web" />
<Reference Include="System.Xml.Linq"> <Reference Include="System.Xml.Linq">
@ -162,6 +106,9 @@
<Reference Include="System.Xml" /> <Reference Include="System.Xml" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Compile Include="..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="DryadLinqApplication.cs" /> <Compile Include="DryadLinqApplication.cs" />
<Compile Include="GraphBuilder.cs" /> <Compile Include="GraphBuilder.cs" />
<Compile Include="LinqToDryadException.cs" /> <Compile Include="LinqToDryadException.cs" />
@ -203,19 +150,23 @@
</ProjectReference> </ProjectReference>
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
</Target> </Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild"> <Target Name="BeforeBuild">

16
DryadLinqGraphManager/GraphBuilder.cs
View File

@ -92,7 +92,7 @@ namespace Microsoft.Research.Dryad.GraphManager
err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath); err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath);
if (!SUCCEEDED(err)) if (!SUCCEEDED(err))
{ {
string msg = String.Format("Could not read DSC input file {0}", info.sources[0]); string msg = String.Format("Could not read Partitioned file input {0}", info.sources[0]);
throw new LinqToDryadException(msg, err); throw new LinqToDryadException(msg, err);
} }
@ -127,7 +127,9 @@ namespace Microsoft.Research.Dryad.GraphManager
DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset"); DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");
err = input.Open(app.GetUniverse(), info.sources[0]); Uri srcUri = new Uri(info.sources[0]);
err = input.Open(app.GetUniverse(), srcUri.GetLeftPart(UriPartial.Path), info.recordType);
if (!SUCCEEDED(err)) if (!SUCCEEDED(err))
{ {
string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError()); string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError());
@ -182,10 +184,16 @@ namespace Microsoft.Research.Dryad.GraphManager
if ( info.ioType == VertexInfo.IOType.PARTITIONEDFILE ) if ( info.ioType == VertexInfo.IOType.PARTITIONEDFILE )
{ {
DrPartitionOutputStream output = new DrPartitionOutputStream(); DrPartitionOutputStream output = new DrPartitionOutputStream();
int err = output.Open(new Uri(info.sources[0]).AbsolutePath, info.partitionUncPath); Uri sourceUri = new Uri(info.sources[0]);
string sourcePath = sourceUri.AbsolutePath;
if (!String.IsNullOrEmpty(sourceUri.Host))
{
sourcePath = @"\\" + sourceUri.Host + sourcePath;
}
int err = output.Open(sourcePath, info.partitionUncPath);
if (!SUCCEEDED(err)) if (!SUCCEEDED(err))
{ {
string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]); string msg = String.Format("Could not open output fileset {0}", sourcePath);
throw new LinqToDryadException(msg, err); throw new LinqToDryadException(msg, err);
} }

60
DryadLinqGraphManager/LinqToDryadJM.cs
View File

@ -73,12 +73,12 @@ namespace Microsoft.Research.Dryad.GraphManager
public static Uri AddDfsJobDirectoryFromArgs(string arg) public static Uri AddDfsJobDirectoryFromArgs(string arg)
{ {
string jobId = Microsoft.Research.Peloponnese.Storage.AzureUtils.ApplicationIdFromEnvironment(); string jobId = Microsoft.Research.Peloponnese.Yarn.Utils.ApplicationIdFromEnvironment();
string jobDirectoryBase = Microsoft.Research.Peloponnese.Storage.AzureUtils.CmdLineDecode(arg.Substring(arg.IndexOf('=') + 1)); string jobDirectoryBase = Microsoft.Research.Peloponnese.Utils.CmdLineDecode(arg.Substring(arg.IndexOf('=') + 1));
string jobDirectorySpecific = jobDirectoryBase.Replace("_JOBID_", jobId); string jobDirectorySpecific = jobDirectoryBase.Replace("_JOBID_", jobId);
UriBuilder logLocation = new UriBuilder(jobDirectorySpecific); UriBuilder logLocation = new UriBuilder(jobDirectorySpecific);
logLocation.Path = logLocation.Path + "calypso.log"; logLocation.Path = logLocation.Path.TrimEnd('/') + "/calypso.log";
DebugHelper.AddReporter(new DrCalypsoReporter(logLocation.Uri.AbsoluteUri)); DebugHelper.AddReporter(new DrCalypsoReporter(logLocation.Uri.AbsoluteUri));
@ -94,7 +94,7 @@ namespace Microsoft.Research.Dryad.GraphManager
{ {
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory)) using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
{ {
string dfsPath = dfsDirectory.AbsolutePath + Path.GetFileName(localPath); string dfsPath = dfsDirectory.AbsolutePath.TrimEnd('/') + "/" + Path.GetFileName(localPath);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath); DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
hdfs.UploadAll(localPath, dfsPath); hdfs.UploadAll(localPath, dfsPath);
} }
@ -102,11 +102,12 @@ namespace Microsoft.Research.Dryad.GraphManager
else if (dfsDirectory.Scheme == "azureblob") else if (dfsDirectory.Scheme == "azureblob")
{ {
string account, key, container, blob; string account, key, container, blob;
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob); Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container); var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
string dfsPath = blob + Path.GetFileName(localPath); string dfsPath = blob.TrimEnd('/') + "/" + Path.GetFileName(localPath);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath); Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
azure.PutDfsFile(localPath, dfsPath); DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsUri.AbsoluteUri);
azure.PutDfsFile(dfsUri, localPath);
} }
else if (dfsDirectory.Scheme == "file") else if (dfsDirectory.Scheme == "file")
{ {
@ -138,10 +139,11 @@ namespace Microsoft.Research.Dryad.GraphManager
else if (dfsDirectory.Scheme == "azureblob") else if (dfsDirectory.Scheme == "azureblob")
{ {
string account, key, container, blob; string account, key, container, blob;
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob); Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container); var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
string dfsPath = blob + dfsName; string dfsPath = blob + dfsName;
azure.PutDfsFile(payloadBytes, dfsPath); Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
azure.PutDfsFile(dfsUri, payloadBytes);
} }
else if (dfsDirectory.Scheme == "file") else if (dfsDirectory.Scheme == "file")
{ {
@ -207,7 +209,39 @@ namespace Microsoft.Research.Dryad.GraphManager
logDir = logDir.Split(',').First().Trim(); logDir = logDir.Split(',').First().Trim();
DrLogging.Initialize(Path.Combine(logDir, "graphmanager"), false); DrLogging.Initialize(Path.Combine(logDir, "graphmanager"), false);
} }
DrLogging.SetLoggingLevel(DrLogTypeManaged.Info);
string logLevel = Environment.GetEnvironmentVariable("DRYAD_LOGGING_LEVEL");
if (logLevel == null)
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Debug);
}
else
{
if (logLevel == "OFF")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Off);
}
else if (logLevel == "CRITICAL")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Assert);
}
else if (logLevel == "ERROR")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Error);
}
else if (logLevel == "WARN")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Warning);
}
else if (logLevel == "INFO")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Info);
}
else
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Debug);
}
}
// Report start time to Artemis - must come after // Report start time to Artemis - must come after
// DrLogging is initialized so stdout is redirected // DrLogging is initialized so stdout is redirected

7
DryadLinqGraphManager/Program.cs
View File

@ -28,6 +28,13 @@ using Microsoft.Research.Dryad;
namespace Microsoft.Research.Dryad.GraphManager namespace Microsoft.Research.Dryad.GraphManager
{ {
internal class ConfigDependency : Microsoft.Research.Peloponnese.Shared.AssemblyDependencyAttribute
{
public ConfigDependency() : base("Microsoft.Research.Dryad.GraphManager.exe.config", false)
{
}
}
class Program class Program
{ {
static Uri dfsDirectory; static Uri dfsDirectory;

28
DryadLinqGraphManager/Properties/AssemblyInfo.cs
View File

@ -22,34 +22,12 @@ using System.Reflection;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
[assembly: Microsoft.Research.Dryad.GraphManager.ConfigDependency()]
// General Information about an assembly is controlled through the following // General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information // set of attributes. Change these attribute values to modify the information
// associated with an assembly. // associated with an assembly.
[assembly: AssemblyTitle("DryadLinqGraphManager")] [assembly: AssemblyTitle("Microsoft.Research.Dryad.GraphManager")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyProduct("DryadLinqGraphManager")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM // The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("55c7f4b4-02ab-4309-b276-3c9ab5a927e0")] [assembly: Guid("55c7f4b4-02ab-4309-b276-3c9ab5a927e0")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.3.0")]
[assembly: AssemblyFileVersion("0.1.3.0")]

6
DryadLinqGraphManager/QueryParser.cs
View File

@ -237,6 +237,12 @@ namespace Microsoft.Research.Dryad.GraphManager
vertex.info = new VertexInfo(); vertex.info = new VertexInfo();
vertex.info.ioType = GetIoType(ioType); vertex.info.ioType = GetIoType(ioType);
XmlNode recordType = storageSet.SelectSingleNode("RecordType");
if (recordType != null)
{
vertex.info.recordType = recordType.InnerXml;
}
XmlNodeList storageUris = storageSet.SelectNodes("SourceURI"); XmlNodeList storageUris = storageSet.SelectNodes("SourceURI");
vertex.info.sources = new string[storageUris.Count]; vertex.info.sources = new string[storageUris.Count];
for (int indexStorageUri=0; indexStorageUri<storageUris.Count; indexStorageUri++) for (int indexStorageUri=0; indexStorageUri<storageUris.Count; indexStorageUri++)

24
DryadLinqGraphManager/app.config
View File

@ -7,24 +7,40 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" /> <bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" /> <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>
</configuration> </configuration>

27
DryadLinqGraphManager/packages.config
View File

@ -1,21 +1,14 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" /> <package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" /> <package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" /> <package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" /> <package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" /> <package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" /> <package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
</packages> </packages>

24
DryadLinqTests/App.config
View File

@ -5,6 +5,14 @@
</startup> </startup>
<runtime> <runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
@ -18,12 +26,20 @@
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" /> <bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" /> <bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly> </dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>

391
DryadLinqTests/ApplyAndForkTests.cs
View File

@ -1,13 +1,59 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text.RegularExpressions;
namespace DryadLinqTests namespace DryadLinqTests
{ {
public class ApplyAndForkTests public class ApplyAndForkTests
{ {
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" ApplyAndForkTests ");
TestLog.Message(" **********************");
var tests = new Dictionary<string, Action>()
{
{"NonHomomorphicUnaryApply", () => NonHomomorphicUnaryApply(context) },
{"HomomorphicUnaryApply", () => HomomorphicUnaryApply(context) },
{"NonHomomorphicBinaryApply", () => NonHomomorphicBinaryApply(context) },
{"LeftHomomorphicBinaryApply", () => LeftHomomorphicBinaryApply(context) },
{"FullHomomorphicBinaryApply_DifferentDataSets", () => FullHomomorphicBinaryApply_DifferentDataSets(context) },
{"FullHomomorphicBinaryApply_IdenticalDataSets", () => FullHomomorphicBinaryApply_IdenticalDataSets(context) },
{"Aggregate_WithCombiner", () => Aggregate_WithCombiner(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static IEnumerable<int> NonHomomorphic_Unary_Func(IEnumerable<int> input) public static IEnumerable<int> NonHomomorphic_Unary_Func(IEnumerable<int> input)
{ {
return input; return input;
@ -57,177 +103,328 @@ namespace DryadLinqTests
throw new Exception("a node received empty right-data"); throw new Exception("a node received empty right-data");
} }
public static bool NonHomomorphicUnaryApply(DryadLinqContext context)
public static bool NonHomomorphicUnaryApply()
{ {
var context = new DryadLinqContext(Config.cluster); string testName = "NonHomomorphicUnaryApply";
context.LocalExecution = false; TestLog.TestStart(testName);
bool passed = true; bool passed = true;
try try
{ {
string outFile = "unittest/output/NonHomomorphicUnaryApply.out"; IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, // cluster
"unittest/inputdata/SimpleFile.txt")); {
context.LocalDebug = false;
string outFile = "unittest/output/NonHomomorphicUnaryApply";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt1 = simple.Select(x => x.First()); var q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x)); passed &= Validate.outFileExists(outFile);
var jobInfo = q1.ToStore<int>(outFile).Submit(); result[0] = q1;
jobInfo.Wait(); }
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile); // local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
} }
catch (DryadLinqException) catch (Exception Ex)
{ {
TestLog.Message("Error: " + Ex.Message);
passed &= false; passed &= false;
} }
TestLog.LogResult(new TestResult(testName, context, passed));
return passed; return passed;
} }
public static bool HomomorphicUnaryApply() public static bool HomomorphicUnaryApply(DryadLinqContext context)
{ {
var context = new DryadLinqContext(Config.cluster); string testName = "HomomorphicUnaryApply";
context.LocalExecution = false; TestLog.TestStart(testName);
bool passed = true; bool passed = true;
try try
{ {
string outFile = "unittest/output/HomomorphicUnaryApply.out"; IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, // cluster
"unittest/inputdata/SimpleFile.txt")); {
context.LocalDebug = false;
string outFile = "unittest/output/HomomorphicUnaryApply";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt1 = simple.Select(x => x.First()); IQueryable<int> q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x)); passed &= Validate.outFileExists(outFile);
var jobInfo = q1.ToStore<int>(outFile).Submit(); result[0] = q1;
jobInfo.Wait(); }
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile); // local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
} }
catch (DryadLinqException) catch (Exception Ex)
{ {
TestLog.Message("Error: " + Ex.Message);
passed &= false; passed &= false;
} }
TestLog.LogResult(new TestResult(testName, context, passed));
return passed; return passed;
} }
public static bool NonHomomorphicBinaryApply() public static bool NonHomomorphicBinaryApply(DryadLinqContext context)
{ {
var context = new DryadLinqContext(Config.cluster); string testName = "NonHomomorphicBinaryApply";
context.LocalExecution = false; TestLog.TestStart(testName);
bool passed = true; bool passed = true;
try try
{ {
string outFile = "unittest/output/NonHomomorphicBinaryApply.out"; IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, // cluster
"unittest/inputdata/SimpleFile.txt")); {
context.LocalDebug = false;
string outFile = "unittest/output/NonHomomorphicBinaryApply";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt1 = simple.Select(x => x.First()); IQueryable<int> q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y)); passed &= Validate.outFileExists(outFile);
var jobInfo = q1.ToStore<int>(outFile).Submit(); result[0] = q1;
jobInfo.Wait(); }
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile); // local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
} }
catch (DryadLinqException) catch (Exception Ex)
{ {
TestLog.Message("Error: " + Ex.Message);
passed &= false; passed &= false;
} }
TestLog.LogResult(new TestResult(testName, context, passed));
return passed; return passed;
} }
public static bool LeftHomomorphicBinaryApply() public static bool LeftHomomorphicBinaryApply(DryadLinqContext context)
{ {
var context = new DryadLinqContext(Config.cluster); string testName = "LeftHomomorphicBinaryApply";
context.LocalExecution = false; TestLog.TestStart(testName);
bool passed = true; bool passed = true;
try try
{ {
string outFile = "unittest/output/LeftHomomorphicBinaryApply.out"; IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, // cluster
"unittest/inputdata/SimpleFile.txt")); {
context.LocalDebug = false;
string outFile = "unittest/output/LeftHomomorphicBinaryApply";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt1 = simple.Select(x => x.First()); IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true); passed &= Validate.outFileExists(outFile);
var jobInfo = q1.ToStore<int>(outFile).Submit(); result[0] = q1;
jobInfo.Wait(); }
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile); // local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
} }
catch (DryadLinqException) catch (Exception Ex)
{ {
TestLog.Message("Error: " + Ex.Message);
passed &= false; passed &= false;
} }
TestLog.LogResult(new TestResult(testName, context, passed));
return passed; return passed;
} }
public static bool FullHomomorphicBinaryApply_DifferentDataSets() public static bool FullHomomorphicBinaryApply_DifferentDataSets(DryadLinqContext context)
{ {
var context = new DryadLinqContext(Config.cluster); string testName = "FullHomomorphicBinaryApply_DifferentDataSets";
context.LocalExecution = false; TestLog.TestStart(testName);
bool passed = true; bool passed = true;
try try
{ {
string outFile = "unittest/output/FullHomomorphicBinaryApply_DifferentDataSets.out"; IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, // cluster
"unittest/inputdata/SimpleFile.txt")); {
context.LocalDebug = false;
string outFile = "unittest/output/FullHomomorphicBinaryApply_DifferentDataSets";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt1 = simple.Select(x => x.First()); IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt2 = simple.Select(x => x.First()); IQueryable<int> q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false); passed &= Validate.outFileExists(outFile);
var jobInfo = q1.ToStore<int>(outFile).Submit(); result[0] = q1;
jobInfo.Wait(); }
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile); // local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
} }
catch (DryadLinqException) catch (Exception Ex)
{ {
TestLog.Message("Error: " + Ex.Message);
passed &= false; passed &= false;
} }
TestLog.LogResult(new TestResult(testName, context, passed));
return passed; return passed;
} }
public static bool FullHomomorphicBinaryApply_IdenticalDataSets() public static bool FullHomomorphicBinaryApply_IdenticalDataSets(DryadLinqContext context)
{ {
var context = new DryadLinqContext(Config.cluster); string testName = "FullHomomorphicBinaryApply_IdenticalDataSets";
context.LocalExecution = false; TestLog.TestStart(testName);
bool passed = true; bool passed = true;
try try
{ {
string outFile = "unittest/output/FullHomomorphicBinaryApply_2.out"; IEnumerable<int>[] result = new IEnumerable<int>[2];
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, // cluster
"unittest/inputdata/SimpleFile.txt")); {
context.LocalDebug = false;
string outFile = "unittest/output/FullHomomorphicBinaryApply_2";
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt1 = simple.Select(x => x.First()); IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
var q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false); var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
var jobInfo = q1.ToStore<int>(outFile).Submit(); jobInfo.Wait();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile); passed &= Validate.outFileExists(outFile);
result[0] = q1;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
} }
catch (DryadLinqException) catch (Exception Ex)
{ {
TestLog.Message("Error: " + Ex.Message);
passed &= false; passed &= false;
} }
TestLog.LogResult(new TestResult(testName, context, passed));
return passed; return passed;
} }
[Associative(typeof(AssociativeRecursive1))] [Associative(typeof(AssociativeRecursive1))]
public static string IntToStringCSVAggregator(string agg, int next) public static string IntToStringCSVAggregator(string agg, int next)
{ {
return agg + "," + next.ToString(); return agg + "," + next.ToString();
} }
@ -243,27 +440,29 @@ namespace DryadLinqTests
} }
} }
public static bool Aggregate_WithCombiner() public static bool Aggregate_WithCombiner(DryadLinqContext context)
{ {
var context = new DryadLinqContext(Config.cluster); string testName = "Aggregate_WithCombiner";
context.LocalExecution = false; TestLog.TestStart(testName);
bool passed = true; bool passed = true;
try try
{ {
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, // cluster
"unittest/inputdata/SimpleFile.txt")); {
context.LocalDebug = false;
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt1 = simple.Select(x => x.First()); string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x)); }
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
} }
catch (DryadLinqException) catch (Exception Ex)
{ {
TestLog.Message("Error: " + Ex.Message);
passed &= false; passed &= false;
} }
TestLog.LogResult(new TestResult(testName, context, passed));
return passed; return passed;
} }
} }

1553
DryadLinqTests/BasicAPITests.cs
View File

File diff suppressed because it is too large Load Diff

53
DryadLinqTests/DryadLinqFSharpTests/App.config Normal file
View File

@ -0,0 +1,53 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
</startup>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="FSharp.Core" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
<bindingRedirect oldVersion="4.0.0.0" newVersion="4.3.0.0" />
<bindingRedirect oldVersion="2.3.5.0" newVersion="4.3.0.0" />
<bindingRedirect oldVersion="2.0.0.0" newVersion="4.3.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

219
DryadLinqTests/DryadLinqFSharpTests/DryadLinqFSharpTests.fsproj Normal file
View File

@ -0,0 +1,219 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<SchemaVersion>2.0</SchemaVersion>
<ProjectGuid>d0153361-1b5d-4715-8ee3-aa824f89cbc3</ProjectGuid>
<OutputType>Exe</OutputType>
<RootNamespace>DryadLinqFSharpTests</RootNamespace>
<AssemblyName>DryadLinqFSharpTests</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<Name>DryadLinqFSharpTests</Name>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<Tailcalls>false</Tailcalls>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<WarningLevel>3</WarningLevel>
<PlatformTarget>AnyCPU</PlatformTarget>
<DocumentationFile>bin\Debug\DryadLinqFSharpTests.XML</DocumentationFile>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<Tailcalls>true</Tailcalls>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<WarningLevel>3</WarningLevel>
<PlatformTarget>AnyCPU</PlatformTarget>
<DocumentationFile>bin\Release\DryadLinqFSharpTests.XML</DocumentationFile>
<Prefer32Bit>true</Prefer32Bit>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Data.Edm">
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Data.OData">
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Data.Services.Client">
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Hadoop.Client">
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Msagl">
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Msagl.Drawing">
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Msagl.GraphViewerGdi">
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Research.DryadLinq">
<HintPath>..\packages\Microsoft.Research.Dryad.0.1.4-beta001\lib\net45\Microsoft.Research.DryadLinq.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge">
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils">
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage">
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="mscorlib" />
<Reference Include="FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Private>True</Private>
</Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="System.Data.Linq" />
<Reference Include="System.Drawing" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Numerics" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.ServiceModel" />
<Reference Include="System.ServiceModel.Discovery" />
<Reference Include="System.Spatial">
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.Xml" />
<Reference Include="System.Xml.Linq" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<Compile Include="Program.fs" />
<None Include="packages.config" />
</ItemGroup>
<PropertyGroup>
<MinimumVisualStudioVersion Condition="'$(MinimumVisualStudioVersion)' == ''">11</MinimumVisualStudioVersion>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<Tailcalls>false</Tailcalls>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<WarningLevel>3</WarningLevel>
<DocumentationFile>bin\Debug\DryadLinqFSharpTests.XML</DocumentationFile>
<Prefer32Bit>true</Prefer32Bit>
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<Tailcalls>true</Tailcalls>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<WarningLevel>3</WarningLevel>
<DocumentationFile>bin\Release\DryadLinqFSharpTests.XML</DocumentationFile>
<Prefer32Bit>true</Prefer32Bit>
<PlatformTarget>x64</PlatformTarget>
</PropertyGroup>
<Import Project="$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets" Condition=" Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets" Condition="Exists('..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

46
DryadLinqTests/DryadLinqFSharpTests/Program.fs Normal file
View File

@ -0,0 +1,46 @@
// Learn more about F# at http://fsharp.net
// See the 'F# Tutorial' project for more help.
open System
open System.Collections.Generic
open System.Linq
open System.IO
open Microsoft.Research.DryadLinq
let context = new DryadLinqContext(1)
let inputUri = @"partfile:///d:/DryadLinqTemp/PartFiles/foo.pt"
let input = context.FromStore<LineRecord>(inputUri)
type Pair = { word : string; count : int }
let Histogram (source : IQueryable<LineRecord>) (k : int) =
let words = source.SelectMany(fun (x : LineRecord) -> x.Line.Split(' ') |> Seq.ofArray)
let groups = words.GroupBy(fun x -> x)
let counts = groups.Select(fun (x : IGrouping<string, string>) -> { word = x.Key; count = x.Count() })
let ordered = counts.OrderByDescending(fun x -> x.count)
ordered.Take(k)
let Histogram1 (source : IQueryable<LineRecord>) (k : int) =
query {
for ws in source do
for w in ws.Line.Split(' ') do
groupBy w into g
select { word = g.Key; count = g.Count() } into winfo
sortByDescending winfo.count
take k
}
let Histogram2 =
query {
for ws in input do
for w in ws.Line.Split(' ') do
groupBy w into g
select { word = g.Key; count = g.Count() } into winfo
sortByDescending winfo.count
take 10
}
for x in Histogram1 input 10 do System.Console.WriteLine(x.word + " : " + x.count.ToString())
//let foo = Histogram input 10
//foo.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/xxx.pt").SubmitAndWait()

23
DryadLinqTests/DryadLinqFSharpTests/packages.config Normal file
View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Dryad" version="0.1.4-beta001" targetFramework="net45" />
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages>

136
DryadLinqTests/DryadLinqTests.csproj
View File

@ -5,7 +5,7 @@
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}</ProjectGuid> <ProjectGuid>{4EA15885-003F-4B4D-A7DB-C172C2F23161}</ProjectGuid>
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>DryadLinqTests</RootNamespace> <RootNamespace>DryadLinqTests</RootNamespace>
@ -30,6 +30,25 @@
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir> <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
<RestorePackages>true</RestorePackages> <RestorePackages>true</RestorePackages>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath> <OutputPath>bin\x64\Debug\</OutputPath>
@ -53,91 +72,110 @@
<ItemGroup> <ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath> <HintPath>packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath> <HintPath>packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath> <HintPath>packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath> <HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64"> <Reference Include="Microsoft.Msagl">
<SpecificVersion>False</SpecificVersion> <HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64"> <Reference Include="Microsoft.Msagl.Drawing">
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Msagl.GraphViewerGdi">
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.DryadLinq, Version=0.1.3.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath> <HintPath>packages\Microsoft.Research.Dryad.0.1.4-beta001\lib\net45\Microsoft.Research.DryadLinq.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge">
<HintPath>packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils">
<HintPath>packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL"> <Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath> <HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions"> <Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath> <SpecificVersion>False</SpecificVersion>
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop"> <Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath> <SpecificVersion>False</SpecificVersion>
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath> <HintPath>packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath> <HintPath>packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath> <HintPath>packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management"> <Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath> <HintPath>packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath> <HintPath>packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath> <HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath> <HintPath>packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath> <HintPath>packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference> </Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL"> <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath> <HintPath>packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference> </Reference>
<Reference Include="System" /> <Reference Include="System" />
<Reference Include="System.Core" /> <Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" /> <Reference Include="System.Data.Linq" />
<Reference Include="System.Drawing" />
<Reference Include="System.Net" /> <Reference Include="System.Net" />
<Reference Include="System.Net.Http" /> <Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions"> <Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath> <SpecificVersion>False</SpecificVersion>
<HintPath>packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference> </Reference>
<Reference Include="System.Net.Http.Primitives"> <Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath> <SpecificVersion>False</SpecificVersion>
<HintPath>packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference> </Reference>
<Reference Include="System.Net.Http.WebRequest" /> <Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.ServiceModel" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath> <HintPath>packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
</Reference> </Reference>
<Reference Include="System.Web" /> <Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" /> <Reference Include="System.Xml.Linq" />
@ -150,11 +188,17 @@
<Compile Include="ApplyAndForkTests.cs" /> <Compile Include="ApplyAndForkTests.cs" />
<Compile Include="BasicAPITests.cs" /> <Compile Include="BasicAPITests.cs" />
<Compile Include="GroupByReduceTests.cs" /> <Compile Include="GroupByReduceTests.cs" />
<Compile Include="MiscBugFixTests.cs" />
<Compile Include="Program.cs" /> <Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="RangePartitionAPICoverageTests.cs" /> <Compile Include="RangePartitionAPICoverageTests.cs" />
<Compile Include="SerializationTests.cs" />
<Compile Include="SerializationTestTypes.cs" />
<Compile Include="SimpleTests.cs" />
<Compile Include="TestLog.cs" />
<Compile Include="TypesInQueryTests.cs" />
<Compile Include="Utils.cs" /> <Compile Include="Utils.cs" />
<Compile Include="Validate.cs" /> <Compile Include="WordCount.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="App.config" /> <None Include="App.config" />
@ -184,19 +228,21 @@
</BootstrapperPackage> </BootstrapperPackage>
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" /> <Import Project="packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''"> <Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" /> <Error Condition="!Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" /> <Error Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target> </Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets'))" />
</Target> </Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets" Condition="Exists('packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild"> <Target Name="BeforeBuild">

46
DryadLinqTests/DryadLinqTests.sln Normal file
View File

@ -0,0 +1,46 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2012
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadLinqTests", "DryadLinqTests.csproj", "{4EA15885-003F-4B4D-A7DB-C172C2F23161}"
EndProject
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "DryadLinqFSharpTests", "DryadLinqFSharpTests\DryadLinqFSharpTests.fsproj", "{D0153361-1B5D-4715-8EE3-AA824F89CBC3}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|Mixed Platforms = Debug|Mixed Platforms
Debug|x64 = Debug|x64
Release|Any CPU = Release|Any CPU
Release|Mixed Platforms = Release|Mixed Platforms
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|x64.ActiveCfg = Debug|x64
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|x64.Build.0 = Debug|x64
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Any CPU.Build.0 = Release|Any CPU
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Mixed Platforms.ActiveCfg = Release|x64
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Mixed Platforms.Build.0 = Release|x64
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|x64.ActiveCfg = Release|x64
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|x64.Build.0 = Release|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|x64.ActiveCfg = Debug|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|x64.Build.0 = Debug|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Any CPU.Build.0 = Release|Any CPU
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Mixed Platforms.ActiveCfg = Release|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Mixed Platforms.Build.0 = Release|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|x64.ActiveCfg = Release|x64
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

963
DryadLinqTests/GroupByReduceTests.cs
View File

File diff suppressed because it is too large Load Diff

967
DryadLinqTests/MiscBugFixTests.cs Normal file
View File

@ -0,0 +1,967 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Text.RegularExpressions;
namespace DryadLinqTests
{
public static class MiscBugFixTests
{
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" MiscBugFixTests ");
TestLog.Message(" **********************");
var tests = new Dictionary<string, Action>()
{
{"Bug11447_GroupByWithComparer", () => Bug11447_GroupByWithComparer(context) },
{"Bug12584_HashPartitionOutputCount", () => Bug12584_HashPartitionOutputCount(context) },
{"Bug13108_DisableSequenceEquals", () => Bug13108_DisableSequenceEquals(context) },
{"Bug13529_and_Bug13593_IndexedOperatorCompilation", () => Bug13529_and_Bug13593_IndexedOperatorCompilation(context) },
{"Bug13130_ReverseOperator", () => Bug13130_ReverseOperator(context) },
{"Bug13736_IndexedTakeWhile", () => Bug13736_IndexedTakeWhile(context) },
{"Bug13534_HashPartitionNegIndexIsError", () => Bug13534_HashPartitionNegIndexIsError(context) },
{"Bug13474_and_Bug13483_FromDscOnBadFileSet", () => Bug13474_and_Bug13483_FromDscOnBadFileSet(context) },
{"Bug13637_EmptyFilesInFilesets", () => Bug13637_EmptyFilesInFilesets(context) },
{"Bug13637_LocalDebugProducingZeroRecords", () => Bug13637_LocalDebugProducingZeroRecords(context) },
{"Bug13245_FromDsc_Submit_throws", () => Bug13245_FromDsc_Submit_throws(context) },
{"Bug13245_QueryUsingNonHpcLinqOperator", () => Bug13245_QueryUsingNonHpcLinqOperator(context) },
{"Bug13302_ConfigEnvironmentCleanup", () => Bug13302_ConfigEnvironmentCleanup(context) },
{"Bug13970_MismatchedDataTypes", () => Bug13970_MismatchedDataTypes(context) },
{"Bug14010_AlreadyDisposedContext", () => Bug14010_AlreadyDisposedContext(context) },
{"Bug14256_LeaseOnTempDscFileset", () => Bug14256_LeaseOnTempDscFileset(context) },
{"Bug14189_OrderPreservation", () => Bug14189_OrderPreservation(context) },
{"Bug14190_MergeJoin_DecreasingOrder", () => Bug14190_MergeJoin_DecreasingOrder(context) },
{"Bug14192_MultiApplySubExpressionReuse", () => Bug14192_MultiApplySubExpressionReuse(context) },
{"Bug14870_LongIndexTakeWhile", () => Bug14870_LongIndexTakeWhile(context) },
{"Bug15159_NotOperatorForNullableBool", () => Bug15159_NotOperatorForNullableBool(context) },
{"Bug15371_NoDataMembersForSerialization", () => Bug15371_NoDataMembersForSerialization(context) },
{"Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes", () => Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static bool Bug11447_GroupByWithComparer(DryadLinqContext context)
{
string testName = "Bug11447_GroupByWithComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
result[0] = a;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
result[1] = a;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug12584_HashPartitionOutputCount(DryadLinqContext context) // ToDo
{
string testName = "Bug12584_HashPartitionOutputCount";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug12584_HashPartitionOutputCount";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.HashPartition(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
info.Wait();
//// partition verification are only valid for cluster execution..
//// check that nOutputPartitions == nInputPartitions.
//// Note: this is today's behavior, but we don't strictly guarantee this and the rules may change.
//var inFS = context.DscService.GetFileSet(DataGenerators.SIMPLE_FILESET_NAME);
//var inPCount = inFS.GetFiles().Count();
//var outFS = context.DscService.GetFileSet("DevUnitTest/Bug12584_out");
//var outPCount = outFS.GetFiles().Count();
//passed &= TestUtils.Assert(outPCount == inPCount, "Output nPartitions should be equal to input nPartitions.");
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13108_DisableSequenceEquals(DryadLinqContext context)
{
string testName = "Bug13108_DisableSequenceEquals";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
bool result = pt1.Select(x => x).SequenceEqual(pt2.Select(x => x));
passed &= false; // NotSupportedException should have been thrown
}
}
catch (Exception Ex)
{
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13529_and_Bug13593_IndexedOperatorCompilation(DryadLinqContext context)
{
string testName = "Bug13529_and_Bug13593_IndexedOperatorCompilation";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
result[0] = pt1.Select((x, i) => x)
.LongSelect((x, i) => x)
.LongSelectMany((x, i) => new[] { x })
.SelectMany((x, i) => new[] { x })
.Where((x, i) => true)
.LongWhere((x, i) => true)
.TakeWhile((x, i) => true)
.LongTakeWhile((x, i) => true)
.SkipWhile((x, i) => false)
.LongSkipWhile((x, i) => false)
.ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
result[1] = pt1.Select((x, i) => x)
.LongSelect((x, i) => x)
.LongSelectMany((x, i) => new[] { x })
.SelectMany((x, i) => new[] { x })
.Where((x, i) => true)
.LongWhere((x, i) => true)
.TakeWhile((x, i) => true)
.LongTakeWhile((x, i) => true)
.SkipWhile((x, i) => false)
.LongSkipWhile((x, i) => false)
.ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13130_ReverseOperator(DryadLinqContext context)
{
string testName = "Bug13130_ReverseOperator";
TestLog.TestStart(testName);
bool passed = true;
//data set #1
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
result[0] = pt1.Reverse().ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
result[1] = pt1.Reverse().ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
//data set #2 ToDo
//data set #3 ToDo
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13736_IndexedTakeWhile(DryadLinqContext context)
{
string testName = "Bug13736_IndexedTakeWhile";
TestLog.TestStart(testName);
bool passed = true;
// dataset #1
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var pt2 = pt1.TakeWhile((x, i) => i < 6).ToArray();
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 6).ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var pt2 = pt1.TakeWhile((x, i) => i < 6).ToArray();
result[1] = pt1.AsEnumerable().TakeWhile((x, i) => i < 6).ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
// dataset #2
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var pt2 = pt1.TakeWhile((x, i) => i < 125).ToArray();
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 125).ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetGroupByReduceDataSet(context);
var pt2 = pt1.TakeWhile((x, i) => i < 125).ToArray();
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 125).ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception)
{
passed &= false;
}
}
catch (Exception)
{
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13534_HashPartitionNegIndexIsError(DryadLinqContext context)
{
string testName = "Bug13534_HashPartitionNegIndexIsError";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
pt1.HashPartition(x => x, 0); // exception should be thrown
passed &= false;
}
}
catch (Exception)
{
}
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
pt1.HashPartition(x => x, -1); // exception should be thrown
passed &= false;
}
}
catch (Exception)
{
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13474_and_Bug13483_FromDscOnBadFileSet(DryadLinqContext context)
{
// ToDo
return false;
}
public static bool Bug13637_EmptyFilesInFilesets(DryadLinqContext context)
{
// ToDo
return false;
}
public static bool Bug13637_LocalDebugProducingZeroRecords(DryadLinqContext context)
{
string testName = "Bug13637_LocalDebugProducingZeroRecords";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug13637_LocalDebugProducingZeroRecords";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<int> pt2 = pt1.Where(x => false).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var queryInfo2 = pt2.Submit();
queryInfo2.Wait();
int[] data2 = pt2.ToArray();
passed &= (data2.Length == 0);
// query producing no records -> anonymous output
IQueryable<int> pt3 = pt1.Where(x => false);
var queryInfo3 = pt3.Submit();
queryInfo3.Wait();
int[] data3 = pt3.ToArray();
passed &= (data3.Length == 0);
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13245_FromDsc_Submit_throws(DryadLinqContext context)
{
string testName = "Bug13245_FromDsc_Submit_throws";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.Submit();
passed &= false; // should throw ArgEx as the input isn't well formed
}
}
catch (ArgumentException)
{
}
catch (Exception)
{
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static IQueryable<int> Blah(this IQueryable<int> source)
{
return Enumerable.Range(1, 1).AsQueryable();
}
public static bool Bug13245_QueryUsingNonHpcLinqOperator(DryadLinqContext context)
{
string testName = "Bug13245_QueryUsingNonHpcLinqOperator";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.Blah().Submit();
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static IEnumerable<int> CheckEnvVar(IEnumerable<int> en)
{
if (Environment.GetEnvironmentVariable("DummyEnvVar") == null)
throw new Exception("the expected environment variable is not defined");
return en;
}
public static bool Bug13302_ConfigEnvironmentCleanup(DryadLinqContext context)
{
string testName = "Bug13302_ConfigEnvironmentCleanup";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.Apply((en) => CheckEnvVar(en)).Submit();
info.Wait();
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug13970_MismatchedDataTypes(DryadLinqContext context)
{
string testName = "Bug13970_MismatchedDataTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetGroupByReduceDataSet(context);
double? output = pt1.AverageAsQuery().Single();
passed &= false;
}
}
catch (Exception)
{
passed &= true; // expected
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14010_AlreadyDisposedContext(DryadLinqContext context)
{
string testName = "Bug14010_AlreadyDisposedContext";
TestLog.TestStart(testName);
bool passed = true;
context.LocalDebug = false;
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
ctx.Dispose();
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
int output = pt1.Select(x => x).First();
passed &= false;
}
catch (Exception)
{
passed &= true;
}
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
ctx.Dispose();
int output = pt1.Select(x => x).First();
passed &= false;
}
catch (Exception)
{
passed &= true;
}
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
ctx.Dispose();
IQueryable<int> query = pt1.Select(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, "abc"), true);
var info = DryadLinqQueryable.Submit(query);
passed &= false;
}
catch (Exception)
{
passed &= true;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14256_LeaseOnTempDscFileset(DryadLinqContext context) // TODO
{
string testName = "Bug14256_LeaseOnTempDscFileset";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14189_OrderPreservation(DryadLinqContext context)
{
string testName = "Bug14189_OrderPreservation";
TestLog.TestStart(testName);
bool passed = true;
try
{
IGrouping<int, int>[] clusterSorted, localSorted;
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
var output = pt1.OrderBy(x => x % 4)
.Select(x => x).Select(x => x).Where(x => true) // this pipeline was not preserving order correctly.
.GroupBy(x => x % 4)
.ToArray();
passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
// sort back on the key for deterministic output.
clusterSorted = output.OrderBy(x => x.Key).ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
var output = pt1.OrderBy(x => x % 4)
.Select(x => x).Select(x => x).Where(x => true) // this pipeline was not preserving order correctly.
.GroupBy(x => x % 4)
.ToArray();
passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
// sort back on the key for deterministic output.
localSorted = output.OrderBy(x => x.Key).ToArray();
}
// check that each group of output has the same elements as the LINQ groups.
for (int i = 0; i < 4; i++)
{
var a = clusterSorted[i].OrderBy(x => x);
var b = localSorted[i].OrderBy(x => x);
passed &= a.SequenceEqual(b); //the output should match linq. Error for group: + i);
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14190_MergeJoin_DecreasingOrder(DryadLinqContext context) // TODO
{
string testName = "Bug14190_MergeJoin_DecreasingOrder";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug14190_MergeJoin_DecreasingOrder";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14192_MultiApplySubExpressionReuse(DryadLinqContext context)
{
string testName = "Bug14192_MultiApplySubExpressionReuse";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug14192_MultiApplySubExpressionReuse";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var data = pt1.Select(x => x);
var info = pt1.Apply(new[] { pt1, pt1 }, (sources) => new int[] { 1, 2, 3 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
info.Wait();
// ToDo
//string queryPlan = TestUtils.GetRecentQueryXmlAsText();
//int nVerticesInPlan = TestUtils.GetVertexStageCount(queryPlan);
//passed &= (nVerticesInPlan == 7); // "Only seven vertices should appear (before bug, there were 10 of which the last three were extraneous.");
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14870_LongIndexTakeWhile(DryadLinqContext context)
{
string testName = "Bug14870_LongIndexTakeWhile";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
// ToDo - move to data generator
int[][] data = new[] {
new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { }, new int[] { } };
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.LongTakeWhile((x, i) => i < 8).ToArray();
passed &= (output.Length == 8); // "Only eight items should be returned."
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug15159_NotOperatorForNullableBool(DryadLinqContext context)
{
string testName = "Bug15159_NotOperatorForNullableBool";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<bool?>[] result = new IEnumerable<bool?>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
result[0] = output.ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
result[1] = output.ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public class NoDataMembersClass
{
}
public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
{
string testName = "Bug15371_NoDataMembersForSerialization";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var result = context.FromStore<NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
passed &= false;
}
}
catch (DryadLinqException Ex)
{
passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(DryadLinqContext context)
{
string testName = "Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
// use an anonymous type with nullable fields, and make sure we at least have some null values in the stream
var output = pt1.Select(x => new { FirstField = (x % 2 == 0) ? new int?(x) : default(int?), SecondField = x.ToString() })
.GroupBy(x => x.FirstField, y => y.SecondField); // use of GB ensures we exercise the emitted GetHashCode() overload
passed &= (output.Count() != 0); // "Query return 0 length output"
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
}
}

57
DryadLinqTests/Program.cs
View File

@ -17,14 +17,15 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License. limitations under the License.
*/ */
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using System.IO; using System.IO;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Reflection;
using Microsoft.Research.DryadLinq; using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;
namespace DryadLinqTests namespace DryadLinqTests
@ -33,32 +34,38 @@ namespace DryadLinqTests
{ {
public static void Main(string[] args) public static void Main(string[] args)
{ {
//BasicAPITests.ToStoreThrowsForNonQuery(); SimpleTests.Run(args);
//BasicAPITests.ToStoreGetEnumeratorThrows();
//BasicAPITests.GetEnumeratorNonToStoreTerminated();
//BasicAPITests.ToStoreSubmitGetEnumerator();
//BasicAPITests.SubmitNonToStoreTerminated();
//BasicAPITests.MaterializeToStoreTerminated();
//BasicAPITests.MaterializeNonToStoreTerminated();
//BasicAPITests.EnumeratePlainData();
//BasicAPITests.RepeatSubmit(); /*
//BasicAPITests.RepeatMaterialize(); *
//BasicAPITests.MaterializeMentionsSameQueryTwice(); Config conf = new Config("dryad-temp", "main1", @"d:\temp\TestLog\");
//BasicAPITests.QueryOnDataBackedDLQ(); DryadLinqContext context = new DryadLinqContext(Config.cluster);
//BasicAPITests.Bug11781_CountandFirstOrDefault();
//BasicAPITests.Bug11782_Aggregate(); string matchPattern = @"";
//BasicAPITests.Bug11782_LowLevelQueryableManipulation();
//BasicAPITests.Bug11638_LongWhere(); TestLog.LogInit(Config.testLogPath + "BasicAPITests.txt");
//BasicAPITests.AssumeRangePartition(); BasicAPITests.Run(context, matchPattern);
//BasicAPITests.Bug11638_LongMethods();
//BasicAPITests.ContextConfigIsReadOnly(); TestLog.LogInit(Config.testLogPath + "ApplyAndForkTests.txt");
//BasicAPITests.ToggleSpeculativeDuplication(); ApplyAndForkTests.Run(context, matchPattern);
//BasicAPITests.Bug15068_ConfigResourcesAPI();
//BasicAPITests.Bug_16341_SubmitThrowsForDifferentContexts(); TestLog.LogInit(Config.testLogPath + "GroupByReduceTests.txt");
//BasicAPITests.Bug14449_ContextShouldExposeVersionIDs(); GroupByReduceTests.Run(context, matchPattern);
//BasicAPITests.Bug_16341_VariousTestsForSubmit();
TestLog.LogInit(Config.testLogPath + "RangePartitionAPICoverageTests.txt");
RangePartitionAPICoverageTests.Run(context, matchPattern);
TestLog.LogInit(Config.testLogPath + "TypesInQueryTests.txt");
TypesInQueryTests.Run(context, matchPattern);
TestLog.LogInit(Config.testLogPath + "SerializationTests.txt");
SerializationTests.Run(context, matchPattern);
TestLog.LogInit(Config.testLogPath + "MiscBugFixTests.txt");
MiscBugFixTests.Run(context, matchPattern);
*/
} }
} }
} }

8
DryadLinqTests/Properties/AssemblyInfo.cs
View File

@ -28,7 +28,7 @@ using System.Runtime.InteropServices;
[assembly: AssemblyDescription("")] [assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")] [assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")] [assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("DryadLinqTests")] [assembly: AssemblyProduct("Dryad")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")] [assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")] [assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")] [assembly: AssemblyCulture("")]
@ -39,7 +39,7 @@ using System.Runtime.InteropServices;
[assembly: ComVisible(false)] [assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM // The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("1fce146c-8597-4cc2-92f6-ebb655cd99ed")] [assembly: Guid("49b725be-1187-4056-9890-7a0cb4c064a4")]
// Version information for an assembly consists of the following four values: // Version information for an assembly consists of the following four values:
// //
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers // You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below: // by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")] // [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.3.0")] [assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.3.0")] [assembly: AssemblyFileVersion("0.1.5.0")]

836
DryadLinqTests/RangePartitionAPICoverageTests.cs
View File

@ -1,12 +1,842 @@
using System; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.DryadLinq.Internal;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Linq.Expressions;
using System.Threading.Tasks; using System.Text.RegularExpressions;
namespace DryadLinqTests namespace DryadLinqTests
{ {
public class RangePartitionAPICoverageTests public class RangePartitionAPICoverageTests
{ {
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" RangePartitionAPICoverageTests ");
TestLog.Message(" **********************");
var tests = new Dictionary<string, Action>()
{
{"RP_keySelector", () => RP_keySelector(context) },
{"RP_keySelector_pcount", () => RP_keySelector_pcount(context) },
{"RP_keySelector_isDescending", () => RP_keySelector_isDescending(context) },
{"RP_keySelector_rangeKeys", () => RP_keySelector_rangeKeys(context) },
{"RP_keySelector_isDescending_pcount", () => RP_keySelector_isDescending_pcount(context) },
{"RP_keySelector_keyComparer_isDescending", () => RP_keySelector_keyComparer_isDescending(context) },
{"RP_keySelector_rangeKeys_keyComparer", () => RP_keySelector_rangeKeys_keyComparer(context) },
{"RP_keySelector_keyComparer_isDescending_pcount", () => RP_keySelector_keyComparer_isDescending_pcount(context) },
{"RP_keySelector_rangeKeys_keyComparer_isDescending", () => RP_keySelector_rangeKeys_keyComparer_isDescending(context) },
{"RP_singlePartition_autoSeparators", () => RP_singlePartition_autoSeparators(context) },
{"RP_rangeSeparators_customComparer", () => RP_rangeSeparators_customComparer(context) },
{"RP_rangeSeparators_nullCustomComparer", () => RP_rangeSeparators_nullCustomComparer(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static IEnumerable<int> CountPartitions<T>(IEnumerable<T> pt)
{
yield return 1;
}
public static IEnumerable<T> MaxInPartition<T>(IEnumerable<T> pt)
{
yield return pt.Max();
}
public static IEnumerable<T> MinInPartition<T>(IEnumerable<T> pt)
{
yield return pt.Min();
}
public static IEnumerable<T2> MaxCompare<T1, T2>(IEnumerable<T1> pt, int i, IEnumerable<T2> maxes)
{
if (0 == i)
{
yield return (T2)Convert.ChangeType(1, typeof(int));
}
int priorMax = (int)Convert.ChangeType(maxes.ToArray()[i - 1], typeof(T2));
foreach (T1 item in pt)
{
int val = (int)Convert.ChangeType(item, typeof(T1));
if (val <= priorMax)
{
yield return (T2)Convert.ChangeType(1, typeof(int));
}
}
}
public static bool RP_keySelector(DryadLinqContext context)
{
string testName = "RP_keySelector";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 3, false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_pcount(DryadLinqContext context)
{
string testName = "RP_keySelector_pcount";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_pcount";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, 2).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 2, false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, 2);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_isDescending(DryadLinqContext context)
{
string testName = "RP_keySelector_isDescending";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_isDescending";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 3, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, true);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_rangeKeys(DryadLinqContext context)
{
string testName = "RP_keySelector_rangeKeys";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
IEnumerable<int>[] result2 = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_rangeKeys_1";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//increasing ranges keys.
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 4, false);
result[0] = pt2.ToList();
string outFile2 = "unittest/output/RP_keySelector_rangeKeys_2";
IQueryable<int> pt1_2 = DataGenerator.GetRangePartitionDataSet(context);
//decreasing ranges keys.
IQueryable<int> pt2_2 = pt1_2.RangePartition(x => x, new[] { 8, 5, 2 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile2), true);
var jobInfo2 = pt2_2.Submit();
jobInfo2.Wait();
passed &= TestRangePartitioned(pt2_2, 4, true);
result2[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//increasing ranges keys.
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 });
result[1] = pt2.ToList();
IQueryable<int> pt1_2 = DataGenerator.GetRangePartitionDataSet(context);
//decreasing ranges keys.
IQueryable<int> pt2_2 = pt1_2.RangePartition(x => x, new[] { 8, 5, 2 });
result2[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
Validate.Check(result2);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_isDescending_pcount(DryadLinqContext context)
{
string testName = "RP_keySelector_isDescending_pcount";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_isDescending_pcount";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, true, 2).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 2, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, true, 2);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_keyComparer_isDescending(DryadLinqContext context)
{
string testName = "RP_keySelector_keyComparer_isDescending";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_keyComparer_isDescending";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 3, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_rangeKeys_keyComparer(DryadLinqContext context)
{
string testName = "RP_keySelector_rangeKeys_keyComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_rangeKeys_keyComparer";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }, Comparer<int>.Default)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 4, false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }, Comparer<int>.Default);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_keySelector_keyComparer_isDescending_pcount(DryadLinqContext context)
{
string testName = "RP_keySelector_keyComparer_isDescending_pcount";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_keyComparer_isDescending_pcount";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true, 2)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 2, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true, 2);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
return passed;
}
public static bool RP_keySelector_rangeKeys_keyComparer_isDescending(DryadLinqContext context)
{
string testName = "RP_keySelector_rangeKeys_keyComparer_isDescending";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_rangeKeys_keyComparer_isDescending";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer<int>.Default, true)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 4, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer<int>.Default, true);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_singlePartition_autoSeparators(DryadLinqContext context)
{
string testName = "RP_singlePartition_autoSeparators";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_singlePartition_autoSeparators";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.Apply(x => x) // force a merge
.RangePartition(x => x)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 1, false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable<int> pt2 = pt1.Apply(x => x) // force a merge
.RangePartition(x => x);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool RP_rangeSeparators_customComparer(DryadLinqContext context)
{
string testName = "RP_rangeSeparators_customComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_rangeSeparators_customComparer";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//these keys should be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should also be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 4, 3, 2, 1 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should work
IQueryable<int> pt2 = pt1.RangePartition(x => x, new int[] { 6, 6, 3, 1 }, new WeirdIntComparer(), false)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 5, new WeirdIntComparer(), false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
string outFile = "unittest/output/RP_rangeSeparators_customComparer";
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//these keys should be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should also be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 4, 3, 2, 1 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should work
IQueryable<int> pt2 = pt1.RangePartition(x => x, new int[] { 6, 6, 3, 1 }, new WeirdIntComparer(), false)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool TestRangePartitioned(IQueryable<int> pt, int expectedPCount, bool expectedIsDescending)
{
bool passed = true;
try
{
IEnumerable<int> ptCount = pt.ApplyPerPartition(x => CountPartitions(x));
int cPartitions = ptCount.Count();
passed &= (expectedPCount == cPartitions);
if (passed)
{
int[] ptMax = pt.ApplyPerPartition(x => MaxInPartition(x)).ToArray();
int[] ptMin = pt.ApplyPerPartition(x => MinInPartition(x)).ToArray();
// compare mins and maxs of each partition
for (int i = 1; i < cPartitions; i++)
{
if (false == expectedIsDescending)
passed &= (ptMin[i] > ptMax[i - 1]);
else
passed &= (ptMax[i] < ptMin[i - 1]);
}
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
return passed;
}
[Serializable]
public class WeirdIntComparer : IComparer<int>
{
public int Compare(int x, int y)
{
int xx = x % 2;
int yy = y % 2;
return xx.CompareTo(yy);
}
}
public static bool RP_rangeSeparators_nullCustomComparer(DryadLinqContext context)
{
string testName = "RP_rangeSeparators_nullCustomComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
var results1 = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null, false).ToArray();
var results2 = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null).ToArray();
//passing is not throwing.
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool TestRangePartitioned(IQueryable<int> pt, int expectedPCount, IComparer<int> comparer, bool expectedIsDescending)
{
bool passed = true;
try
{
IEnumerable<int> ptCount = pt.ApplyPerPartition(x => CountPartitions(x));
int cPartitions = ptCount.Count();
passed &= (expectedPCount == cPartitions);
if (passed)
{
int[] ptMax = pt.ApplyPerPartition(x => MaxInPartition(x)).ToArray();
int[] ptMin = pt.ApplyPerPartition(x => MinInPartition(x)).ToArray();
// compare mins and maxs of each partition
for (int i = 1; i < cPartitions; i++)
{
passed &= (comparer.Compare(ptMin[i], ptMax[i - 1]) == 0); // ToDo
}
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
return passed;
}
} }
} }

595
DryadLinqTests/SerializationTestTypes.cs Normal file
View File

@ -0,0 +1,595 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace DryadLinqTests
{
//////////////////////////////////////////////////////////////////////
//
// Decorated and undecorated types
//
// A UDT with no attribute shouldn't autoserialize without an explicit [Serializable] attribute
public class UDT_Undecorated
{
public UDT_Undecorated(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
}
public int m_field1;
public int m_field2;
}
// UDT which is marked as serializable. This should be autoserialized
[Serializable]
public class UDT_MarkedSerializable
{
public UDT_MarkedSerializable(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
}
public int m_field1;
public int m_field2;
}
// UDT that has the old style static serialization methods. This should be rejected.
public class UDT_StaticSerializer
{
public UDT_StaticSerializer(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
}
public int m_field1;
public int m_field2;
public static UDT_StaticSerializer Read(DryadLinqBinaryReader reader)
{
var val = new UDT_StaticSerializer(0);
val.m_field1 = reader.ReadInt32();
val.m_field2 = reader.ReadInt32();
return val;
}
public static void Write(DryadLinqBinaryWriter writer, UDT_StaticSerializer val)
{
writer.Write(val.m_field1);
writer.Write(val.m_field2);
}
}
//////////////////////////////////////////////////////////////////////
//
// UDTs with custom serializers
//
// UDT that has an attribute declaring itself as its custom serializer
[CustomDryadLinqSerializer(typeof(UDT_SelfCustomSerializer))]
public struct UDT_SelfCustomSerializer : IDryadLinqSerializer<UDT_SelfCustomSerializer>
{
public UDT_SelfCustomSerializer(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
}
public int m_field1;
public int m_field2;
public UDT_SelfCustomSerializer Read(DryadLinqBinaryReader reader)
{
var val = new UDT_SelfCustomSerializer(0);
val.m_field1 = reader.ReadInt32();
val.m_field2 = reader.ReadInt32();
return val;
}
public void Write(DryadLinqBinaryWriter writer, UDT_SelfCustomSerializer val)
{
writer.Write(val.m_field1);
writer.Write(val.m_field2);
}
}
// UDT that declares another type as its custom serializer
[CustomDryadLinqSerializer(typeof(CustomUDTSerializer))]
public struct UDT_ExternalCustomSerializer
{
public UDT_ExternalCustomSerializer(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
}
public int m_field1;
public int m_field2;
}
// this is the custom serializer for UDT_ExternalCustomSerializer
public class CustomUDTSerializer : IDryadLinqSerializer<UDT_ExternalCustomSerializer>
{
public UDT_ExternalCustomSerializer Read(DryadLinqBinaryReader reader)
{
var val = new UDT_ExternalCustomSerializer(0);
val.m_field1 = reader.ReadInt32();
val.m_field2 = reader.ReadInt32();
return val;
}
public void Write(DryadLinqBinaryWriter writer, UDT_ExternalCustomSerializer val)
{
writer.Write(val.m_field1);
writer.Write(val.m_field2);
}
}
// UDT with a CustomHpcSerializer attribute that points to an invalid type for the serializer
[CustomDryadLinqSerializer(typeof(int))]
public struct UDT_BadCustomSerializerType1
{
public UDT_BadCustomSerializerType1(int val)
{
m_field1 = val + 1;
}
public int m_field1;
}
// UDT with a CustomHpcSerializer attribute that points to a serializer type that targets a different UDT
[CustomDryadLinqSerializer(typeof(CustomUDTSerializer))]
public struct UDT_BadCustomSerializerType2
{
public UDT_BadCustomSerializerType2(int val)
{
m_field1 = val + 1;
}
public int m_field1;
}
//////////////////////////////////////////////////////////////////////
//
// Inheritance
//
// A UDT that has sub types. This should be rejected
[Serializable]
public class UDT_BaseType
{
public UDT_BaseType(int val)
{
m_baseTypeField = val + 42;
}
public int m_baseTypeField;
}
// A UDT that has derives from a type other than object. This should be rejected
[Serializable]
public class UDT_DerivedType : UDT_BaseType
{
public UDT_DerivedType(int val)
: base(val)
{
m_derivedTypeField = val + 84;
}
public int m_derivedTypeField;
}
//////////////////////////////////////////////////////////////////////
//
// Field types
//
// A UDT with no data. This should be rejected
[Serializable]
public class UDT_EmptyType
{
public UDT_EmptyType(int val)
{
}
}
// A UDT with a public field of type System.Object. This should be rejected
[Serializable]
public class UDT_ObjectField
{
public UDT_ObjectField(int val)
{
m_intField = val + 84;
m_objectField = null;
}
public int m_intField;
public object m_objectField;
}
// A UDT with a public field of type System.Object[]. This should be rejected
[Serializable]
public class UDT_ObjectArrayField
{
public UDT_ObjectArrayField(int val)
{
m_intField = val + 84;
m_objectArrayField = new object[1];
m_objectArrayField[0] = null;
}
public int m_intField;
public object[] m_objectArrayField;
}
// A UDT with a public field of type List<System.Object>. This should be rejected
[Serializable]
public class UDT_ObjectListField
{
public UDT_ObjectListField(int val)
{
m_intField = val + 84;
m_objectListField = new List<object>();
m_objectListField.Add(null);
}
public int m_intField;
public List<object> m_objectListField;
}
// A UDT with an object field, and a custom serializer. This should not be rejected
[CustomDryadLinqSerializer(typeof(UDT_ObjectFieldAndCustomSerializer))]
public class UDT_ObjectFieldAndCustomSerializer : IDryadLinqSerializer<UDT_ObjectFieldAndCustomSerializer>
{
private object m_objectRecord = (object)"";
// Required by CustomHpcSerializer
public UDT_ObjectFieldAndCustomSerializer() { }
public UDT_ObjectFieldAndCustomSerializer(int val)
{
m_objectRecord = (object)String.Format("{0}", val);
}
#region IHpcSerializer implementation
public UDT_ObjectFieldAndCustomSerializer Read(DryadLinqBinaryReader reader)
{
string tmp = reader.ReadString();
int val = Int32.Parse(tmp);
return new UDT_ObjectFieldAndCustomSerializer(val);
}
public void Write(DryadLinqBinaryWriter writer, UDT_ObjectFieldAndCustomSerializer record)
{
writer.Write((string)record.m_objectRecord);
}
#endregion
}
// An empty UDT with a custom serializer. This should be rejected, because even though the user has control over serialization
// we will encounter runtime problems if the custom serializer reads/writer 0 bytes. This behavior is simply to discourage empty CS code.
[CustomDryadLinqSerializer(typeof(UDT_EmptyTypeWithCustomSerializer))]
public class UDT_EmptyTypeWithCustomSerializer : IDryadLinqSerializer<UDT_EmptyTypeWithCustomSerializer>
{
// Required by CustomHpcSerializer
public UDT_EmptyTypeWithCustomSerializer() { }
public UDT_EmptyTypeWithCustomSerializer(int val)
{
}
#region IHpcSerializer implementation
public UDT_EmptyTypeWithCustomSerializer Read(DryadLinqBinaryReader reader)
{
return new UDT_EmptyTypeWithCustomSerializer(0);
}
public void Write(DryadLinqBinaryWriter writer, UDT_EmptyTypeWithCustomSerializer record)
{
}
#endregion
}
//////////////////////////////////////////////////////////////////////
//
// Visibility
//
// UDT with a field of non-public type. We cannot handle these
[Serializable]
public class UDT_FieldOfNonPublicType
{
private enum SecretCodeLevel
{
Secret,
SuperSecret,
}
public UDT_FieldOfNonPublicType(int val)
{
m_field = (SecretCodeLevel)val;
}
private SecretCodeLevel m_field;
}
// UDT with a private field of a public type. We do handle these using emitted IL code.
[Serializable]
public class UDT_PrivateFieldOfPublicType
{
public UDT_PrivateFieldOfPublicType(int val)
{
m_field = val + 1;
}
private int m_field;
}
//////////////////////////////////////////////////////////////////////
//
// Nesting
//
[Serializable]
public class UDT_Nested_InnerAndOuterSerializable
{
public UDT_Nested_InnerAndOuterSerializable(int val)
{
m_field = val + 1;
m_field2 = new NestedSerializable(val);
}
private int m_field;
private NestedSerializable m_field2;
[Serializable]
public class NestedSerializable
{
public NestedSerializable(int val)
{
m_field = val / 2.0;
}
private double m_field;
}
}
[Serializable]
public class UDT_Nested_InnerEnum_InnerAndOuterSerializable
{
public UDT_Nested_InnerEnum_InnerAndOuterSerializable(int val)
{
m_field = val + 1;
m_field2 = (NestedEnum)(val % 3);
}
private int m_field;
private NestedEnum m_field2;
//[Serializable]
public enum NestedEnum
{
Foo = 0,
Bar = 1,
Baz = 2,
}
}
[Serializable]
public class UDT_Nested_OuterSerializableInnerNotSerializable
{
public UDT_Nested_OuterSerializableInnerNotSerializable(int val)
{
m_field = val + 1;
m_field2 = new NestedNotSerializable(val);
}
private int m_field;
private NestedNotSerializable m_field2;
public class NestedNotSerializable
{
public NestedNotSerializable(int val)
{
m_field = val / 2.0;
}
private double m_field;
}
}
public class UDT_Nested_OuterNotSerializableInnerSerializable
{
public UDT_Nested_OuterNotSerializableInnerSerializable(int val)
{
m_field = val + 1;
m_field2 = new NestedSerializable(val);
}
private int m_field;
private NestedSerializable m_field2;
[Serializable]
public class NestedSerializable
{
public NestedSerializable(int val)
{
m_field = val / 2.0;
}
private double m_field;
}
}
//////////////////////////////////////////////////////////////////////
//
// Self reference
//
// First level circular type
[Serializable]
public class UDT_FirstLevelCircular
{
public UDT_FirstLevelCircular(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
m_circularRef = null;
}
public int m_field1;
public int m_field2;
public UDT_FirstLevelCircular m_circularRef;
}
// First level circular type with an array reference to self
[Serializable]
public class UDT_FirstLevelCircularArrayRef
{
public UDT_FirstLevelCircularArrayRef(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
m_circularRefArray = new UDT_FirstLevelCircular[5];
}
public int m_field1;
public int m_field2;
public UDT_FirstLevelCircular[] m_circularRefArray;
}
// Second level circular type
[Serializable]
public class UDT_SecondLevelCircular
{
public UDT_SecondLevelCircular(int val)
{
m_field1 = val + 1;
m_field2 = val + 2;
m_child = new UDT_CircularRefChild(this);
}
public int m_field1;
public int m_field2;
public UDT_CircularRefChild m_child;
}
[Serializable]
public class UDT_CircularRefChild
{
public UDT_CircularRefChild(UDT_SecondLevelCircular parent)
{
m_parent = parent;
}
public UDT_SecondLevelCircular m_parent;
}
// Circular type with custom serializer. Should not be rejected
[CustomDryadLinqSerializer(typeof(UDT_CircularTypeWithCustomSerializer))]
public class UDT_CircularTypeWithCustomSerializer : IDryadLinqSerializer<UDT_CircularTypeWithCustomSerializer>
{
public UDT_CircularTypeWithCustomSerializer() { }
public UDT_CircularTypeWithCustomSerializer(int val)
{
m_field1 = val;
// create each new object with log2(val) self references hanging off of m_next
if (val == 0)
m_next = null;
else
m_next = new UDT_CircularTypeWithCustomSerializer(val / 2);
}
public int m_field1;
public UDT_CircularTypeWithCustomSerializer m_next;
//
// sample recursive custom serializer
//
public UDT_CircularTypeWithCustomSerializer Read(DryadLinqBinaryReader reader)
{
UDT_CircularTypeWithCustomSerializer obj = new UDT_CircularTypeWithCustomSerializer();
bool bHasValidNext = reader.ReadBool();
obj.m_field1 = reader.ReadInt32();
if (bHasValidNext)
{
obj.m_next = this.Read(reader); // recursively read the next
}
else
{
obj.m_next = null; // terminate recursion
}
return obj;
}
public void Write(DryadLinqBinaryWriter writer, UDT_CircularTypeWithCustomSerializer x)
{
if (x.m_next != null)
{
writer.Write(true); // bHasValidNext for the reader side
writer.Write(x.m_field1);
this.Write(writer, x.m_next); // write out recursively
}
else
{
writer.Write(false); // bHasValidNext = false for the reader side, makes sure recursive reads stop
writer.Write(x.m_field1);
}
}
}
// This type itself isn't circular, but contains a field of a circular type that is custom serialized. Autoserialization should work for this guy
[Serializable]
public class UDT_TypeContainingCustomSerializedCircularType
{
public UDT_TypeContainingCustomSerializedCircularType(int val)
{
m_field1 = val;
m_circularType = new UDT_CircularTypeWithCustomSerializer(val);
}
public int m_field1;
public UDT_CircularTypeWithCustomSerializer m_circularType;
}
}

380
DryadLinqTests/SerializationTests.cs Normal file
View File

@ -0,0 +1,380 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Text.RegularExpressions;
namespace DryadLinqTests
{
public class SerializationTests
{
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" SerializationTests ");
TestLog.Message(" **********************");
context.LocalDebug = false;
IQueryable<int> source = DataGenerator.GetSimpleFileSets(context);
var tests = new Dictionary<string, Action>()
{
{"UDT_Undecorated", () =>
SerializationTests.TestUDT<UDT_Undecorated>(context, source, "Basic UDT with no attribute. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
{"UDT_MarkedSerializable", () =>
SerializationTests.TestUDT<UDT_MarkedSerializable>(context, source, "Basic UDT which is marked as serializable.", null, 0)},
{"UDT_StaticSerializer", () =>
SerializationTests.TestUDT<UDT_StaticSerializer>(context, source, "Basic UDT with old stype static serializers. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
{"UDT_SelfCustomSerializer", () =>
SerializationTests.TestUDT<UDT_SelfCustomSerializer>(context, source, "UDT implementing custom serializer for itself.", null, 0)},
{"UDT_ExternalCustomSerializer", () =>
SerializationTests.TestUDT<UDT_ExternalCustomSerializer>(context, source, "UDT with a custom serializer", null, 0)},
{"UDT_BadCustomSerializerType1", () =>
SerializationTests.TestUDT<UDT_BadCustomSerializerType1>(context, source, "UDT decorated with an invalid custom serializer type.", typeof(DryadLinqException), 0)},
{"UDT_BadCustomSerializerType2", () =>
SerializationTests.TestUDT<UDT_BadCustomSerializerType2>(context, source, "UDT decorated with another UDT's custom serializer.", typeof(DryadLinqException), 0)},
{"UDT_BaseType", () =>
SerializationTests.TestUDT<UDT_BaseType>(context, source, "UDT with subtypes. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleSubtypes"))},
{"UDT_DerivedType", () =>
SerializationTests.TestUDT<UDT_DerivedType>(context, source, "UDT with a base type. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleDerivedtypes"))},
{"UDT_FieldOfNonPublicType", () =>
SerializationTests.TestUDT<UDT_FieldOfNonPublicType>(context, source, "UDT with a field of a non-public type. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("UDTHasFieldOfNonPublicType"))},
{"UDT_Nested_OuterSerializableInnerNotSerializable", () =>
SerializationTests.TestUDT<UDT_Nested_OuterSerializableInnerNotSerializable>(context, source, "Nested UDT, outer type Serializable, inner type not Serializable. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable"))},
{"UDT_Nested_OuterNotSerializableInnerSerializable", () =>
SerializationTests.TestUDT<UDT_Nested_OuterNotSerializableInnerSerializable>(context, source, "Nested UDT, outer type not Serializable, inner type Serializable. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable"))},
{"UDT_Nested_InnerAndOuterSerializable", () =>
SerializationTests.TestUDT<UDT_Nested_InnerAndOuterSerializable>(context, source, "Nested UDT, both outer and innter types marked Serializable.", null, 0)},
{"UDT_Nested_InnerEnum_InnerAndOuterSerializable", () =>
SerializationTests.TestUDT<UDT_Nested_InnerEnum_InnerAndOuterSerializable>(context, source, "Nested UDT, inner enum, both outer and innter types marked Serializable.", null, 0)},
{"int?", () =>
SerializationTests.TestUDT<int?>(context, source, "Nullable type as the UDT. Should be autoserialized.", null, 0)},
{"UDT_FirstLevelCircular", () =>
SerializationTests.TestUDT<UDT_FirstLevelCircular>(context, source, "Circular UDT. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
{"UDT_FirstLevelCircularArrayRef", () =>
SerializationTests.TestUDT<UDT_FirstLevelCircularArrayRef>(context, source, "Circular UDT with array reference to self. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
{"UDT_SecondLevelCircular", () =>
SerializationTests.TestUDT<UDT_SecondLevelCircular>(context, source, "Circular UDT with indirect reference to self. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
{"UDT_CircularTypeWithCustomSerializer", () =>
SerializationTests.TestUDT<UDT_CircularTypeWithCustomSerializer>(context, source, "Circular UDT with a custom serializer. Should not be rejected.", null, 0)},
{"UDT_TypeContainingCustomSerializedCircularType", () =>
SerializationTests.TestUDT<UDT_TypeContainingCustomSerializedCircularType>(context, source, "An UDT containing a field of a circular UDT with a custom serializer. Should be autoserialized.", null, 0)},
{"UDT_ObjectField", () =>
SerializationTests.TestUDT<UDT_ObjectField>(context, source, "UDT with a field of type System.Object. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"UDT_ObjectArrayField", () =>
SerializationTests.TestUDT<UDT_ObjectArrayField>(context, source, "UDT with a field of type System.Object[]. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"UDT_ObjectListField", () =>
SerializationTests.TestUDT<UDT_ObjectListField>(context, source, "UDT with a field of type List<object>. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"object", () =>
SerializationTests.TestUDT<object>(context, source, "UDT = System.Object. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"object[]", () =>
SerializationTests.TestUDT<object[]>(context, source, "UDT = System.Object[]. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"List<object>", () =>
SerializationTests.TestUDT<List<object>>(context, source, "UDT = List<object>. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
{"UDT_EmptyType", () =>
SerializationTests.TestUDT<UDT_EmptyType>(context, source, "Empty UDT. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers"))},
{"UDT_ObjectFieldAndCustomSerializer", () =>
SerializationTests.TestUDT<UDT_ObjectFieldAndCustomSerializer>(context, source, "UDT with an Object field, and a custom serializer. Should not be rejected.", null, 0)},
{"UDT_EmptyTypeWithCustomSerializer", () =>
SerializationTests.TestUDT<UDT_EmptyTypeWithCustomSerializer>(context, source, "Empty UDT with a custom serializer. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers"))},
{"UDT_PrivateFieldOfPublicType", () =>
SerializationTests.TestUDT<UDT_PrivateFieldOfPublicType>(context, source, "UDT with a private field of a public type. Should be autoserialized.", null, 0)},
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
// the generic entry method used by most tests in this file
public static bool TestUDT<TRecord>(DryadLinqContext context, IQueryable<int> source, string testMessage, Type expectedExceptionType, int expectedErrorCode)
{
string testName = testMessage;
TestLog.TestStart(testName);
bool passed = true;
// first make sure SelectFunc knows how to handle the type.
try
{
object o = SelectFunc<TRecord>(0);
}
catch
{
TestLog.Message(String.Format("SelectFunc doesn't support type {0}", typeof(TRecord).Name));
passed &= false;
goto Done;
}
try
{
var results = source.Select(x => SelectFunc<TRecord>(x));
int count = 0;
foreach (var r in results) count++;
if (count == 0)
{
TestLog.Message("...FAILED! No elements returned from query");
passed &= false;
goto Done;
}
}
catch (Exception exp)
{
if (expectedExceptionType == null)
{
TestLog.Message(String.Format("...FAILED! Caught {0} while none was expected: ", exp.GetType().Name));
passed &= false;
goto Done;
}
else if (expectedExceptionType != exp.GetType())
{
TestLog.Message(String.Format("...FAILED! Caught {0} while while {1} was expected: {0}", exp.GetType().Name, expectedExceptionType.Name));
passed &= false;
goto Done;
}
else if (exp is DryadLinqException && expectedErrorCode != 0 && ((DryadLinqException)exp).ErrorCode != expectedErrorCode)
{
TestLog.Message("...FAILED! Caught DryadLinqException but fault code is wrong.");
passed &= false;
goto Done;
}
TestLog.Message("....PASSED (caught correct exception)");
goto Done;
}
if (expectedExceptionType == null)
{
TestLog.Message(".....PASSED");
}
else
{
TestLog.Message("...FAILED! No exception was thrown from query.");
passed &= false;
}
Done:
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static TRecord SelectFunc<TRecord>(int x)
{
if (typeof(TRecord) == typeof(object))
{
return (TRecord)(new object());
}
if (typeof(TRecord) == typeof(object[]))
{
return (TRecord)((object)new object[0]);
}
if (typeof(TRecord) == typeof(List<object>))
{
return (TRecord)((object)new List<object>());
}
if (typeof(TRecord) == typeof(UDT_Undecorated))
{
return (TRecord)((object)new UDT_Undecorated(x));
}
if (typeof(TRecord) == typeof(UDT_SelfCustomSerializer))
{
return (TRecord)((object)new UDT_SelfCustomSerializer(x));
}
if (typeof(TRecord) == typeof(UDT_MarkedSerializable))
{
return (TRecord)((object)new UDT_MarkedSerializable(x));
}
if (typeof(TRecord) == typeof(UDT_StaticSerializer))
{
return (TRecord)((object)new UDT_StaticSerializer(x));
}
if (typeof(TRecord) == typeof(UDT_ExternalCustomSerializer))
{
return (TRecord)((object)new UDT_ExternalCustomSerializer(x));
}
if (typeof(TRecord) == typeof(UDT_BadCustomSerializerType1))
{
return (TRecord)((object)new UDT_BadCustomSerializerType1(x));
}
if (typeof(TRecord) == typeof(UDT_BadCustomSerializerType2))
{
return (TRecord)((object)new UDT_BadCustomSerializerType2(x));
}
if (typeof(TRecord) == typeof(UDT_BaseType))
{
return (TRecord)((object)new UDT_BaseType(x));
}
if (typeof(TRecord) == typeof(UDT_DerivedType))
{
return (TRecord)((object)new UDT_DerivedType(x));
}
if (typeof(TRecord) == typeof(UDT_DerivedType))
{
return (TRecord)((object)new UDT_DerivedType(x));
}
if (typeof(TRecord) == typeof(UDT_FieldOfNonPublicType))
{
return (TRecord)((object)new UDT_FieldOfNonPublicType(x));
}
if (typeof(TRecord) == typeof(UDT_PrivateFieldOfPublicType))
{
return (TRecord)((object)new UDT_PrivateFieldOfPublicType(x));
}
if (typeof(TRecord) == typeof(UDT_Nested_OuterSerializableInnerNotSerializable))
{
return (TRecord)((object)new UDT_Nested_OuterSerializableInnerNotSerializable(x));
}
if (typeof(TRecord) == typeof(UDT_Nested_OuterNotSerializableInnerSerializable))
{
return (TRecord)((object)new UDT_Nested_OuterNotSerializableInnerSerializable(x));
}
if (typeof(TRecord) == typeof(UDT_Nested_InnerAndOuterSerializable))
{
return (TRecord)((object)new UDT_Nested_InnerAndOuterSerializable(x));
}
if (typeof(TRecord) == typeof(UDT_Nested_InnerEnum_InnerAndOuterSerializable))
{
return (TRecord)((object)new UDT_Nested_InnerEnum_InnerAndOuterSerializable(x));
}
if (typeof(TRecord) == typeof(int?))
{
return (TRecord)((object)((int?)x));
}
if (typeof(TRecord) == typeof(UDT_FirstLevelCircular))
{
return (TRecord)((object)new UDT_FirstLevelCircular(x));
}
if (typeof(TRecord) == typeof(UDT_FirstLevelCircularArrayRef))
{
return (TRecord)((object)new UDT_FirstLevelCircularArrayRef(x));
}
if (typeof(TRecord) == typeof(UDT_SecondLevelCircular))
{
return (TRecord)((object)new UDT_SecondLevelCircular(x));
}
if (typeof(TRecord) == typeof(UDT_CircularTypeWithCustomSerializer))
{
return (TRecord)((object)new UDT_CircularTypeWithCustomSerializer(x));
}
if (typeof(TRecord) == typeof(UDT_TypeContainingCustomSerializedCircularType))
{
return (TRecord)((object)new UDT_TypeContainingCustomSerializedCircularType(x));
}
if (typeof(TRecord) == typeof(UDT_ObjectField))
{
return (TRecord)((object)new UDT_ObjectField(x));
}
if (typeof(TRecord) == typeof(UDT_ObjectArrayField))
{
return (TRecord)((object)new UDT_ObjectArrayField(x));
}
if (typeof(TRecord) == typeof(UDT_ObjectListField))
{
return (TRecord)((object)new UDT_ObjectListField(x));
}
if (typeof(TRecord) == typeof(UDT_ObjectFieldAndCustomSerializer))
{
return (TRecord)((object)new UDT_ObjectFieldAndCustomSerializer(x));
}
if (typeof(TRecord) == typeof(UDT_EmptyType))
{
return (TRecord)((object)new UDT_EmptyType(x));
}
if (typeof(TRecord) == typeof(UDT_EmptyTypeWithCustomSerializer))
{
return (TRecord)((object)new UDT_EmptyTypeWithCustomSerializer(x));
}
throw new InvalidOperationException("Unrecognized TRecord");
}
}
}

108
DryadLinqTests/SimpleTests.cs Normal file
View File

@ -0,0 +1,108 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
namespace DryadLinqTests
{
public class C1
{
public int x;
public float y;
public C1(int a, float b)
{
this.x = a;
this.y = b;
}
}
public class SimpleTests
{
public static void Run(string[] args)
{
Test1(args);
//Test2(args);
//Test3(args);
//Test4(args);
//Test5(args);
}
public static void Test1(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var lines = input.Where(x => x.Line.Contains("white"));
foreach (var x in lines) Console.WriteLine(x);
//var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
//result.SubmitAndWait();
}
public static void Test2(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var q1 = input.Where(x => x.Line.Contains("white"));
var q2 = input.Where(x => x.Line.Contains("the"));
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
DryadLinqQueryable.SubmitAndWait(res1, res2);
}
public static void Test3(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
result.SubmitAndWait();
}
public static void Test4(string[] args)
{
DryadLinqContext context = new DryadLinqContext("svc-d2-01");
var input = context.FromStore<LineRecord>("hdfs://svc-d2-01:8033/user/misard/foo.txt");
var lines = input.Where(x => x.Line.Contains("white"));
var result = lines.ToStore("hdfs://svc-d2-01:8033/user/yuanbyu/foo.txt", true);
result.SubmitAndWait();
}
public static void Test5(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1);
Uri dataUri = AzureUtils.ToAzureUri("msrsvc", "test", "testwrite");
IEnumerable<LineRecord> lines = DataProvider.ReadData<LineRecord>(context, dataUri);
foreach (var x in lines)
{
Console.WriteLine(x);
}
}
}
}

118
DryadLinqTests/TestLog.cs Normal file
View File

@ -0,0 +1,118 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
namespace DryadLinqTests
{
public class TestResult
{
public DryadLinqContext Context { get; private set; }
public string TestName { get; private set; }
public bool Passed { get; private set; }
//public string Error { get; private set; }
public TestResult(string testName, DryadLinqContext context, bool passed)
{
TestName = testName;
Passed = passed;
Context = context;
}
}
public class TestLog
{
static internal int nTestsRun;
static internal int nTestsPassed;
static internal string fileName = "";
internal static void LogInit(string path)
{
fileName = path;
// create empty file
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName))
{
}
}
// NOTE: QA test harnesses may rely on the formatting
internal static void TestStart(string testName)
{
Message(" --- Starting: [" + testName + "] --- ");
}
// NOTE: QA tests may rely on the formatting
internal static void LogResult(TestResult result)
{
nTestsRun++;
if (result.Passed)
{
nTestsPassed++;
}
Message(" * " + (result.Passed ? "Pass" : "FAIL") );
Message(" --- Completed: [" + result.TestName + "] --- ");
NewLine();
}
internal static void LogResult(bool result)
{
nTestsRun++;
if (result)
{
nTestsPassed++;
}
NewLine();
}
internal static void NewLine()
{
if (fileName.Length > 0)
{
// append data
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName, true))
{
file.WriteLine();
}
}
Console.WriteLine();
}
internal static void Message(string msg)
{
if (fileName.Length > 0)
{
// append data
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName, true))
{
file.WriteLine(msg);
}
}
Console.WriteLine(msg);
}
}
}

515
DryadLinqTests/TypesInQueryTests.cs Normal file
View File

@ -0,0 +1,515 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Text.RegularExpressions;
namespace DryadLinqTests
{
public class TypesInQueryTests
{
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" TypesInQueryTests ");
TestLog.Message(" **********************");
var tests = new Dictionary<string, Action>()
{
{"NonSealedTypeRecords", () => NonSealedTypeRecords(context) },
{"DerivedTypeRecords", () => DerivedTypeRecords(context) },
{"ObjectRecords", () => ObjectRecords(context) },
{"BadRecordsNotSerialized", () => BadRecordsNotSerialized(context) },
{"GroupByWithAnonymousTypes_Bug15675", () => GroupByWithAnonymousTypes_Bug15675(context) },
{"GroupByWithAnonymousTypes_Pipeline", () => GroupByWithAnonymousTypes_Pipeline(context) },
{"GroupByWithAnonymousTypes_MultipleAnonymousTypes", () => GroupByWithAnonymousTypes_MultipleAnonymousTypes(context) },
{"GroupByWithAnonymousTypes_GenericWithAnonTypeParam", () => GroupByWithAnonymousTypes_GenericWithAnonTypeParam(context) },
{"GroupByWithAnonymousTypes_ArrayOfAnon", () => GroupByWithAnonymousTypes_ArrayOfAnon(context) },
{"GroupByWithAnonymousTypes_NestedAnonTypes", () => GroupByWithAnonymousTypes_NestedAnonTypes(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static bool NonSealedTypeRecords(DryadLinqContext context)
{
string testName = "NonSealedTypeRecords";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<NonSealedClass>[] result = new IEnumerable<NonSealedClass>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => new NonSealedClass()).ToArray();
result[0] = output;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => new NonSealedClass()).ToArray();
result[1] = output;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool DerivedTypeRecords(DryadLinqContext context)
{
string testName = "DerivedTypeRecords";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<ParentClass>[] result1 = new IEnumerable<ParentClass>[2];
IEnumerable<ChildClass>[] result2 = new IEnumerable<ChildClass>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output1 = pt1.Select(x => new ParentClass()).ToArray();
var output2 = pt1.Select(x => new ChildClass()).ToArray();
result1[0] = output1;
result2[0] = output2;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output1 = pt1.Select(x => new ParentClass()).ToArray();
var output2 = pt1.Select(x => new ChildClass()).ToArray();
result1[1] = output1;
result2[1] = output2;
}
// compare result
try
{
Validate.Check(result1);
Validate.Check(result2);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool ObjectRecords(DryadLinqContext context)
{
string testName = "ObjectRecords";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<object>[] result = new IEnumerable<object>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => new object()).ToArray();
result[0] = output;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => new object()).ToArray();
result[1] = output;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool BadRecordsNotSerialized(DryadLinqContext context)
{
string testName = "BadRecordsNotSerialized";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<string>[] result = new IEnumerable<string>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => (x % 2 == 0 ? new ChildClass() : new ParentClass())).Select(x => x is ChildClass ? "child" : "parent").ToArray();
result[0] = output;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select(x => (x % 2 == 0 ? new ChildClass() : new ParentClass())).Select(x => x is ChildClass ? "child" : "parent").ToArray();
result[1] = output;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
//Bug 15675 -- GroupBy creates a DryadOrderByNode which is confused by the anonymous type.
// specifically, the call to DryadSort incorrectly referenced both the anonymous type and the code-gen proxy for the anonymous type.
// -- the fix was to clean up how & when we process the types involved in the query so that only the proxy would be referenced.
public static bool GroupByWithAnonymousTypes_Bug15675(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_Bug15675";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new { Num = i % 10 })
.GroupBy(x => x.Num, x => x.Num)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new { Num = i % 10 })
.GroupBy(x => x.Num, x => x.Num)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool GroupByWithAnonymousTypes_Pipeline(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_Pipeline";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new { Num = i % 10 })
.Where(x => true)
.GroupBy(x => x.Num, x => x.Num)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new { Num = i % 10 })
.Where(x => true)
.GroupBy(x => x.Num, x => x.Num)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool GroupByWithAnonymousTypes_MultipleAnonymousTypes(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_MultipleAnonymousTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new { Num = i % 10 })
.Where(x => true)
.Select(i => new { Num2 = i.Num })
.GroupBy(x => x.Num2, x => x.Num2)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new { Num = i % 10 })
.Where(x => true)
.Select(i => new { Num2 = i.Num })
.GroupBy(x => x.Num2, x => x.Num2)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool GroupByWithAnonymousTypes_GenericWithAnonTypeParam(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_GenericWithAnonTypeParam";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => MakeNewMyGenericType(new { Num = i % 10 }))
.GroupBy(x => x.Field.Num, x => x.Field.Num)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => MakeNewMyGenericType(new { Num = i % 10 }))
.GroupBy(x => x.Field.Num, x => x.Field.Num)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool GroupByWithAnonymousTypes_ArrayOfAnon(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_ArrayOfAnon";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new[] { new { Num = i % 10 } })
.GroupBy(x => x[0].Num, x => x[0].Num)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new[] { new { Num = i % 10 } })
.GroupBy(x => x[0].Num, x => x[0].Num)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool GroupByWithAnonymousTypes_NestedAnonTypes(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_NestedAnonTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new { Num = new { NumInner = new int?(i % 10) } }) // nullable-fields present particular challenge.
.GroupBy(x => x.Num.NumInner, x => x.Num.NumInner)
.ToArray();
// local
context.LocalDebug = true;
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new { Num = new { NumInner = new int?(i % 10) } }) // nullable-fields present particular challenge.
.GroupBy(x => x.Num.NumInner, x => x.Num.NumInner)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
// This is a helper method to allow 'naming the anonymous type'
// -- there is no name to give 'anonymous' inline to the original query,
// but a generic-method only has to call it by pseudonym T, which is a useful workaround.
public static MyGenericType<T> MakeNewMyGenericType<T>(T item)
{
return new MyGenericType<T>(item);
}
}
[Serializable]
public class MyGenericType<T>
{
public T Field;
public MyGenericType(T data)
{
Field = data;
}
}
[Serializable]
public class NonSealedClass
{
public int X;
}
[Serializable]
public class ParentClass
{
public int X;
}
[Serializable]
public class ChildClass : ParentClass
{
public int Y;
}
}

273
DryadLinqTests/Utils.cs
View File

@ -1,20 +1,59 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.DryadLinq.Internal;
using Microsoft.Research.Peloponnese.Storage;
using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.Blob;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
namespace DryadLinqTests namespace DryadLinqTests
{ {
public static class Config public class Config
{ {
public static string accountName = @"MyAccountName"; public static string accountName = @"";
public static string storageKey = @"MyStorageKey"; public static string storageKey = @"";
public static string containerName = @"MyContainerName"; public static string containerName = @"";
public static string cluster = "MyCcluster"; public static string cluster = @"";
public static string testLogPath = @"";
public Config(string clusterName, string container, string logPath)
{
cluster = clusterName;
AzureSubscriptions azs = new AzureSubscriptions();
Task<AzureCluster> clusterTask = azs.GetClusterAsync(clusterName);
clusterTask.Wait();
accountName = clusterTask.Result.StorageAccount;
storageKey = clusterTask.Result.StorageKey;
containerName = container;
testLogPath = logPath;
}
} }
public class DataGenerator public class DataGenerator
@ -23,6 +62,16 @@ namespace DryadLinqTests
{ {
} }
public static IQueryable<int> CreateSimpleFileSetsEx()
{
int[] input = { 0, 1, 2 };
IEnumerable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
IEnumerable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
IEnumerable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 4, 4));
return rangePartition.AsQueryable();
}
public static IEnumerable<IEnumerable<int>> CreateSimpleFileSets() public static IEnumerable<IEnumerable<int>> CreateSimpleFileSets()
{ {
IEnumerable<IEnumerable<int>> data = new int[][] IEnumerable<IEnumerable<int>> data = new int[][]
@ -46,9 +95,100 @@ namespace DryadLinqTests
return data; return data;
} }
public static IEnumerable<IEnumerable<int>> CreateRangePartitionDataSet()
{
// we need a lot of data to ensure sampler will get some data.
// A few thousand should suffice.
IEnumerable<IEnumerable<int>> data = new int[][]
{
Enumerable.Range(1,1000).ToArray(),
Enumerable.Range(20000,2000).ToArray(),
Enumerable.Range(40000,5000).ToArray(),
};
return data;
}
public static IQueryable<int> GetSimpleFileSets(DryadLinqContext context)
{
//IEnumerable<IEnumerable<int>> data = new int[][]
// {
// new[] { 0, 1, 2, 3 },
// new[] { 4, 5, 6, 7 },
// new[] { 8, 9, 10, 11},
// };
//IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
// "unittest/inputdata/SimpleFile.txt"));
IQueryable<int> input = context.FromEnumerable(new int[1]);
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
IQueryable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 4, 4));
//IQueryable<int> store = rangePartition.ToStore(@"unittest/inputdata/SimpleFile.txt");
return rangePartition;
}
public static IQueryable<int> GetGroupByReduceDataSet(DryadLinqContext context)
{
//IEnumerable<IEnumerable<int>> data = new int[][] {
// Enumerable.Range(1,100).ToArray(),
// Enumerable.Range(101,100).ToArray(),
// };
IQueryable<int> input = context.FromEnumerable(new int[1]);
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 2)); // {0, 1}
IQueryable<int> partitions = range.HashPartition(x => x, 2); // create 2 partitions
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 100 + 1, 100));
return rangePartition;
}
public static IQueryable<int> GetRangePartitionDataSet(DryadLinqContext context)
{
// we need a lot of data to ensure sampler will get some data.
// A few thousand should suffice.
//IEnumerable<IEnumerable<int>> data = new int[][] {
// Enumerable.Range(1,1000).ToArray(),
// Enumerable.Range(20000,2000).ToArray(),
// Enumerable.Range(40000,5000).ToArray(),
// };
IQueryable<int> input = context.FromEnumerable(new int[1]);
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
IQueryable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 20000 + 1, 1000));
return rangePartition;
}
} }
[Serializable]
public class ReverseComparer<T> : IComparer<T>
{
IComparer<T> _originalComparer;
public ReverseComparer(IComparer<T> originalComparer)
{
_originalComparer = originalComparer ?? Comparer<T>.Default;
}
public int Compare(T x, T y)
{
return (_originalComparer.Compare(y, x)); //note reversed order of operands
}
public override bool Equals(object obj)
{
ReverseComparer<T> objTyped = obj as ReverseComparer<T>;
return objTyped != null && _originalComparer.Equals(objTyped._originalComparer);
}
public override int GetHashCode()
{
// Modify the hash code so that it differs from the hash code for the underlying comparer.
// It would also probably be good enough to just return _originalComparer.GetHashCode().
return unchecked((_originalComparer.GetHashCode() + 123457) * 10007);
}
}
public class Utils public class Utils
{ {
public static bool DeleteFile(string accountName, string accountKey, string containerName, string fileName, bool delSubDirs) public static bool DeleteFile(string accountName, string accountKey, string containerName, string fileName, bool delSubDirs)
@ -96,15 +236,13 @@ namespace DryadLinqTests
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob; containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
container.SetPermissions(containerPermissions); container.SetPermissions(containerPermissions);
CloudBlockBlob remoteFile = container.GetBlockBlobReference(fileName); IEnumerable<IListBlobItem> files = container.ListBlobs(fileName, true);
if (!remoteFile.Exists()) return (files.Count() > 0);
return false;
} }
catch (Exception) catch (Exception)
{ {
return false; return false;
} }
return true;
} }
internal static DryadLinqContext MakeBasicConfig(string cluster) //??? internal static DryadLinqContext MakeBasicConfig(string cluster) //???
@ -153,5 +291,120 @@ namespace DryadLinqTests
} }
return context; return context;
} }
internal static DryadLinqRecordReader<TRecord> MakeDryadRecordReader<TRecord>(DryadLinqContext context, string readPath)
{
DryadLinqFactory<TRecord> factory = (DryadLinqFactory<TRecord>)DryadLinqCodeGen.GetFactory(context, typeof(TRecord));
NativeBlockStream nativeStream = ReflectionHelper.CreateDryadLinqFileStream(readPath, FileMode.Open, FileAccess.Read);
// ??? NativeBlockStream nativeStream = ReflectionHelper.CreateDryadLinqFileStream(readPath, FileMode.Open, FileAccess.Read, DscCompressionScheme.None);
DryadLinqRecordReader<TRecord> reader = factory.MakeReader(nativeStream);
return reader;
}
}
public class Validate
{
public static void
Check<T>(
IEnumerable<T>[] ss,
IComparer<T> comparer = null,
bool sort = true,
bool verbose = false,
IComparer<T> sortcomparer = null
)
{
if (ss.Length == 0) return;
if (comparer == null)
{
comparer = Comparer<T>.Default;
if (comparer == null)
{
throw new ArgumentNullException("Can't not be null.");
}
}
if (sortcomparer == null)
sortcomparer = comparer;
T[][] aa = new T[ss.Length][];
for (int i = 0; i < aa.Length; i++)
{
aa[i] = ss[i].ToArray();
if (sort) Array.Sort(aa[i], sortcomparer);
}
int len = aa[0].Length;
for (int i = 1; i < aa.Length; i++)
{
if (aa[i].Length != len)
{
throw new Exception("Wrong number of elements.");
}
}
for (int i = 0; i < len; i++)
{
T elem = aa[0][i];
for (int j = 1; j < aa.Length; j++)
{
if (verbose)
{
//TestOutput.WriteLine("Comparing {0} to {1}", elem.ToString(), aa[j][i].ToString());
}
if (comparer.Compare(elem, aa[j][i]) != 0)
{
throw new Exception("Elements failed to match: " + elem + " != " + aa[j][i]);
}
}
}
}
internal static bool outFileExists(string outFile)
{
try
{
return Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (Exception)
{
return false;
}
}
}
public static class ReflectionHelper
{
/// <summary>
/// Create DryadLinqFileStream object via reflection
/// </summary>
///<param name="parameters"></param>
/// <returns></returns>
public static NativeBlockStream CreateDryadLinqFileStream(params object[] parameters)
{
return Assembly.LoadWithPartialName("Microsoft.Hpc.Linq").GetType("Microsoft.Hpc.Linq.Internal.HpcLinqFileStream") //???
.GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, parameters.Select(p => p.GetType()).ToArray(), null)
.Invoke(parameters) as NativeBlockStream;
}
private static Type s_errorCodeType = null;
public static int GetDryadLinqErrorCode(string name)
{
if (s_errorCodeType == null)
{
Assembly asm = Assembly.Load("Microsoft.Research.DryadLinq");
Type[] types = asm.GetTypes();
foreach (var t in types)
{
if (t.Name == "DryadLinqErrorCode")
{
s_errorCodeType = t;
break;
}
}
}
var finfo = s_errorCodeType.GetField(name);
return (int)finfo.GetValue(null);
}
} }
} }

267
DryadLinqTests/Validate.cs
View File

@ -1,267 +0,0 @@
///-------------------------------------------------------------------------------------------------
// file: Validate.cs
//
// summary: Implements the validate class
///-------------------------------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
namespace BenchmarkFramework {
///-------------------------------------------------------------------------------------------------
/// <summary> Validation utils </summary>
///-------------------------------------------------------------------------------------------------
public class Validate {
public static void
Check<T>(
IEnumerable<T>[] ss,
IComparer<T> comparer = null,
bool sort = true,
bool verbose = false,
IComparer<T> sortcomparer = null
) {
if(ss.Length == 0) return;
if(comparer == null) {
comparer = Comparer<T>.Default;
if(comparer == null) {
throw new ArgumentNullException("Can't not be null.");
}
}
if(sortcomparer == null)
sortcomparer = comparer;
T[][] aa = new T[ss.Length][];
for(int i = 0; i < aa.Length; i++) {
aa[i] = ss[i].ToArray();
if(sort) Array.Sort(aa[i], sortcomparer);
}
int len = aa[0].Length;
for(int i = 1; i < aa.Length; i++) {
if(aa[i].Length != len) {
throw new Exception("Wrong number of elements.");
}
}
for(int i = 0; i < len; i++) {
T elem = aa[0][i];
for(int j = 1; j < aa.Length; j++) {
if(verbose) {
//TestOutput.WriteLine("Comparing {0} to {1}", elem.ToString(), aa[j][i].ToString());
}
if(comparer.Compare(elem, aa[j][i]) != 0) {
throw new Exception("Elements failed to match: " + elem + " != " + aa[j][i]);
}
}
}
}
//public static void
//GroupCheck<K, V>(
// IEnumerable<IGrouping<K, V>>[] ss,
// IComparer<K> kComparer = null,
// IComparer<V> vComparer = null
// ) {
// if(ss.Length == 0) return;
// if(kComparer == null) {
// kComparer = Comparer<K>.Default;
// if(kComparer == null) {
// throw new ArgumentNullException("Can't not be null.");
// }
// }
// if(vComparer == null) {
// vComparer = Comparer<V>.Default;
// if(vComparer == null) {
// throw new ArgumentNullException("Can't not be null.");
// }
// }
// IGrouping<K, V>[][] aa = new IGrouping<K, V>[ss.Length][];
// for(int i = 0; i < aa.Length; i++) {
// aa[i] = ss[i].ToArray();
// K[] keys = aa[i].Select(x => x.Key).ToArray();
// Array.Sort(keys, aa[i], kComparer);
// }
// int len = aa[0].Length;
// for(int i = 1; i < aa.Length; i++) {
// if(aa[i].Length != len) {
// throw new Exception("Wrong number of elements.");
// }
// }
// for(int i = 0; i < len; i++) {
// IEnumerable<V> elem = aa[0][i];
// for(int j = 1; j < aa.Length; j++) {
// Check(new IEnumerable<V>[] { elem, aa[j][i] }, vComparer);
// }
// }
//}
}
/*
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantDoubleComparer : IComparer<double> {
Double EPSILON;
public TolerantDoubleComparer(Double _epsilon = 0.000001f) {
EPSILON = _epsilon;
}
public int Compare(Double a, Double b) {
Double delta = a - b;
if(Math.Abs(delta) <= EPSILON)
return 0;
return delta < 0.0f ? -1 : 1;
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantFloatComparer : IComparer<float> {
float EPSILON;
public TolerantFloatComparer(float _epsilon = 0.000001f) {
EPSILON = _epsilon;
}
public int Compare(float a, float b) {
float delta = a - b;
if(Math.Abs(delta) <= EPSILON)
return 0;
return delta < 0.0f ? -1 : 1;
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantVectorComparer : IComparer<Vector> {
float EPSILON;
public TolerantVectorComparer(float _epsilon = 0.0001f) {
EPSILON = _epsilon;
}
public int Compare(Vector a, Vector b) {
for(int i = 0; i < a.m_elems.Length; i++) {
float delta = a.m_elems[i] - b.m_elems[i];
if(Math.Abs(delta) > EPSILON)
return delta < 0.0f ? -1 : 1;
}
return 0;
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Interface for epsilon comparable single. </summary>
///
/// <remarks> Crossbac, 1/16/2014. </remarks>
///
/// <typeparam name="T"> Generic type parameter. </typeparam>
///-------------------------------------------------------------------------------------------------
public interface IEpsilonComparableSingle<T> {
int EpsilonCompare(T a, T b, float epsilon);
}
///-------------------------------------------------------------------------------------------------
/// <summary> Interface for epsilon comparable double. </summary>
///
/// <remarks> Crossbac, 1/16/2014. </remarks>
///
/// <typeparam name="T"> Generic type parameter. </typeparam>
///-------------------------------------------------------------------------------------------------
public interface IEpsilonComparableDouble<T> {
int EpsilonCompare(T a, T b, double epsilon);
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class EpsilonComparer<T> : IComparer<T> where T : IEpsilonComparableSingle<T> {
float EPSILON;
public EpsilonComparer(float _epsilon = 0.0001f) {
EPSILON = _epsilon;
}
public int Compare(T a, T b) {
return a.EpsilonCompare(a, b, EPSILON);
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer for images. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantImageComparer : IComparer<Image> {
float EPSILON;
public TolerantImageComparer(float _epsilon = 0.0001f) {
EPSILON = _epsilon;
}
public int Compare(Image a, Image b) {
for(int i = 0; i < a.m_elems.Length; i++) {
float delta = a.m_elems[i] - b.m_elems[i];
if(Math.Abs(delta) > EPSILON)
return delta < 0.0f ? -1 : 1;
}
return 0;
}
}
/// <summary>
/// Compare two instances of Pair<int,int>.
/// Implement for concrete type as Pair<T1,T2> cannot straighforwardly implement IComparable.
/// </summary>
///
/// <remarks> jcurrey, 3/11/2013. </remarks>
public class PairIntIntComparer : IComparer<Pair<int, int>> {
public int Compare(Pair<int, int> a, Pair<int, int> b) {
int keyComparison = a.Key.CompareTo(b.Key);
if(keyComparison == 0) {
return a.Value.CompareTo(b.Value);
} else {
return keyComparison;
}
}
}
*/
}

78
DryadLinqTests/WordCount.cs Normal file
View File

@ -0,0 +1,78 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
// comment the following line to run on Azure
#define local
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
namespace DryadLinqTests
{
public class WordCount
{
public static void WordCountExample()
{
#if local
// This overload runs the computation on your local computer using a single worker
var config = new DryadLinqContext(1);
var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
// You can create inputs from any IEnumerable source using this method
var input = config.FromEnumerable(lines);
#else
string clusterName = "Replace with your HDInsight 3.0 cluster name";
// to use the davinci.txt example input below, select your cluster's default
// storage account and container, which automatically includes the sample text
string accountName = "Replace with a storage account name";
string containerName = "Replace with a storage container name";
// This overload creates an Azure-based computation
var config = new DryadLinqContext(clusterName);
config.JobFriendlyName = "DryadLINQ Sample Wordcount";
// plain text files should be read as type LineRecord
var input = config.FromStore<LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
"example/data/gutenberg/davinci.txt"));
#endif
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
#if local
// any collection computed by the query can be materialized back at the client,
// not just the 'output' collection. For large collections this is expensive!
foreach (LineRecord line in toOutput)
{
Console.WriteLine(line.Line);
}
#else
// the 'true' parameter to ToStore means the output will be over-written if you run
// the job more than once
var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
"wc-out.txt"), true).SubmitAndWait();
#endif
}
}
}

24
DryadLinqTests/packages.config
View File

@ -1,21 +1,23 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" /> <package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" /> <package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" /> <package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" /> <package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" /> <package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Dryad" version="0.1.4-beta001" targetFramework="net45" />
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" /> <package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" /> <package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" /> <package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages> </packages>

37
DryadVertex/VertexHost/system/channel/channel.vcxproj
View File

@ -1,6 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -24,13 +25,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -54,6 +55,13 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<NuGetPackageImportStamp>b0b5ea03</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl> <Midl>
@ -61,7 +69,7 @@
</Midl> </Midl>
<ClCompile> <ClCompile>
<Optimization>Disabled</Optimization> <Optimization>Disabled</Optimization>
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\..\peloponnese\HadoopBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>WIN32;_DEBUG;_LIB;WIN32_LEAN_AND_MEAN;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;WIN32_LEAN_AND_MEAN;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<MinimalRebuild>false</MinimalRebuild> <MinimalRebuild>false</MinimalRebuild>
<BasicRuntimeChecks>Default</BasicRuntimeChecks> <BasicRuntimeChecks>Default</BasicRuntimeChecks>
@ -72,10 +80,6 @@
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat> <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories> <AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile> </ClCompile>
<Lib>
<AdditionalDependencies>Microsoft.Research.Peloponnese.HadoopBridge.lib</AdditionalDependencies>
<AdditionalLibraryDirectories>..\..\..\..\..\peloponnese\bin\$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Midl> <Midl>
@ -88,13 +92,9 @@
</PrecompiledHeader> </PrecompiledHeader>
<WarningLevel>Level3</WarningLevel> <WarningLevel>Level3</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat> <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\..\peloponnese\HadoopBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories> <AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile> </ClCompile>
<Lib>
<AdditionalDependencies>Microsoft.Research.Peloponnese.HadoopBridge.lib</AdditionalDependencies>
<AdditionalLibraryDirectories>..\..\..\..\..\peloponnese\bin\$(Configuration)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemGroup> <ItemGroup>
<ClInclude Include="include\channelbuffer.h" /> <ClInclude Include="include\channelbuffer.h" />
@ -149,13 +149,16 @@
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
</ImportGroup> </ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
</Target> </Target>
</Project> </Project>

3
DryadVertex/VertexHost/system/channel/packages.config
View File

@ -1,4 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" /> <package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
</packages> </packages>

48
DryadVertex/VertexHost/system/channel/src/channelbufferhdfs.cpp
View File

@ -929,6 +929,37 @@ RChannelBufferHdfsWriter::RChannelBufferHdfsWriter(const char* uri)
Hdfs::InstanceAccessor ia(bridge); Hdfs::InstanceAccessor ia(bridge);
ia.Discard(); ia.Discard();
} }
char envUserName[128];
DWORD ret = ::GetEnvironmentVariableA("USER", envUserName, sizeof(envUserName));
if (ret > 0 && ret < sizeof(envUserName)-1)
{
m_user.Set(envUserName);
DrLogI("Hdfs writer set user to %s from environment", m_user.GetString());
}
else
{
if (ret == 0 && GetLastError() != ERROR_ENVVAR_NOT_FOUND)
{
DrLogW("Can't get USER environment variable: %s", DRERRORSTRING(DrGetLastError()));
}
if (ret >= sizeof(envUserName)-1)
{
DrLogW("USER environment is too long: %d", ret);
}
DWORD bufSize = sizeof(envUserName);
BOOL bRet = ::GetUserNameA(envUserName, &bufSize);
if (bRet == 0)
{
DrLogW("Can't get username: %s", DRERRORSTRING(DrGetLastError()));
}
else
{
m_user.Set(envUserName);
DrLogI("Hdfs writer set user to %s from username", m_user.GetString());
}
}
} }
} }
} }
@ -1021,7 +1052,7 @@ bool RChannelBufferHdfsWriter::Open(Hdfs::Instance** pInstance,
Hdfs::InstanceAccessor ia(instance); Hdfs::InstanceAccessor ia(instance);
Hdfs::Writer* writer; Hdfs::Writer* writer;
bool openedWriter = ia.OpenWriter(filePath.GetString(), false, &writer); bool openedWriter = ia.OpenCreate(filePath.GetString(), 1024 * 1024, -1, &writer);
if (!openedWriter) if (!openedWriter)
{ {
char* errorMsg = ia.GetExceptionMessage(); char* errorMsg = ia.GetExceptionMessage();
@ -1036,6 +1067,21 @@ bool RChannelBufferHdfsWriter::Open(Hdfs::Instance** pInstance,
ia.Discard(); ia.Discard();
return false; return false;
} }
bool setPermissions = ia.SetOwnerAndPermission(filePath.GetString(), m_user.GetString(), NULL, 0644);
if (!setPermissions)
{
char* errorMsg = ia.GetExceptionMessage();
DrStr64 description;
description.SetF("Can't set HDFS file permissions '%s': %s",
m_uri.GetString(), errorMsg);
HadoopNative::DisposeString(errorMsg);
m_completionItem.Attach(MakeErrorItem(DryadError_ChannelOpenError,
description.GetString()));
ia.Discard();
return false;
}
DrLogI("Opened Hdfs writer for %s", m_uri.GetString()); DrLogI("Opened Hdfs writer for %s", m_uri.GetString());

1
DryadVertex/VertexHost/system/channel/src/channelbufferhdfs.h
View File

@ -149,6 +149,7 @@ private:
bool Open(Hdfs::Instance** pInstance, Hdfs::Writer** pWriter); bool Open(Hdfs::Instance** pInstance, Hdfs::Writer** pWriter);
bool AddToQueue(WriteEntry* entry); bool AddToQueue(WriteEntry* entry);
DrStr64 m_user;
DrStr64 m_uri; DrStr64 m_uri;
DryadBList<WriteEntry> m_queue; DryadBList<WriteEntry> m_queue;
UInt32 m_queueLength; UInt32 m_queueLength;

2
DryadVertex/VertexHost/system/channel/src/channelbuffernativewriter.cpp
View File

@ -2205,7 +2205,7 @@ void RChannelBufferWriterNativeFile::EagerCloseFile()
} }
else else
{ {
DrLogI( "Set final file pointer"); DrLogI( "Set final file pointer. Length %I64u", finalLength.QuadPart);
} }
ok = SetEndOfFile(m_bufferedFileHandle); ok = SetEndOfFile(m_bufferedFileHandle);

10
DryadVertex/VertexHost/system/classlib/classlib.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -21,13 +21,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>false</CLRSupport> <CLRSupport>false</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>false</CLRSupport> <CLRSupport>false</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -150,4 +150,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
</ImportGroup> </ImportGroup>
</Project> </Project>

10
DryadVertex/VertexHost/system/common/common.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -19,14 +19,14 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet> <CharacterSet>MultiByte</CharacterSet>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>MultiByte</CharacterSet> <CharacterSet>MultiByte</CharacterSet>
<CLRSupport>false</CLRSupport> <CLRSupport>false</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -138,4 +138,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
</ImportGroup> </ImportGroup>
</Project> </Project>

37
DryadVertex/VertexHost/system/dprocess/dprocess.vcxproj
View File

@ -1,6 +1,8 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -22,7 +24,7 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
</PropertyGroup> </PropertyGroup>
@ -30,7 +32,7 @@
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -54,6 +56,13 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<NuGetPackageImportStamp>7f5f48b5</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl> <Midl>
@ -72,10 +81,6 @@
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat> <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories> <AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile> </ClCompile>
<Lib>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Midl> <Midl>
@ -91,10 +96,6 @@
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;..\channel\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;..\channel\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories> <AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile> </ClCompile>
<Lib>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemGroup> <ItemGroup>
<ClInclude Include="include\dryadvertex.h" /> <ClInclude Include="include\dryadvertex.h" />
@ -123,13 +124,19 @@
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
</ImportGroup> </ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
</Target> </Target>
</Project> </Project>

4
DryadVertex/VertexHost/system/dprocess/packages.config
View File

@ -1,4 +1,6 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" /> <package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="Native" />
</packages> </packages>

5
DryadVertex/VertexHost/system/dprocess/src/vertexfactory.cpp
View File

@ -277,7 +277,6 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
DryadVertexProgramRef* pProgram) DryadVertexProgramRef* pProgram)
{ {
DryadVertexProgramBase* program; DryadVertexProgramBase* program;
// //
// If factory is not supplied, try to get it from factory registry. // If factory is not supplied, try to get it from factory registry.
// If still unable, fail with error. // If still unable, fail with error.
@ -294,6 +293,7 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
"Factory Registry called with no arguments"); "Factory Registry called with no arguments");
return DryadError_VertexInitialization; return DryadError_VertexInitialization;
} }
// //
// Get vertex factory. If one cannot be found, report initialization error // Get vertex factory. If one cannot be found, report initialization error
@ -301,6 +301,7 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
factory = LookUpFactory(argumentList[0]); factory = LookUpFactory(argumentList[0]);
if (factory == NULL) if (factory == NULL)
{ {
DrLogW("Factory Registry called with unknown factory UID %s.", argumentList[0].GetString());
DrStr128 errorString; DrStr128 errorString;
errorString.SetF("Factory Registry called with unknown factory UID %s", errorString.SetF("Factory Registry called with unknown factory UID %s",
argumentList[0].GetString()); argumentList[0].GetString());
@ -309,7 +310,7 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
errorString); errorString);
return DryadError_VertexInitialization; return DryadError_VertexInitialization;
} }
// //
// report new vertex creation // report new vertex creation
// //

16
DryadVertex/VertexHost/system/managedchannel/AzureReader.cs
View File

@ -27,7 +27,7 @@ using System.Runtime.InteropServices;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Azure;
namespace Microsoft.Research.Dryad.Channel namespace Microsoft.Research.Dryad.Channel
{ {
@ -74,7 +74,7 @@ namespace Microsoft.Research.Dryad.Channel
{ {
Log.LogInformation("Opening read for " + source.AbsoluteUri); Log.LogInformation("Opening read for " + source.AbsoluteUri);
string account, key, container, blobName; string account, key, container, blobName;
AzureUtils.FromAzureUri(source, out account, out key, out container, out blobName); Utils.FromAzureUri(source, out account, out key, out container, out blobName);
client = new AzureDfsClient(account, key, container, false, new PeloponneseLogger(Log.Logger)); client = new AzureDfsClient(account, key, container, false, new PeloponneseLogger(Log.Logger));
client.SetParallelThreadCount(4); client.SetParallelThreadCount(4);
@ -94,7 +94,7 @@ namespace Microsoft.Research.Dryad.Channel
private async Task OpenBlob() private async Task OpenBlob()
{ {
string account, key, container, blobName; string account, key, container, blobName;
AzureUtils.FromAzureUri(source, out account, out key, out container, out blobName); Utils.FromAzureUri(source, out account, out key, out container, out blobName);
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(source.Query); NameValueCollection query = System.Web.HttpUtility.ParseQueryString(source.Query);
@ -105,9 +105,11 @@ namespace Microsoft.Research.Dryad.Channel
++blobIndex; ++blobIndex;
} }
Log.LogInformation("Opening read for blob " + blobName); Uri blobUri = Utils.ToAzureUri(account, container, blobName, null, key);
Log.LogInformation("Opening read for blob " + blobUri.AbsoluteUri);
readStream = (await client.GetDfsFileStreamAsync(blobName, ExecutionTimeout, new PeloponneseLogger(Log.Logger))).Stream; readStream = (await client.GetDfsStreamReaderAsync(blobUri, ExecutionTimeout, new PeloponneseLogger(Log.Logger)));
long offset = -1; long offset = -1;
if (query["offset"] != null) if (query["offset"] != null)
@ -215,7 +217,7 @@ namespace Microsoft.Research.Dryad.Channel
int toRead = (int)Math.Min((long)managedBuffer.Length, bytesToRead); int toRead = (int)Math.Min((long)managedBuffer.Length, bytesToRead);
Log.LogInformation("About to read buffer length " + toRead); Log.LogInformation("About to read buffer length " + toRead);
readData.nRead = await AzureUtils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () => readData.nRead = await Utils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
{ {
return await readStream.ReadAsync(managedBuffer, 0, toRead); return await readStream.ReadAsync(managedBuffer, 0, toRead);
}); });
@ -301,7 +303,7 @@ namespace Microsoft.Research.Dryad.Channel
{ {
int toRead = (int)Math.Min(buffer.LongLength, endOffset - currentOffset); int toRead = (int)Math.Min(buffer.LongLength, endOffset - currentOffset);
int nRead = await AzureUtils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () => int nRead = await Utils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
{ {
return await readStream.ReadAsync(buffer, 0, toRead); return await readStream.ReadAsync(buffer, 0, toRead);
}); });

2
DryadVertex/VertexHost/system/managedchannel/AzureWriter.cs
View File

@ -25,7 +25,7 @@ using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Azure;
namespace Microsoft.Research.Dryad.Channel namespace Microsoft.Research.Dryad.Channel
{ {

26
DryadVertex/VertexHost/system/managedchannel/Properties/AssemblyInfo.cs
View File

@ -24,32 +24,8 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following // General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information // set of attributes. Change these attribute values to modify the information
// associated with an assembly. // associated with an assembly.
[assembly: AssemblyTitle("managedchannel")] [assembly: AssemblyTitle("Microsoft.Research.Dryad.ManagedChannel")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("managedchannel")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM // The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("f806a093-c5ae-4752-82d3-26bcc2341b29")] [assembly: Guid("f806a093-c5ae-4752-82d3-26bcc2341b29")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.3.0")]
[assembly: AssemblyFileVersion("0.1.3.0")]

28
DryadVertex/VertexHost/system/managedchannel/app.config
View File

@ -4,24 +4,44 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" /> <bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" /> <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.HDInsight.Threading" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.0" newVersion="1.2.1.0" />
</dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>
</configuration> </configuration>

109
DryadVertex/VertexHost/system/managedchannel/managedchannel.csproj
View File

@ -1,6 +1,8 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -9,11 +11,12 @@
<OutputType>Library</OutputType> <OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Dryad.Channel</RootNamespace> <RootNamespace>Microsoft.Research.Dryad.Channel</RootNamespace>
<AssemblyName>DryadManagedChannel</AssemblyName> <AssemblyName>Microsoft.Research.Dryad.ManagedChannel</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir> <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir>
<RestorePackages>true</RestorePackages> <RestorePackages>true</RestorePackages>
<NuGetPackageImportStamp>72e7670e</NuGetPackageImportStamp>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
@ -35,98 +38,38 @@
<AllowUnsafeBlocks>false</AllowUnsafeBlocks> <AllowUnsafeBlocks>false</AllowUnsafeBlocks>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath> <HintPath>..\..\..\..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath> <HintPath>..\..\..\..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath> <HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath> <HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath> <HintPath>..\..\..\..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference> </Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL"> <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath> <HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference> </Reference>
<Reference Include="System" /> <Reference Include="System" />
<Reference Include="System.Core" /> <Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" /> <Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" /> <Reference Include="System.Net" />
<Reference Include="System.Net.Http" /> <Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" /> <Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath> <HintPath>..\..\..\..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference> </Reference>
<Reference Include="System.Web" /> <Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" /> <Reference Include="System.Xml.Linq" />
@ -136,6 +79,9 @@
<Reference Include="System.Xml" /> <Reference Include="System.Xml" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Compile Include="..\..\..\..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="AzureReader.cs" /> <Compile Include="AzureReader.cs" />
<Compile Include="AzureWriter.cs" /> <Compile Include="AzureWriter.cs" />
<Compile Include="FileWriter.cs" /> <Compile Include="FileWriter.cs" />
@ -151,19 +97,20 @@
<None Include="packages.config" /> <None Include="packages.config" />
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
</Target> </Target>
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild"> <Target Name="BeforeBuild">

26
DryadVertex/VertexHost/system/managedchannel/packages.config
View File

@ -1,21 +1,13 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" /> <package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" /> <package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" /> <package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" /> <package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" /> <package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" /> <package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
</packages> </packages>

43
DryadVertex/VertexHost/vertex/VertexHostMain/Program.cs Normal file
View File

@ -0,0 +1,43 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace VertexHostMain
{
internal class ConfigDependency : Microsoft.Research.Peloponnese.Shared.AssemblyDependencyAttribute
{
public
ConfigDependency()
: base("Microsoft.Research.Dryad.VertexHost.exe.config", false)
{
}
}
class Program
{
static int Main(string[] args)
{
return VertexHost.Main(args);
}
}
}

32
DryadVertex/VertexHost/vertex/VertexHostMain/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,32 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
[assembly: VertexHostMain.ConfigDependency()]
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Microsoft.Research.Dryad.VertexHost")]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("5534b858-8472-4dc3-8f24-5b59f8805156")]

79
DryadVertex/VertexHost/vertex/VertexHostMain/VertexHostMain.csproj Normal file
View File

@ -0,0 +1,79 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>VertexHostMain</RootNamespace>
<AssemblyName>Microsoft.Research.Dryad.VertexHost</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
<NuGetPackageImportStamp>fb5afcd2</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>..\..\..\..\bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
<OutputPath>..\..\..\..\bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\..\..\..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\vertexHost\VertexHost.vcxproj">
<Project>{0cf3d1d5-9bbe-4175-979b-ec6138ef4f37}</Project>
<Name>VertexHostBody</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
</Target>
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

48
DryadVertex/VertexHost/vertex/VertexHostMain/app.config Normal file
View File

@ -0,0 +1,48 @@
<?xml version="1.0"?>
<configuration>
<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/></startup>
<runtime>
<gcServer enabled="true"></gcServer>
<gcConcurrent enabled="false"></gcConcurrent>
<generatePublisherEvidence enabled="false"/>
<gcAllowVeryLargeObjects enabled="true" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

4
DryadVertex/VertexHost/vertex/VertexHostMain/packages.config Normal file
View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
</packages>

10
DryadVertex/VertexHost/vertex/WrapperNativeInfo/WrapperNativeInfo.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -21,13 +21,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -100,4 +100,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
</ImportGroup> </ImportGroup>
</Project> </Project>

2
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/DryadLINQNativeChannels.def
View File

@ -1,4 +1,4 @@
LIBRARY "DryadLINQNativeChannels.dll" LIBRARY "Microsoft.Research.Dryad.DryadLinq.NativeWrapper.dll"
EXPORTS EXPORTS
GetTotalLength GetTotalLength

15
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/WrapperNativeInfoDll.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -19,14 +19,14 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -52,8 +52,8 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">DryadLINQNativeChannels</TargetName> <TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad.DryadLinq.NativeWrapper</TargetName>
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">DryadLINQNativeChannels</TargetName> <TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad.DryadLinq.NativeWrapper</TargetName>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl> <Midl>
@ -111,6 +111,7 @@
<ClInclude Include="stdafx.h" /> <ClInclude Include="stdafx.h" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ClCompile Include="version.cpp" />
<ClCompile Include="wrappernativeinfostubs.cpp" /> <ClCompile Include="wrappernativeinfostubs.cpp" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
@ -119,4 +120,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
</ImportGroup> </ImportGroup>
</Project> </Project>

3
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/WrapperNativeInfoDll.vcxproj.filters
View File

@ -23,6 +23,9 @@
<ClCompile Include="wrappernativeinfostubs.cpp"> <ClCompile Include="wrappernativeinfostubs.cpp">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="version.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="DryadLINQNativeChannels.def"> <None Include="DryadLINQNativeChannels.def">

40
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/version.cpp Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
#ifdef _MANAGED
#pragma managed
using namespace System::Reflection;
using namespace System::Runtime::InteropServices;
[assembly: AssemblyTitle("Microsoft.Research.Dryad.DryadLinq.NativeWrapper")];
[assembly: AssemblyDescription("")];
[assembly: AssemblyConfiguration("")];
[assembly: AssemblyCompany("Microsoft Corporation")];
[assembly: AssemblyProduct("Microsoft.Research.Dryad.DryadLinq.NativeWrapper")];
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
[assembly: AssemblyTrademark("")];
[assembly: AssemblyCulture("")];
[assembly: ComVisible(false)];
[assembly: AssemblyVersion("0.2.1.0")];
[assembly: AssemblyFileVersion("0.2.1.0")];
#endif

50
DryadVertex/VertexHost/vertex/managedwrappervertex/ManagedWrapperVertex.cpp
View File

@ -28,7 +28,6 @@ limitations under the License.
#include <mscoree.h> #include <mscoree.h>
#include <wrappernativeinfo.h> #include <wrappernativeinfo.h>
#pragma managed #pragma managed
#pragma warning(disable:4947) // so that we can use Assembly::LoadWithPartialName() #pragma warning(disable:4947) // so that we can use Assembly::LoadWithPartialName()
@ -198,18 +197,25 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
logName = System::IO::Path::Combine(logDirectory, logName); logName = System::IO::Path::Combine(logDirectory, logName);
} }
int threadsPerWorker = 1;
System::String^ threadsPerWorkerStr = System::Environment::GetEnvironmentVariable("DRYAD_THREADS_PER_WORKER");
if (threadsPerWorkerStr != nullptr)
{
threadsPerWorker = Int32::Parse(threadsPerWorkerStr);
}
DrLogI("ManagedWrapperVertex: threadsPerWorker %u", threadsPerWorker);
DrLogI("ManagedWrapperVertex: %p %u %u", nativeInfo, numberOfInputChannels, numberOfOutputChannels); DrLogI("ManagedWrapperVertex: %p %u %u", nativeInfo, numberOfInputChannels, numberOfOutputChannels);
DrLogI("ManagedWrapperVertex: Calling %s.%s", GetArgument(2), GetArgument(3)); DrLogI("ManagedWrapperVertex: Calling %s.%s", GetArgument(2), GetArgument(3));
DrLogging::FlushLog(); DrLogging::FlushLog();
DrStr128 errorMsg; DrStr128 errorMsg;
DrError error; DrError error;
{ {
// //
// Instead of invoking the vertex entry point directly from here, we delegate it to the bridge method in the Microsoft.Hpc.Linq assembly, specifically: // Instead of invoking the vertex entry point directly from here, we delegate it to the bridge method in the Microsoft.Research.DryadLinq assembly, specifically:
// static int Microsoft.Hpc.Linq.Internal.VertexEnv.VertexBridge(string vertexBridgeArgs) // static int Microsoft.Research.DryadLinq.Internal.VertexEnv.VertexBridge(string vertexBridgeArgs)
// //
// This indirect method of invoking the vertex entry point is used so that any type load / assembly load problems coming from user code // This indirect method of invoking the vertex entry point is used so that any type load / assembly load problems coming from user code
// can be caught and reported with full details using the same mechanism that other vertex failures go through (exception dumped into vertexexception.txt etc.) // can be caught and reported with full details using the same mechanism that other vertex failures go through (exception dumped into vertexexception.txt etc.)
@ -217,8 +223,10 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
// The format of vertexBridgeArgs is simply a comma separated string packing vertex assembly, class, method name, and the *actual* vertex method args (==the native channel string) // The format of vertexBridgeArgs is simply a comma separated string packing vertex assembly, class, method name, and the *actual* vertex method args (==the native channel string)
// L"<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>" // L"<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
// //
System::String ^bridgeAssemblyPartialName = gcnew System::String(L"Microsoft.Research.DryadLinq"); System::String^ classFullName = gcnew System::String(GetArgument(2));
System::String ^bridgeClassName = gcnew System::String(L"Microsoft.Research.DryadLinq.Internal.VertexEnv"); System::String^ assemblyName = classFullName->Substring(0, classFullName->LastIndexOf('.'));
System::String ^bridgeAssemblyName = gcnew System::String(assemblyName);
System::String ^bridgeClassName = gcnew System::String(assemblyName + ".Internal.VertexEnv");
System::String ^bridgeMethodName = gcnew System::String(L"VertexBridge"); System::String ^bridgeMethodName = gcnew System::String(L"VertexBridge");
// //
@ -235,21 +243,19 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
vertexMethodArgs->Append(gcnew System::String(arg.GetString())); vertexMethodArgs->Append(gcnew System::String(arg.GetString()));
} }
// //
// Get assembly path, class name, and method name, and construct the vertex bridge args with the following format: // Get assembly path, class name, and method name, and construct the vertex bridge args with the following format:
// "<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>" // "<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
// //
System::Text::StringBuilder ^vertexBridgeArg = gcnew System::Text::StringBuilder(); System::Text::StringBuilder ^vertexBridgeArg = gcnew System::Text::StringBuilder();
vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"c:\\HpcTemp\\user\\jobID\\Microsoft.Hpc.Linq0.dll"; vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"\\HpcTemp\\user\\jobID\\Microsoft.Research.DryadLinq0.dll";
vertexBridgeArg->Append(","); vertexBridgeArg->Append(",");
vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Hpc.Linq.HpcLinq__Vertex"; vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Research.DryadLinq.DryadLinq__Vertex";
vertexBridgeArg->Append(","); vertexBridgeArg->Append(",");
vertexBridgeArg->Append(gcnew System::String(GetArgument(3))); // vertex entry method name L"Select__1"; vertexBridgeArg->Append(gcnew System::String(GetArgument(3))); // vertex entry method name L"Select__1";
vertexBridgeArg->Append(","); vertexBridgeArg->Append(",");
vertexBridgeArg->Append(vertexMethodArgs->ToString()); vertexBridgeArg->Append(vertexMethodArgs->ToString());
DrLogI("ManagedWrapperVertex: Calling into Vertex Bridge to invoke Vertex Entry: %s", GetArgument(3)); DrLogI("ManagedWrapperVertex: Calling into Vertex Bridge to invoke Vertex Entry: %s", GetArgument(3));
DrLogging::FlushLog(); DrLogging::FlushLog();
@ -260,12 +266,22 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
// //
try try
{ {
System::Console::WriteLine("Assembly name " + bridgeAssemblyPartialName); System::Reflection::Assembly ^vertexBridgeAsm;
System::Reflection::Assembly ^vertexBridgeAsm = System::Reflection::Assembly::LoadWithPartialName(bridgeAssemblyPartialName); try
System::Type ^vertexBridgeType = vertexBridgeAsm->GetType(gcnew System::String(bridgeClassName)); {
System::Reflection::MethodInfo ^vertexBridgeMethod = vertexBridgeType->GetMethod(gcnew System::String(bridgeMethodName), vertexBridgeAsm = System::Reflection::Assembly::Load(bridgeAssemblyName);
static_cast<System::Reflection::BindingFlags>(System::Reflection::BindingFlags::NonPublic | }
System::Reflection::BindingFlags::Static)); catch (System::Exception ^ie)
{
DrLogI("ManagedWrapperVertex: Failed to load assembly %s: %s", bridgeAssemblyName, ie->ToString());
System::String^ asmLoc = System::IO::Path::Combine("..", bridgeAssemblyName + ".dll");
vertexBridgeAsm = System::Reflection::Assembly::LoadFrom(asmLoc);
}
System::Type ^vertexBridgeType = vertexBridgeAsm->GetType(bridgeClassName);
System::Reflection::MethodInfo ^vertexBridgeMethod
= vertexBridgeType->GetMethod(bridgeMethodName,
static_cast<System::Reflection::BindingFlags>(System::Reflection::BindingFlags::NonPublic |
System::Reflection::BindingFlags::Static));
cli::array<System::Object^> ^invokeArgs = gcnew array<System::Object^>(2); cli::array<System::Object^> ^invokeArgs = gcnew array<System::Object^>(2);
invokeArgs[0] = logName; invokeArgs[0] = logName;
@ -302,7 +318,7 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
error = (DrError)hr; error = (DrError)hr;
errorMsg.Set("Error returned from managed runtime invocation, "); errorMsg.Set("Error returned from managed runtime invocation, ");
errorMsg.Append(DRERRORSTRING(error)); errorMsg.Append(DRERRORSTRING(error));
errorMsg.Append("\n"); errorMsg.Append("\n");
DrLogE( "Error returned from managed runtime invocation. %s (%d)", DRERRORSTRING(error), error); DrLogE( "Error returned from managed runtime invocation. %s (%d)", DRERRORSTRING(error), error);
// //

10
DryadVertex/VertexHost/vertex/managedwrappervertex/ManagedWrapperVertex.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -21,13 +21,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -99,4 +99,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
</ImportGroup> </ImportGroup>
</Project> </Project>

27
DryadVertex/VertexHost/vertex/vertexHost/VertexHost.vcxproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -18,19 +18,20 @@
<Keyword>Win32Proj</Keyword> <Keyword>Win32Proj</Keyword>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir> <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir>
<RestorePackages>true</RestorePackages> <RestorePackages>true</RestorePackages>
<ProjectName>VertexHostBody</ProjectName>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -56,6 +57,13 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<NuGetPackageImportStamp>26f18b05</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<TargetName>Microsoft.Research.Dryad.$(ProjectName)</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<TargetName>Microsoft.Research.Dryad.$(ProjectName)</TargetName>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl> <Midl>
@ -112,6 +120,7 @@
</PostBuildEvent> </PostBuildEvent>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemGroup> <ItemGroup>
<ClCompile Include="version.cpp" />
<ClCompile Include="vertexHost.cpp" /> <ClCompile Include="vertexHost.cpp" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
@ -124,13 +133,13 @@
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
</ImportGroup> </ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
</Target> </Target>
</Project> </Project>

1
DryadVertex/VertexHost/vertex/vertexHost/VertexHost.vcxproj.filters
View File

@ -2,6 +2,7 @@
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup> <ItemGroup>
<ClCompile Include="vertexHost.cpp" /> <ClCompile Include="vertexHost.cpp" />
<ClCompile Include="version.cpp" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="app.config" /> <None Include="app.config" />

32
DryadVertex/VertexHost/vertex/vertexHost/app.config
View File

@ -1,26 +1,42 @@
<?xml version="1.0" encoding="utf-8" ?> <?xml version="1.0" encoding="utf-8" ?>
<configuration> <configuration>
<runtime> <runtime>
<gcServer enabled="true"></gcServer>
<gcConcurrent enabled="false"></gcConcurrent>
<generatePublisherEvidence enabled="false"/>
<gcAllowVeryLargeObjects enabled="true" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" /> <bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly> </dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>

2
DryadVertex/VertexHost/vertex/vertexHost/packages.config
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" /> <package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
</packages> </packages>

40
DryadVertex/VertexHost/vertex/vertexHost/version.cpp Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
#ifdef _MANAGED
#pragma managed
using namespace System::Reflection;
using namespace System::Runtime::InteropServices;
[assembly: AssemblyTitle("Microsoft.Research.Dryad.VertexHost")];
[assembly: AssemblyDescription("")];
[assembly: AssemblyConfiguration("")];
[assembly: AssemblyCompany("Microsoft Corporation")];
[assembly: AssemblyProduct("Microsoft.Research.Dryad.VertexHost")];
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
[assembly: AssemblyTrademark("")];
[assembly: AssemblyCulture("")];
[assembly: ComVisible(false)];
[assembly: AssemblyVersion("0.2.1.0")];
[assembly: AssemblyFileVersion("0.2.1.0")];
#endif

209
DryadVertex/VertexHost/vertex/vertexHost/vertexHost.cpp
View File

@ -174,27 +174,27 @@ void SetLoggingLevel()
GetLoggingFileName(logFileName); GetLoggingFileName(logFileName);
DrLogging::Initialize(logFileName); DrLogging::Initialize(logFileName);
WCHAR traceLevel [MAX_PATH]; WCHAR loggingLevel [MAX_PATH];
HRESULT hr = DrGetEnvironmentVariable(L"DRYAD_TRACE_LEVEL", traceLevel); HRESULT hr = DrGetEnvironmentVariable(L"DRYAD_LOGGING_LEVEL", loggingLevel);
if(hr == DrError_OK) if(hr == DrError_OK)
{ {
if(wcscmp(traceLevel, L"OFF") == 0) if(wcscmp(loggingLevel, L"OFF") == 0)
{ {
DrLogging::SetLoggingLevel(LogLevel_Off); DrLogging::SetLoggingLevel(LogLevel_Off);
} }
else if(wcscmp(traceLevel, L"CRITICAL") == 0) else if(wcscmp(loggingLevel, L"CRITICAL") == 0)
{ {
DrLogging::SetLoggingLevel(LogLevel_Assert); DrLogging::SetLoggingLevel(LogLevel_Assert);
} }
else if(wcscmp(traceLevel, L"ERROR") == 0) else if(wcscmp(loggingLevel, L"ERROR") == 0)
{ {
DrLogging::SetLoggingLevel(LogLevel_Error); DrLogging::SetLoggingLevel(LogLevel_Error);
} }
else if(wcscmp(traceLevel, L"WARN") == 0) else if(wcscmp(loggingLevel, L"WARN") == 0)
{ {
DrLogging::SetLoggingLevel(LogLevel_Warning); DrLogging::SetLoggingLevel(LogLevel_Warning);
} }
else if(wcscmp(traceLevel, L"INFO") == 0) else if(wcscmp(loggingLevel, L"INFO") == 0)
{ {
DrLogging::SetLoggingLevel(LogLevel_Info); DrLogging::SetLoggingLevel(LogLevel_Info);
} }
@ -249,104 +249,119 @@ static void ExceptionHandler(System::Object^ sender, System::UnhandledExceptionE
// //
// Start up vertex host // Start up vertex host
// //
[System::Security::SecurityCriticalAttribute] public ref class VertexHost
[System::Runtime::ExceptionServices::HandleProcessCorruptedStateExceptionsAttribute]
#if defined(_AMD64_)
int wmain(int argc, wchar_t* wargv[])
#else
int __cdecl wmain(int argc, wchar_t* wargv[])
#endif
{ {
try public:
[System::Security::SecurityCriticalAttribute]
[System::Runtime::ExceptionServices::HandleProcessCorruptedStateExceptionsAttribute]
static int Main(array<System::String^>^ managedArgs)
{ {
// try
// Enable logging based on environment variable
//
SetLoggingLevel();
DrInitErrorTable();
DrInitExitCodeTable();
DrInitLastAccessTable();
// Set unhandled exception handler to catch anything thrown from
// managed code
System::AppDomain^ currentDomain = System::AppDomain::CurrentDomain;
currentDomain->UnhandledException += gcnew System::UnhandledExceptionEventHandler(ExceptionHandler);
//
// trace for startup
//
DrLogI("Vertex Host starting");
//
// Get environment variable to know whether to break into debugger
//
BreakForDebugger();
//
// We call Register on the Managed Wrapper vertex factory to force its library to be linked.
// Registration actually occurs during static initialization.
//
s_factoryHWrapper.Register();
//
// Get command line arguments
//
char** argv;
DrGetUtf8CommandArgs(argc, wargv, &argv);
//
// Initialize the dryad communication layer with the command line arguments
//
int nOpts;
DrError e;
e = DryadInitialize(argc, argv, &nOpts);
if (e != DrError_OK)
{ {
int argc = managedArgs->Length;
wchar_t** wargv = new wchar_t*[argc+1];
for (int i=0; i<argc; ++i)
{
pin_ptr<const wchar_t> wch = PtrToStringChars(managedArgs[i]);
wargv[i] = _wcsdup(wch);
}
wargv[argc] = NULL;
// //
// Report error in initializing cluster layer // Enable logging based on environment variable
// //
DrLogE("Couldn't initialise Cluster"); SetLoggingLevel();
DrInitErrorTable();
DrInitExitCodeTable();
DrInitLastAccessTable();
// Set unhandled exception handler to catch anything thrown from
// managed code
System::AppDomain^ currentDomain = System::AppDomain::CurrentDomain;
currentDomain->UnhandledException += gcnew System::UnhandledExceptionEventHandler(ExceptionHandler);
//
// trace for startup
//
DrLogE("Vertex Host starting");
//
// Get environment variable to know whether to break into debugger
//
BreakForDebugger();
//
// We call Register on the Managed Wrapper vertex factory to force its library to be linked.
// Registration actually occurs during static initialization.
//
s_factoryHWrapper.Register();
//
// Get command line arguments
//
char** argv;
DrGetUtf8CommandArgs(argc, wargv, &argv);
for (int i=0; i<managedArgs->Length; ++i)
{
free(wargv[i]);
}
delete [] wargv;
//
// Initialize the dryad communication layer with the command line arguments
//
int nOpts;
DrError e;
e = DryadInitialize(argc, argv, &nOpts);
if (e != DrError_OK)
{
//
// Report error in initializing cluster layer
//
DrLogE("Couldn't initialise Cluster");
return 1;
}
//
// Update the argument list to just those parameters that weren't used by cluster init
//
EliminateArguments(&argc, argv, 1, nOpts);
//
// Call main function to continue execution of vertex
//
int exitCode = DryadVertexMain(argc, argv, NULL);
//
// Close the cluster connection after dryadvertexmain returns
//
e = DryadShutdown();
if (e == DrError_OK)
{
//
// Report success
//
DrLogI("Completed uninitialise cluster");
}
else
{
//
// Report failure
//
DrLogE("Couldn't uninitialise cluster");
}
return exitCode;
}
catch (System::Exception^ e)
{
DrLogA("Unhandled exception: %s", DrString(e->ToString()).GetChars());
return 1; return 1;
} }
//
// Update the argument list to just those parameters that weren't used by cluster init
//
EliminateArguments(&argc, argv, 1, nOpts);
//
// Call main function to continue execution of vertex
//
int exitCode = DryadVertexMain(argc, argv, NULL);
//
// Close the cluster connection after dryadvertexmain returns
//
e = DryadShutdown();
if (e == DrError_OK)
{
//
// Report success
//
DrLogI("Completed uninitialise cluster");
}
else
{
//
// Report failure
//
DrLogE("Couldn't uninitialise cluster");
}
return exitCode;
} }
catch (System::Exception^ e) };
{
DrLogA("Unhandled exception: %s", DrString(e->ToString()).GetChars());
return 1;
}
}
// //
// Simple data class which contains the byte array and its length. // Simple data class which contains the byte array and its length.

27
GraphManager/GraphManager.vcxproj
View File

@ -1,6 +1,9 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -25,13 +28,13 @@
<ConfigurationType>DynamicLibrary</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CLRSupport>Pure</CLRSupport> <CLRSupport>Pure</CLRSupport>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>Pure</CLRSupport> <CLRSupport>Pure</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -59,6 +62,7 @@
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad</TargetName> <TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad</TargetName>
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad</TargetName> <TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad</TargetName>
<NuGetPackageImportStamp>46822e65</NuGetPackageImportStamp>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl> <Midl>
@ -223,13 +227,22 @@
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
</ImportGroup> </ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
</Target> </Target>
</Project> </Project>

2
GraphManager/filesystem/DrAzureBlobClient.h
View File

@ -20,7 +20,7 @@ limitations under the License.
#pragma once #pragma once
using namespace Microsoft::Research::Peloponnese::Storage; using namespace Microsoft::Research::Peloponnese::Azure;
DRCLASS(DrAzureInputStream) : public DrInputStream DRCLASS(DrAzureInputStream) : public DrInputStream
{ {

267
GraphManager/filesystem/DrHdfsClient.cpp
View File

@ -129,73 +129,42 @@ DrHdfsInputStream::DrHdfsInputStream()
m_hdfsInstance = DrNull; m_hdfsInstance = DrNull;
} }
HRESULT DrHdfsInputStream::Open(DrUniversePtr universe, DrNativeString streamUri) HRESULT DrHdfsInputStream::Open(DrUniversePtr universe, DrNativeString streamUri, DrNativeString recordType)
{ {
DrString uri = DrString(streamUri); DrString uri = DrString(streamUri);
DrString record = DrString(recordType);
DrLogI("Opening instance for %s", uri.GetChars()); DrLogI("Opening instance for %s record type %s", uri.GetChars(), record.GetChars());
return OpenInternal(universe, uri); return OpenInternal(universe, uri, record);
} }
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri) #ifdef _MANAGED
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType)
{ {
m_streamUri = streamUri; m_streamUri = streamUri;
HRESULT err = S_OK; HRESULT err = S_OK;
#ifdef _MANAGED
try try
{ {
#endif DrLogI("Opening instance for %s: %s", streamUri.GetChars(), recordType.GetChars());
DrLogI("Opening instance for %s", streamUri.GetChars());
m_hdfsInstance = GetHdfsServiceInstance(streamUri); m_hdfsInstance = GetHdfsServiceInstance(streamUri);
#ifdef _MANAGED
DrLogI("Getting file info for %s", streamUri.GetChars());
HdfsFileInfo^ stream = m_hdfsInstance->GetFileInfo(streamUri.GetString(), true); HdfsFileInfo^ stream = m_hdfsInstance->GetFileInfo(streamUri.GetString(), true);
m_fileNameArray = stream->fileNameArray; m_fileNameArray = stream->fileNameArray;
UInt32 totalPartitionCount = static_cast<UInt32>(stream->blockArray->Length); UInt32 totalPartitionCount;
#else if (recordType.Compare("Microsoft.Research.DryadLinq.LineRecord") == 0)
bool ret = HdfsBridgeNative::Initialize();
if (!ret)
{ {
DrLogE("Error calling HdfsBridgeNative::Initialize()"); DrLogI("Getting block-level file info for %s", streamUri.GetChars());
return E_FAIL; totalPartitionCount = static_cast<UInt32>(stream->blockArray->Length);
} }
else
if (m_hdfsInstance == NULL)
{ {
DrLogE("Error calling GetHdfsServiceInstance(streamUri)"); DrLogI("Getting file info for %s", streamUri.GetChars());
return E_FAIL; totalPartitionCount = m_fileNameArray->Length;
} }
URL_COMPONENTSA UrlComponents = {0};
UrlComponents.dwStructSize = sizeof(UrlComponents);
UrlComponents.dwUrlPathLength = 1;
UrlComponents.dwHostNameLength = 1;
BOOL fOK = InternetCrackUrlA(streamUri.GetChars(), streamUri.GetCharsLength(), 0, &UrlComponents);
if (!fOK)
{
DrLogE("Error getting stream path from HDFS URI.");
return E_FAIL;
}
m_hostname.Set(UrlComponents.lpszHostName);
m_portNum = UrlComponents.nPort;
InstanceAccessor ia(m_hdfsInstance);
FileStat* fileStat = NULL;
ia.OpenFileStat(UrlComponents.lpszUrlPath, true, &fileStat);
UINT32 totalPartitionCount = 0;
HdfsBridgeNative::FileStatAccessor fs(fileStat);
totalPartitionCount = fs.GetNumberOfBlocks();
m_fileNameArray = (const char **)fs.GetFileNameArray();
#endif
DrLogI("Partition count %d", totalPartitionCount); DrLogI("Partition count %d", totalPartitionCount);
@ -205,35 +174,97 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
m_partOffsets = DrNew DrUINT64Array(totalPartitionCount); m_partOffsets = DrNew DrUINT64Array(totalPartitionCount);
m_partFileIds = DrNew DrUINT32Array(totalPartitionCount); m_partFileIds = DrNew DrUINT32Array(totalPartitionCount);
for (UINT32 i=0; i<totalPartitionCount; ++i) if (recordType.Compare("Microsoft.Research.DryadLinq.LineRecord") == 0)
{ {
#ifdef _MANAGED for (UINT32 i = 0; i < totalPartitionCount; ++i)
HdfsBlockInfo^ partition = stream->blockArray[i];
#else
HdfsBridgeNative::HdfsBlockLocInfo* partition = fs.GetBlockInfo(i);
#endif
m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(partition->Size);
m_partOffsets[i] = partition->Offset;
m_partFileIds[i] = partition->fileIndex;
#ifdef _MANAGED
for (int j = 0; j < partition->Hosts->Length; ++j)
#else
for (int j = 0; j < partition->numberOfHosts; ++j)
#endif
{ {
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]); HdfsBlockInfo^ partition = stream->blockArray[i];
if (location != DrNull) m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(partition->Size);
m_partOffsets[i] = partition->Offset;
m_partFileIds[i] = partition->fileIndex;
for (int j = 0; j < partition->Hosts->Length; ++j)
{ {
m_affinity[i]->AddLocality(location); DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
m_affinity[i]->AddLocality(location);
}
} }
} }
#ifndef _MANAGED }
delete partition; else
#endif {
} int fileBlockIndex = 0;
#ifdef _MANAGED for (UINT32 i = 0; i < totalPartitionCount; ++i)
{
m_partOffsets[i] = 0;
m_partFileIds[i] = i;
HdfsBlockInfo^ partition = stream->blockArray[fileBlockIndex];
DrAssert(partition->fileIndex == i);
long long fileSize = partition->Size;
HashSet<DrResourceRef>^ locations = DrNew HashSet<DrResourceRef>();
for (int j = 0; j < partition->Hosts->Length; ++j)
{
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
locations->Add(location);
}
}
++fileBlockIndex;
while (fileBlockIndex < stream->blockArray->Length && stream->blockArray[fileBlockIndex]->fileIndex == i)
{
partition = stream->blockArray[fileBlockIndex];
fileSize += partition->Size;
if (locations->Count > 0)
{
HashSet<DrResourceRef>^ newLocations = DrNew HashSet<DrResourceRef>();
for (int j = 0; j < partition->Hosts->Length; ++j)
{
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
newLocations->Add(location);
}
}
locations->IntersectWith(newLocations);
}
}
m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(fileSize);
System::Text::StringBuilder^ locationText;
if (locations->Count > 0)
{
locationText = gcnew System::Text::StringBuilder("File " + m_fileNameArray[i] + " merged locations:");
}
else
{
locationText = gcnew System::Text::StringBuilder("File " + m_fileNameArray[i] + " no shared locations");
}
HashSet<DrResourceRef>::Enumerator^ enumerator = locations->GetEnumerator();
while (enumerator->MoveNext())
{
m_affinity[i]->AddLocality(enumerator->Current);
locationText->Append(" ");
locationText->Append(enumerator->Current->GetName().GetString());
}
DrString locationLog(locationText->ToString());
DrLogI("%s", locationLog.GetChars());
}
}
} }
catch (System::Exception ^e) catch (System::Exception ^e)
{ {
@ -247,10 +278,85 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
// TODO: How do we clean this up? // TODO: How do we clean this up?
//hdfsInstance->Discard(); //hdfsInstance->Discard();
} }
#endif
return err; return err;
} }
#else
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType)
{
m_streamUri = streamUri;
HRESULT err = S_OK;
DrLogI("Opening instance for %s", streamUri.GetChars());
m_hdfsInstance = GetHdfsServiceInstance(streamUri);
bool ret = HdfsBridgeNative::Initialize();
if (!ret)
{
DrLogE("Error calling HdfsBridgeNative::Initialize()");
return E_FAIL;
}
if (m_hdfsInstance == NULL)
{
DrLogE("Error calling GetHdfsServiceInstance(streamUri)");
return E_FAIL;
}
URL_COMPONENTSA UrlComponents = {0};
UrlComponents.dwStructSize = sizeof(UrlComponents);
UrlComponents.dwUrlPathLength = 1;
UrlComponents.dwHostNameLength = 1;
BOOL fOK = InternetCrackUrlA(streamUri.GetChars(), streamUri.GetCharsLength(), 0, &UrlComponents);
if (!fOK)
{
DrLogE("Error getting stream path from HDFS URI.");
return E_FAIL;
}
m_hostname.Set(UrlComponents.lpszHostName);
m_portNum = UrlComponents.nPort;
InstanceAccessor ia(m_hdfsInstance);
FileStat* fileStat = NULL;
ia.OpenFileStat(UrlComponents.lpszUrlPath, true, &fileStat);
UINT32 totalPartitionCount = 0;
HdfsBridgeNative::FileStatAccessor fs(fileStat);
totalPartitionCount = fs.GetNumberOfBlocks();
m_fileNameArray = (const char **)fs.GetFileNameArray();
DrLogI("Partition count %d", totalPartitionCount);
/* Allocate these arrays even if they're size 0, to avoid
NullReferenceException later */
m_affinity = DrNew DrAffinityArray(totalPartitionCount);
m_partOffsets = DrNew DrUINT64Array(totalPartitionCount);
m_partFileIds = DrNew DrUINT32Array(totalPartitionCount);
for (UINT32 i=0; i<totalPartitionCount; ++i)
{
HdfsBridgeNative::HdfsBlockLocInfo* partition = fs.GetBlockInfo(i);
m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(partition->Size);
m_partOffsets[i] = partition->Offset;
m_partFileIds[i] = partition->fileIndex;
for (int j = 0; j < partition->numberOfHosts; ++j)
{
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
m_affinity[i]->AddLocality(location);
}
}
delete partition;
}
return err;
}
#endif
DrNativeString DrHdfsInputStream::GetError() DrNativeString DrHdfsInputStream::GetError()
{ {
@ -468,6 +574,25 @@ HRESULT DrHdfsOutputStream::FinalizeSuccessfulParts(DrOutputPartitionArrayRef pa
errorText.SetF("%s", DrString(m_error).GetChars()); errorText.SetF("%s", DrString(m_error).GetChars());
return E_FAIL; return E_FAIL;
} }
String^ userName = Environment::GetEnvironmentVariable("USER");
if (userName == nullptr)
{
userName = Environment::UserName;
}
try
{
m_hdfsInstance->SetOwnerAndPermission(m_baseUri, userName, nullptr, Convert::ToInt16("0755", 8));
}
catch (Exception^ e)
{
DrString drDst(m_baseUri);
DrString err(e->ToString());
DrLogE("Can't set %s permissions finalizing HDFS output: %s", drDst.GetChars(), err.GetChars());
m_error = "Can't set " + m_baseUri + " permissions finalizing HDFS output: " + e->ToString();
errorText.SetF("%s", DrString(m_error).GetChars());
return E_FAIL;
}
} }
else else
{ {

4
GraphManager/filesystem/DrHdfsClient.h
View File

@ -30,8 +30,8 @@ DRCLASS(DrHdfsInputStream) : public DrInputStream
{ {
public: public:
DrHdfsInputStream(); DrHdfsInputStream();
HRESULT Open(DrUniversePtr universe, DrNativeString streamUri); HRESULT Open(DrUniversePtr universe, DrNativeString streamUri, DrNativeString recordType);
HRESULT OpenInternal(DrUniversePtr universe, DrString streamUri); HRESULT OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType);
DrNativeString GetError(); DrNativeString GetError();
virtual DrString GetStreamName() DROVERRIDE; virtual DrString GetStreamName() DROVERRIDE;

32
GraphManager/filesystem/DrPartitionFile.cpp
View File

@ -79,6 +79,7 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
DrStringR remoteName, DrStringR remoteName,
DrPartitionInputStream::OverridePtr over, DrPartitionInputStream::OverridePtr over,
bool mustOverride, bool mustOverride,
bool pathIsRooted,
DrString line, DrString line,
DrUniversePtr universe) DrUniversePtr universe)
{ {
@ -112,8 +113,16 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
sep = lineCopy.IndexOfChar(','); sep = lineCopy.IndexOfChar(',');
if (sep == DrStr_InvalidIndex) if (sep == DrStr_InvalidIndex)
{ {
DrLogW("Malformed line %s: no list of machines", line.GetChars()); int n = sscanf_s(lineCopy.GetChars(), "%I64u", &parsedSize);
return false; if (n != 1)
{
DrLogW("Malformed line %s: can't parse size", line.GetChars());
return false;
}
affinity->SetWeight(parsedSize);
lineCopy = DrString("");
} }
else else
{ {
@ -135,8 +144,14 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
if (lineCopy.GetCharsLength() == 0) if (lineCopy.GetCharsLength() == 0)
{ {
DrLogW("Malformed line %s: no partition machines", line.GetChars()); if (!pathIsRooted || mustOverride)
return false; {
DrLogW("Malformed line %s: no partition machines", line.GetChars());
return false;
}
remoteName.Set(" %Invalid% ");
return true;
} }
int numberOfReplicas = 0; int numberOfReplicas = 0;
@ -204,7 +219,7 @@ HRESULT DrPartitionInputStream::Open(DrUniversePtr universe, DrNativeString stre
HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString streamName) HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString streamName)
{ {
HRESULT err = S_OK; HRESULT err = S_OK;
DrLogI("Opening input file %s", streamName.GetChars(), DRERRORSTRING(err)); DrLogI("Opening input file %s", streamName.GetChars(), DRERRORSTRING(err));
FILE* f; FILE* f;
errno_t ferr = fopen_s(&f, streamName.GetChars(), "rb"); errno_t ferr = fopen_s(&f, streamName.GetChars(), "rb");
if (ferr != 0) if (ferr != 0)
@ -231,6 +246,12 @@ HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString st
mustOverride = true; mustOverride = true;
} }
bool pathIsRooted = false;
if (m_pathNameOnComputer.IndexOfChar(':') != DrStr_InvalidIndex)
{
pathIsRooted = true;
}
int numberOfParts; int numberOfParts;
int n = sscanf_s(partitionSizeLine.GetChars(), "%d", &numberOfParts); int n = sscanf_s(partitionSizeLine.GetChars(), "%d", &numberOfParts);
if (n != 1) if (n != 1)
@ -280,6 +301,7 @@ HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString st
remoteName, remoteName,
m_override[i], m_override[i],
mustOverride, mustOverride,
pathIsRooted,
partitionLine, partitionLine,
universe) == false) universe) == false)
{ {

19
GraphManager/jobmanager/version.cpp
View File

@ -19,5 +19,22 @@ limitations under the License.
*/ */
#ifdef _MANAGED #ifdef _MANAGED
[assembly:System::Runtime::InteropServices::ComVisible(false)]; #pragma managed
using namespace System::Reflection;
using namespace System::Runtime::InteropServices;
[assembly: AssemblyTitle("Microsoft.Research.Dryad")];
[assembly: AssemblyDescription("")];
[assembly: AssemblyConfiguration("")];
[assembly: AssemblyCompany("Microsoft Corporation")];
[assembly: AssemblyProduct("Microsoft.Research.Dryad")];
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
[assembly: AssemblyTrademark("")];
[assembly: AssemblyCulture("")];
[assembly: ComVisible(false)];
[assembly: AssemblyVersion("1.0.0.0")];
[assembly: AssemblyFileVersion("1.0.0.0")];
#endif #endif

6
GraphManager/kernel/DrCluster.cpp
View File

@ -570,7 +570,7 @@ DrString DrClusterInternal::TranslateFileToURI(DrString leafName, DrString direc
} }
void DrClusterInternal::ScheduleProcess(DrAffinityListRef affinities, void DrClusterInternal::ScheduleProcess(DrAffinityListRef affinities,
DrString name, DrString commandLine, DrString name, DrString commandLineArgs,
DrProcessTemplatePtr processTemplate, DrProcessTemplatePtr processTemplate,
DrPSRListenerPtr listener) DrPSRListenerPtr listener)
{ {
@ -603,8 +603,8 @@ void DrClusterInternal::ScheduleProcess(DrAffinityListRef affinities,
DrLogI("Starting schedule process for %s.%s", DrLogI("Starting schedule process for %s.%s",
processTemplate->GetProcessClass().GetChars(), name.GetChars()); processTemplate->GetProcessClass().GetChars(), name.GetChars());
IProcess^ rawProcess = m_cluster->NewProcess(process, commandLine.GetString()); IProcess^ rawProcess = m_cluster->NewProcess(process, processTemplate->GetCommandLineBase().GetString(), commandLineArgs.GetString());
process->SetProcess(rawProcess); process->SetProcess(rawProcess);
DrLogI("Assigned GUID %s to process for %s.%s", DrLogI("Assigned GUID %s to process for %s.%s",

3
GraphManager/kernel/DrCluster.h
View File

@ -45,9 +45,10 @@ public:
DrResourcePtr srcResource, DrResourcePtr dstResource, int compressionMode) = 0; DrResourcePtr srcResource, DrResourcePtr dstResource, int compressionMode) = 0;
virtual void ScheduleProcess(DrAffinityListRef affinities, virtual void ScheduleProcess(DrAffinityListRef affinities,
DrString name, DrString commandLine, DrString name, DrString commandLineArgs,
DrProcessTemplatePtr processTemplate, DrProcessTemplatePtr processTemplate,
DrPSRListenerPtr listener) = 0; DrPSRListenerPtr listener) = 0;
virtual void CancelScheduleProcess(DrProcessHandlePtr process) = 0; virtual void CancelScheduleProcess(DrProcessHandlePtr process) = 0;
virtual void WaitForStateChange(DrProcessHandlePtr process, DrPSRListenerPtr listener) = 0; virtual void WaitForStateChange(DrProcessHandlePtr process, DrPSRListenerPtr listener) = 0;

3
GraphManager/kernel/DrClusterInternal.h
View File

@ -165,9 +165,10 @@ public:
DrResourcePtr srcResource, DrResourcePtr dstResource, int compressionMode) DROVERRIDE; DrResourcePtr srcResource, DrResourcePtr dstResource, int compressionMode) DROVERRIDE;
virtual void ScheduleProcess(DrAffinityListRef affinities, virtual void ScheduleProcess(DrAffinityListRef affinities,
DrString name, DrString commandLine, DrString name, DrString commandLineArgs,
DrProcessTemplatePtr processTemplate, DrProcessTemplatePtr processTemplate,
DrPSRListenerPtr listener) DROVERRIDE; DrPSRListenerPtr listener) DROVERRIDE;
virtual void CancelScheduleProcess(DrProcessHandlePtr process) DROVERRIDE; virtual void CancelScheduleProcess(DrProcessHandlePtr process) DROVERRIDE;
virtual void WaitForStateChange(DrProcessHandlePtr process, DrPSRListenerPtr listener) DROVERRIDE; virtual void WaitForStateChange(DrProcessHandlePtr process, DrPSRListenerPtr listener) DROVERRIDE;

5
GraphManager/packages.config
View File

@ -1,4 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" /> <package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="Native" />
</packages> </packages>

5
GraphManager/reporting/DrCalypsoReporting.cpp
View File

@ -24,7 +24,8 @@ limitations under the License.
#include <msclr\lock.h> #include <msclr\lock.h>
using namespace System::IO; using namespace System::IO;
using namespace Microsoft::Research::Peloponnese::Storage; using namespace Microsoft::Research::Peloponnese::Azure;
using namespace Microsoft::Research::Peloponnese::Hdfs;
using namespace msclr; using namespace msclr;
// //
@ -126,7 +127,7 @@ public:
DrCalypsoReporter::DrCalypsoReporter(DrNativeString uriString) DrCalypsoReporter::DrCalypsoReporter(DrNativeString uriString)
{ {
System::Uri^ uri = DrNew System::Uri(uriString); System::Uri^ uri = DrNew System::Uri(uriString);
if (uri->Scheme == AzureUtils::BlobScheme) if (uri->Scheme == Utils::BlobScheme)
{ {
m_logStream = DrNew AzureLogAppendStream(uri, 0x20, false, false, gcnew PeloponneseLogger()); m_logStream = DrNew AzureLogAppendStream(uri, 0x20, false, false, gcnew PeloponneseLogger());
m_flushInterval = 1000; m_flushInterval = 1000;

3
GraphManager/vertex/DrCohort.cpp
View File

@ -470,8 +470,7 @@ void DrCohort::StartProcess(DrGraphPtr graph, int version)
processName.SetF("%s v.%d", m_description.GetChars(), version); processName.SetF("%s v.%d", m_description.GetChars(), version);
DrString commandLine; DrString commandLine;
commandLine.SetF("%s --noredirect --startfrompn %d", commandLine.SetF("--noredirect --startfrompn %d", m_list->Size());
m_processTemplate->GetCommandLineBase().GetChars(), m_list->Size());
for (i=0; i<m_list->Size(); ++i) for (i=0; i<m_list->Size(); ++i)
{ {

2
GraphManager/vertex/DrVertex.cpp
View File

@ -1743,7 +1743,7 @@ void DrTeeVertex::ReactToFailedVertex(DrVertexOutputGeneratorPtr failedGenerator
DrAssert(m_inputEdges->GetNumberOfEdges() == 1); DrAssert(m_inputEdges->GetNumberOfEdges() == 1);
DrEdge e = m_inputEdges->GetEdge(0); DrEdge e = m_inputEdges->GetEdge(0);
DrLogI("Tee vertex %d.%d: calling ReactToFailedVertex on remote edge", this->m_id, GetVersion()); DrLogI("Tee vertex %d.%d: calling ReactToFailedVertex on remote edge", this->m_id, failedGenerator->GetVersion());
e.m_remoteVertex->ReactToFailedVertex(failedGenerator, DrNull, DrNull, DrNull, originalReason); e.m_remoteVertex->ReactToFailedVertex(failedGenerator, DrNull, DrNull, DrNull, originalReason);
/* fill in a new generator if it's already there, e.g. if the upstream vertex is a DrStorageVertex */ /* fill in a new generator if it's already there, e.g. if the upstream vertex is a DrStorageVertex */

56
JobBrowser/Calypso/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,56 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("DryadAnalysis")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("MSIT")]
[assembly: AssemblyProduct("DryadAnalysis")]
[assembly: AssemblyCopyright("Copyright © MSIT 2008")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("286845bc-d060-49ff-9de8-4381471a0dc6")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

1290
JobBrowser/JOM/ClusterConfiguration.cs
View File

File diff suppressed because it is too large Load Diff

377
JobBrowser/JOM/ClusterStatus.cs
View File

@ -19,12 +19,22 @@ limitations under the License.
*/ */
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using Microsoft.Research.Peloponnese.Storage; using System.Web;
using System.Xml;
using System.Xml.Linq;
using Microsoft.Research.Peloponnese.Hdfs;
using Microsoft.Research.Peloponnese.Shared;
using Microsoft.Research.Peloponnese.Yarn;
using Microsoft.Research.Tools; using Microsoft.Research.Tools;
using System.Text.RegularExpressions;
using System.Text;
using JobStatus = Microsoft.Research.Peloponnese.ClusterUtils.JobStatus;
namespace Microsoft.Research.JobObjectModel namespace Microsoft.Research.JobObjectModel
{ {
@ -68,12 +78,16 @@ namespace Microsoft.Research.JobObjectModel
/// <summary> /// <summary>
/// See if a status is already cached. /// See if a status is already cached.
/// </summary> /// </summary>
/// <param name="clusterName">Name of cluster.</param> /// <param name="config">Cluster configuration.</param>
/// <returns>The cached status.</returns> /// <returns>The cached status.</returns>
public static ClusterStatus LookupStatus(string clusterName) public static ClusterStatus LookupStatus(ClusterConfiguration config)
{ {
if (ClusterStatuses.ContainsKey(clusterName)) if (ClusterStatuses.ContainsKey(config.Name))
return ClusterStatuses[clusterName]; {
var retval = ClusterStatuses[config.Name];
if (retval.Config.Equals(config))
return retval;
}
return null; return null;
} }
@ -172,8 +186,8 @@ namespace Microsoft.Research.JobObjectModel
/// <param name="manager">Communication manager.</param> /// <param name="manager">Communication manager.</param>
public virtual void RefreshStatus(DryadLinqJobSummary summary, CommManager manager) public virtual void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
{ {
// refresh the whole list // refresh the whole list: too expensive
this.RecomputeClusterJobList(summary.VirtualCluster, manager); // this.RecomputeClusterJobList(summary.VirtualCluster, manager);
ClusterJobInformation info = this.DiscoverClusterJob(summary, manager); ClusterJobInformation info = this.DiscoverClusterJob(summary, manager);
if (info == null) if (info == null)
{ {
@ -193,6 +207,7 @@ namespace Microsoft.Research.JobObjectModel
/// <summary> /// <summary>
/// A fake cluster keeps some information about past jobs on a local filesystem, to allow post-mortem debugging. /// A fake cluster keeps some information about past jobs on a local filesystem, to allow post-mortem debugging.
/// </summary> /// </summary>
@ -442,20 +457,17 @@ namespace Microsoft.Research.JobObjectModel
/// <summary> /// <summary>
/// Status of an Azure DFS cluster. /// Status of an Azure DFS cluster.
/// </summary> /// </summary>
public class AzureDfsClusterStatus : ClusterStatus public abstract class DfsClusterStatus : ClusterStatus
{ {
private AzureDfsClusterConfiguration config;
/// <summary> /// <summary>
/// Create a cluster containing just the local machine. /// Create a cluster containing just the local machine.
/// </summary> /// </summary>
/// <param name="config">Configuration for the local machine.</param> /// <param name="config">Configuration for the local machine.</param>
public AzureDfsClusterStatus(ClusterConfiguration config) protected DfsClusterStatus(ClusterConfiguration config)
: base(config) : base(config)
{ {
if (!(config is AzureDfsClusterConfiguration)) if (!(config is DfsClusterConfiguration))
throw new ArgumentException("Expected a AzureYarnClusterConfiguration, got a " + config.GetType()); throw new ArgumentException("Expected a DfsClusterConfiguration, got a " + config.GetType());
this.config = config as AzureDfsClusterConfiguration;
} }
/// <summary> /// <summary>
@ -468,31 +480,6 @@ namespace Microsoft.Research.JobObjectModel
return result; return result;
} }
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var jobs = this.config.AzureClient.EnumerateDirectory("").ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(job);
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(job, info);
}
manager.Progress(100*done++/jobs.Count);
}
manager.Progress(100);
}
/// <summary> /// <summary>
/// Discover the (unique) dryadlinq job corresponding to a cluster job. /// Discover the (unique) dryadlinq job corresponding to a cluster job.
/// </summary> /// </summary>
@ -503,7 +490,7 @@ namespace Microsoft.Research.JobObjectModel
{ {
DryadLinqJobSummary result = new DryadLinqJobSummary( DryadLinqJobSummary result = new DryadLinqJobSummary(
clusterJob.Cluster, clusterJob.Cluster,
this.config.TypeOfCluster, this.Config.TypeOfCluster,
"", // virtual cluster "", // virtual cluster
"", // machine "", // machine
clusterJob.ClusterJobID, // jobId clusterJob.ClusterJobID, // jobId
@ -529,6 +516,78 @@ namespace Microsoft.Research.JobObjectModel
throw new InvalidOperationException(); throw new InvalidOperationException();
} }
/// <summary>
/// Cancel the specified job.
/// </summary>
/// <param name="job">Job whose execution is cancelled.</param>
/// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job)
{
return false;
}
}
/// <summary>
/// Status of an Azure DFS cluster.
/// </summary>
public class AzureDfsClusterStatus : DfsClusterStatus
{
private AzureDfsClusterConfiguration config;
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="config">Configuration for the local machine.</param>
public AzureDfsClusterStatus(ClusterConfiguration config)
: base(config)
{
if (!(config is AzureDfsClusterConfiguration))
throw new ArgumentException("Expected a AzureDfsClusterConfiguration, got a " + config.GetType());
this.config = config as AzureDfsClusterConfiguration;
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var jobs = this.config.AzureClient.ExpandFileOrDirectory(AzureDfsFile.UriFromPath(this.config, "")).ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
string jobRootFolder = AzureDfsFile.PathFromUri(this.config, job);
ClusterJobInformation info = this.GetJobInfo(jobRootFolder);
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(job.AbsolutePath, info);
}
manager.Progress(100*done++/jobs.Count);
}
manager.Progress(100);
}
/// <summary>
/// Extract blob name from a path.
/// </summary>
/// <param name="container">Container name.</param>
/// <param name="path">Path.</param>
/// <returns>The blob part of path.</returns>
public static string GetBlobName(string container, string path)
{
if (path.StartsWith("/" + container + "/"))
path = path.Substring(container.Length + 2);
int q = path.IndexOf('?');
if (q > 0)
path = path.Substring(0, q);
return path;
}
/// <summary> /// <summary>
/// Extract the job information from a folder with logs on the local machine. /// Extract the job information from a folder with logs on the local machine.
/// </summary> /// </summary>
@ -540,14 +599,19 @@ namespace Microsoft.Research.JobObjectModel
DateTime lastHeartBeat = DateTime.MinValue; DateTime lastHeartBeat = DateTime.MinValue;
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown; ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
bool found = false; bool found = false;
string jobName = jobRootFolder;
var jobsFolders = this.config.AzureClient.EnumerateDirectory(jobRootFolder).ToList(); Uri uri = AzureDfsFile.UriFromPath(this.config, jobRootFolder);
var jobsFolders = this.config.AzureClient.ExpandFileOrDirectory(uri).ToList();
jobRootFolder = GetBlobName(this.config.Container, jobRootFolder);
string jobName = jobRootFolder;
foreach (var file in jobsFolders) foreach (var file in jobsFolders)
{ {
if (file.EndsWith("heartbeat")) if (file.AbsolutePath.EndsWith("heartbeat"))
{ {
var blob = this.config.AzureClient.Container.GetPageBlobReference(file); string blobName = GetBlobName(this.config.Container, file.AbsolutePath);
var blob = this.config.AzureClient.Container.GetPageBlobReference(blobName);
blob.FetchAttributes(); blob.FetchAttributes();
var props = blob.Metadata; var props = blob.Metadata;
if (props.ContainsKey("status")) if (props.ContainsKey("status"))
@ -597,11 +661,11 @@ namespace Microsoft.Research.JobObjectModel
found = true; found = true;
} }
else if (file.Contains("DryadLinqProgram__") && else if (file.AbsolutePath.Contains("DryadLinqProgram__") &&
// newer heartbeats contain the date // newer heartbeats contain the date
date != DateTime.MinValue) date != DateTime.MinValue)
{ {
var blob = this.config.AzureClient.Container.GetBlockBlobReference(file); var blob = this.config.AzureClient.Container.GetBlockBlobReference(AzureDfsFile.PathFromUri(this.config, file));
blob.FetchAttributes(); blob.FetchAttributes();
var props = blob.Properties; var props = blob.Properties;
if (props.LastModified.HasValue) if (props.LastModified.HasValue)
@ -628,7 +692,6 @@ namespace Microsoft.Research.JobObjectModel
/// <param name="manager">Communication manager.</param> /// <param name="manager">Communication manager.</param>
public override void RefreshStatus(DryadLinqJobSummary summary, CommManager manager) public override void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
{ {
// refresh the whole list
ClusterJobInformation info = this.GetJobInfo(summary.JobID); ClusterJobInformation info = this.GetJobInfo(summary.JobID);
if (info == null) if (info == null)
{ {
@ -645,8 +708,224 @@ namespace Microsoft.Research.JobObjectModel
/// <returns>True if the cancellation succeeded.</returns> /// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job) public override bool CancelJob(DryadLinqJobSummary job)
{ {
AzureUtils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId); Microsoft.Research.Peloponnese.Azure.Utils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId);
return true; return false;
}
}
/// <summary>
/// Cluster status of a WebHdfs cluster.
/// </summary>
public class WebHdfsClusterStatus : DfsClusterStatus
{
private WebHdfsClusterConfiguration config;
/// <summary>
/// Yarn client to access job status.
/// </summary>
private NativeYarnClient yarnClient;
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public WebHdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is WebHdfsClusterConfiguration))
throw new ArgumentException("Expected a WebHdfsClusterConfiguration, got a " + conf.GetType());
this.config = conf as WebHdfsClusterConfiguration;
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
private ClusterJobInformation GetJobInfo(string jobRootFolder)
{
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
long time;
long size;
this.config.DfsClient.GetFileStatus(uri, out time, out size);
DateTime date = DfsFile.TimeFromLong(time);
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
string jobName = Path.GetFileName(jobRootFolder);
string errorMsg = "";
try
{
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
var jobstatus = jobinfo.GetStatus();
errorMsg = jobinfo.ErrorMsg;
switch (jobstatus)
{
case JobStatus.NotSubmitted:
case JobStatus.Waiting:
status = ClusterJobInformation.ClusterJobStatus.Unknown;
break;
case JobStatus.Running:
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case JobStatus.Success:
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
break;
case JobStatus.Cancelled:
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
case JobStatus.Failure:
status = ClusterJobInformation.ClusterJobStatus.Failed;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception)
{
}
TimeSpan running = TimeSpan.Zero;
var info = new ClusterJobInformation(this.config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
return info;
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable once UnusedParameter.Global
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
var jobsEnum = this.config.DfsClient.EnumerateSubdirectories(uri);
List<Uri> jobs = jobsEnum != null ? jobsEnum.ToList() : new List<Uri>();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(DfsFile.PathFromUri(this.config.JobsFolderUri, job));
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(info.ClusterJobID, info);
}
manager.Progress(100 * done++ / jobs.Count);
}
manager.Progress(100);
}
}
/// <summary>
/// Cluster status of a WebHdfs cluster.
/// </summary>
public class HdfsClusterStatus : DfsClusterStatus
{
private HdfsClusterConfiguration config;
/// <summary>
/// Yarn client to access job status.
/// </summary>
private NativeYarnClient yarnClient;
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public HdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is HdfsClusterConfiguration))
throw new ArgumentException("Expected an HdfsClusterConfiguration, got a " + conf.GetType());
this.config = conf as HdfsClusterConfiguration;
// make a fake call to initialize the cluster on the foreground thread
// HDFS does not work if initialized on the background thread.
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
this.config.DfsClient.IsFileExists(uri); // ignore result
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
private ClusterJobInformation GetJobInfo(string jobRootFolder)
{
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
long time;
long size;
this.config.DfsClient.GetFileStatus(uri, out time, out size);
DateTime date = DfsFile.TimeFromLong(time);
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
string jobName = Path.GetFileName(jobRootFolder);
string errorMsg = "";
try
{
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
var jobstatus = jobinfo.GetStatus();
errorMsg = jobinfo.ErrorMsg;
switch (jobstatus)
{
case JobStatus.NotSubmitted:
case JobStatus.Waiting:
status = ClusterJobInformation.ClusterJobStatus.Unknown;
break;
case JobStatus.Running:
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case JobStatus.Success:
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
break;
case JobStatus.Cancelled:
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
case JobStatus.Failure:
status = ClusterJobInformation.ClusterJobStatus.Failed;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception)
{
}
TimeSpan running = TimeSpan.Zero;
var info = new ClusterJobInformation(config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
return info;
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable once UnusedParameter.Global
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
var jobs = this.config.DfsClient.EnumerateSubdirectories(uri).ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(DfsFile.PathFromUri(this.config.JobsFolderUri, job));
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(info.ClusterJobID, info);
}
manager.Progress(100 * done++ / jobs.Count);
}
manager.Progress(100);
} }
} }
} }

266
JobBrowser/JOM/JobObjectModel.csproj
View File

@ -1,6 +1,12 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.6-alpha042\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.6-alpha042\build\Microsoft.Research.Peloponnese.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -9,39 +15,17 @@
<ProjectGuid>{77739535-7FAC-4487-887F-FEBA197E7572}</ProjectGuid> <ProjectGuid>{77739535-7FAC-4487-887F-FEBA197E7572}</ProjectGuid>
<OutputType>Library</OutputType> <OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>DistributedDataCollection</RootNamespace> <RootNamespace>JobObjectModel</RootNamespace>
<AssemblyName>DistributedDataCollection</AssemblyName> <AssemblyName>JobObjectModel</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<StartupObject> <NuGetPackageImportStamp>b735dff6</NuGetPackageImportStamp>
</StartupObject>
<FileUpgradeFlags>
</FileUpgradeFlags>
<UpgradeBackupLocation>
</UpgradeBackupLocation>
<OldToolsVersion>3.5</OldToolsVersion>
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath> <OutputPath>bin\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants> <DefineConstants>DEBUG;TRACE</DefineConstants>
<DocumentationFile>bin\Debug\DistributedDataCollection.XML</DocumentationFile> <DocumentationFile>bin\Debug\JobObjectModel.XML</DocumentationFile>
<DebugType>full</DebugType> <DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget> <PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport> <ErrorReport>prompt</ErrorReport>
@ -56,108 +40,13 @@
<ErrorReport>prompt</ErrorReport> <ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup> </PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.DataSetExtensions">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup> <ItemGroup>
<Compile Include="ClusterAccess.cs" /> <Compile Include="ClusterAccess.cs" />
<Compile Include="ClusterConfiguration.cs" /> <Compile Include="ClusterConfiguration.cs" />
<Compile Include="ClusterStatus.cs" /> <Compile Include="ClusterStatus.cs" />
<Compile Include="dryadlog.cs" /> <Compile Include="dryadlog.cs" />
<Compile Include="jobinfo.cs" /> <Compile Include="jobinfo.cs" />
<Compile Include="JobSummary.cs" />
<Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Storage.cs" /> <Compile Include="Storage.cs" />
</ItemGroup> </ItemGroup>
@ -168,40 +57,121 @@
</ProjectReference> </ProjectReference>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5"> <None Include="app.config" />
<Visible>False</Visible> <None Include="packages.config">
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName> <SubType>Designer</SubType>
<Install>false</Install> </None>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="app.config" /> <Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<None Include="packages.config" /> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.HDInsight.Net.Http.Formatting, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.HDInsight.Net.Http.Formatting.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.3.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System">
<HintPath>..\..\..\..\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5\System.dll</HintPath>
</Reference>
<Reference Include="System.Data" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.XML" />
<Reference Include="System.Xml.Linq" />
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets'))" />
</Target> </Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild"> <Target Name="BeforeBuild">

488
JobBrowser/JOM/JobSummary.cs Normal file
View File

@ -0,0 +1,488 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System;
using System.IO;
using System.Text;
using Microsoft.Research.Tools;
namespace Microsoft.Research.JobObjectModel
{
/// <summary>
/// Error during conversation with cluster.
/// </summary>
public sealed class ClusterException : Exception
{
/// <summary>
/// Create an exception about handling a cluster.
/// </summary>
/// <param name="message">Exception message.</param>
public ClusterException(string message) : base(message) { }
}
/// <summary>
/// Identifier for a Dryad process; for now we are using globally-unique process GUID.
/// A Dryad Job identifier is always the same as the job manager process guid.
/// </summary>
[Serializable]
public class DryadProcessIdentifier : IEquatable<DryadProcessIdentifier>
{
/// <summary>
/// Process identifier.
/// </summary>
private string processIdentifier;
/// <summary>
/// Used just for XML serialization.
/// </summary>
public DryadProcessIdentifier()
{ }
/// <summary>
/// create an indentifier struct.
/// Be sure to initialize all fields before use;
/// </summary>
/// <param name="pid">The id of the process (platform-dependent).</param>
public DryadProcessIdentifier(string pid)
{
this.processIdentifier = pid;
}
/// <summary>
/// Process identifier; public for serialization only.
/// </summary>
public string ProcessIdentifier
{
get { return this.processIdentifier; }
set { this.processIdentifier = value; }
}
/// <summary>
/// If true the process identifier is not known.
/// </summary>
public bool IsUnknown { get { return this.ProcessIdentifier == null; } }
/// <summary>
/// Human-readable description of the process identifier.
/// </summary>
/// <returns>An empty string if the pid is not set.</returns>
public override string ToString()
{
if (this.ProcessIdentifier != null) return this.ProcessIdentifier;
return "";
}
/// <summary>
/// Equality test.
/// </summary>
/// <param name="obj">Object to compare to.</param>
/// <returns>True if both objects represent the same process id.</returns>
public override bool Equals(object obj)
{
if (!(obj is DryadProcessIdentifier))
return false;
return this.Equals((DryadProcessIdentifier)obj);
}
#region IEquatable<DryadProcessIdentifier> Members
/// <summary>
/// Equality test.
/// </summary>
/// <param name="other">Process id to compare to.</param>
/// <returns>True if the id's represent the same process.</returns>
public bool Equals(DryadProcessIdentifier other)
{
if (this.IsUnknown)
return other.IsUnknown;
if (other.IsUnknown)
return false;
return this.ProcessIdentifier.Equals(other.ProcessIdentifier);
}
/// <summary>
/// Overriden implementation of getHashCode.
/// </summary>
/// <returns>The hashcode of the process id.</returns>
public override int GetHashCode()
{
// ReSharper disable once BaseObjectGetHashCodeCallInGetHashCode
return base.GetHashCode();
}
#endregion
}
/// <summary>
/// Brief summary of an executed DryadLINQ job.
/// </summary>
[Serializable]
public sealed class DryadLinqJobSummary : IEquatable<DryadLinqJobSummary>
{
/// <summary>
/// Empty constructor for XML serialization.
/// </summary>
public DryadLinqJobSummary()
{ }
/// <summary>
/// Initialize a job summary.
/// </summary>
/// <param name="cluster">Cluster where the job ran.</param>
/// <param name="clusterType">A string corresponding to the type of ClusterConfiguration.</param>
/// <param name="machine">Machine where job manager ran.</param>
/// <param name="jobId">Id of job.</param>
/// <param name="jmProcessGuid">Guid of job manager process.</param>
/// <param name="clusterJobId">Id of job on the cluster.</param>
/// <param name="friendlyname">Friendly name used.</param>
/// <param name="username">Who ran the job.</param>
/// <param name="date">Start date (not completion date).</param>
/// <param name="status">Job status.</param>
/// <param name="endTime">Estimated end running time.</param>
/// <param name="virtualcluster">Virtual cluster where job ran.</param>
public DryadLinqJobSummary(
string cluster,
ClusterConfiguration.ClusterType clusterType,
string virtualcluster,
string machine,
string jobId,
string clusterJobId,
DryadProcessIdentifier jmProcessGuid,
string friendlyname,
string username,
DateTime date,
DateTime endTime,
ClusterJobInformation.ClusterJobStatus status)
{
this.VirtualCluster = virtualcluster;
this.Cluster = cluster;
this.ClusterType = clusterType;
this.Machine = machine;
this.Name = friendlyname;
this.User = username;
this.Date = date;
this.EndTime = endTime;
this.Status = status;
this.ManagerProcessGuid = jmProcessGuid;
this.JobID = jobId;
this.ClusterJobId = clusterJobId;
}
/// <summary>
/// Cluster where the job ran.
/// </summary>
public string Cluster { get; /*private*/ set; }
/// <summary>
/// Id of cluster job that originated this DryadLinq job (can be used to find the cluster job from the dryadlinq job).
/// </summary>
public string ClusterJobId { get; /*private*/ set; }
/// <summary>
/// Cluster where the job ran.
/// </summary>
public DateTime EndTime { get; /*private*/ set; }
/// <summary>
/// String describing cluster type.
/// </summary>
public ClusterConfiguration.ClusterType ClusterType { get; /*private*/ set; }
/// <summary>
/// Virtual cluster where job ran.
/// </summary>
public string VirtualCluster { get; /*private*/ set; }
/// <summary>
/// (Friendly) name of the job.
/// </summary>
public string Name { get; /*private*/ set; }
/// <summary>
/// User who submitted job.
/// </summary>
public string User { get; /*private*/ set; }
/// <summary>
/// ID of job on the cluster.
/// </summary>
public string JobID { get; /*private*/ set; }
/// <summary>
/// The Guid of the job manager process.
/// </summary>
public DryadProcessIdentifier ManagerProcessGuid { set; get; }
/// <summary>
/// User who submitted job.
/// </summary>
public string GetAlias()
{
int pos = User.IndexOf(@"\");
return User.Substring(pos + 1);
}
/// <summary>
/// Date when job was submitted.
/// </summary>
public DateTime Date { get; /*private*/ set; }
/// <summary>
/// Did the job fail?
/// </summary>
public ClusterJobInformation.ClusterJobStatus Status { get; /*internal*/ set; }
/// <summary>
/// Machine where the job manager ran.
/// </summary>
public string Machine { get; /*private*/ set; }
/// <summary>
/// Get a short name for this job summary.
/// </summary>
/// <returns>Short name of job summary.</returns>
public string ShortName()
{
// we use the starting time to uniquify the job name
return this.Date.ToString("s") + "-" + this.Name;
}
/// <summary>
/// True if these two summaries are the same. The status and end time do not matter, since the job may still be running.
/// </summary>
/// <param name="other">Summary to compare against.</param>
/// <returns>True if they are equal.</returns>
public bool Equals(DryadLinqJobSummary other)
{
return this.Cluster == other.Cluster &&
this.ClusterJobId == other.ClusterJobId &&
this.Date == other.Date &&
this.Machine == other.Machine &&
this.Name == other.Name &&
this.User == other.User;
}
/// <summary>
/// Hashcode proper for the equality test.
/// </summary>
/// <returns>The object hashcode.</returns>
public override int GetHashCode()
{
return this.ClusterJobId.GetHashCode() ^ this.ClusterJobId.GetHashCode() ^ this.Date.GetHashCode() ^ this.Machine.GetHashCode() ^ this.Name.GetHashCode() ^ this.User.GetHashCode();
}
/// <summary>
/// A string describing the unique identifying part of the summary.
/// Two different summaries may represent the same job at different times.
/// </summary>
/// <returns>The part common to all jobs.</returns>
public string AsIdentifyingString()
{
StringBuilder builder = new StringBuilder();
builder.AppendFormat("Cluster={0} ClusterJobID={1} Date={2} Machine={3} Name={4} User={5}",
this.Cluster, this.ClusterJobId, this.Date, this.Machine, this.Name, this.User);
return builder.ToString();
}
}
/// <summary>
/// This class is an abstraction of a cluster-level job, as opposed to a DryadLINQ job.
/// In Cosmos that's called a task, in HPC that's called a Job.
/// (In cosmos a task is a recurring job. In DryadLINQ running on top of cosmos, a task always contains exactly one job.)
/// </summary>
public class ClusterJobInformation : IEquatable<ClusterJobInformation>
{
/// <summary>
/// Status of a cluster job.
/// </summary>
public enum ClusterJobStatus
{
/// <summary>
/// Job is still running.
/// </summary>
Running,
/// <summary>
/// Job has finished successfully.
/// </summary>
Succeeded,
/// <summary>
/// Job has finished and has failed.
/// </summary>
Failed,
/// <summary>
/// Job has been cancelled. Not precise on cosmos clusters.
/// </summary>
Cancelled,
/// <summary>
/// Could not determine job status.
/// </summary>
Unknown,
};
/// <summary>
/// True if job is finished, false if not, or unknown.
/// </summary>
/// <param name="status">Job status.</param>
/// <returns>True if the job is no longer running.</returns>
public static bool JobIsFinished(ClusterJobStatus status)
{
switch (status)
{
case ClusterJobInformation.ClusterJobStatus.Failed:
case ClusterJobInformation.ClusterJobStatus.Succeeded:
case ClusterJobInformation.ClusterJobStatus.Cancelled:
return true;
case ClusterJobInformation.ClusterJobStatus.Running:
case ClusterJobInformation.ClusterJobStatus.Unknown:
return false;
default:
throw new InvalidDataException("Invalid job status " + status);
}
}
/// <summary>
/// Create a cluster job structure from a bunch of information.
/// </summary>
/// <param name="cluster">Cluster where the job is running.</param>
/// <param name="clusterJobGuid">Cluster job guid.</param>
/// <param name="jobName">Name of the cluster job.</param>
/// <param name="username">User who submitted cluster job.</param>
/// <param name="date">Last execution of cluster job.</param>
/// <param name="status">Execution status.</param>
/// <param name="runningTime">Time the job ran.</param>
/// <param name="virtualCluster">Cluster where the job has run.</param>
public ClusterJobInformation(
string cluster,
string virtualCluster,
string clusterJobGuid,
string jobName,
string username,
DateTime date,
TimeSpan runningTime,
ClusterJobStatus status)
{
this.VirtualCluster = virtualCluster;
this.Cluster = cluster;
this.ClusterJobID = clusterJobGuid;
this.Name = jobName;
this.User = username;
this.Date = date;
this.EstimatedRunningTime = runningTime;
this.Status = status;
this.JobSummary = null;
}
/// <summary>
/// Name of cluster job.
/// </summary>
public string Name { get; set; }
/// <summary>
/// Uset who submitted cluster job.
/// </summary>
public string User { get; set; }
/// <summary>
/// Date when job was submitted.
/// </summary>
public DateTime Date { get; set; }
/// <summary>
/// ID of Job on cluster.
/// </summary>
public string ClusterJobID { get; set; }
/// <summary>
/// Status of the execution.
/// </summary>
public ClusterJobStatus Status { get; set; }
/// <summary>
/// Cluster where the job ran.
/// </summary>
public string Cluster { get; set; }
/// <summary>
/// In some installations a cluster is composed of multiple virtual clusters.
/// </summary>
public string VirtualCluster { get; set; }
/// <summary>
/// Is the cluster job information still available on the cluster?
/// </summary>
public bool IsUnavailable { get; set; }
/// <summary>
/// Cache here the associated job, if available. Null if not cached.
/// </summary>
private DryadLinqJobSummary JobSummary { get; set; }
/// <summary>
/// Estimated time the job ran.
/// </summary>
public TimeSpan EstimatedRunningTime { get; set; }
/// <summary>
/// If known, set the associated job summary.
/// </summary>
/// <param name="summary">Job summary for this cluster job.</param>
public void SetAssociatedSummary(DryadLinqJobSummary summary)
{
this.JobSummary = summary;
}
/// <summary>
/// Discover the dryadlinq job associated with a cluster job.
/// </summary>
/// <param name="status">Cluster configuration.</param>
/// <returns>The job, if any</returns>
/// <param name="reporter">Delegate used to report errors.</param>
public DryadLinqJobSummary DiscoverDryadLinqJob(ClusterStatus status, StatusReporter reporter)
{
if (this.IsUnavailable)
return null;
if (this.JobSummary != null)
return this.JobSummary;
DryadLinqJobSummary j = status.DiscoverDryadLinqJobFromClusterJob(this, reporter);
if (j == null)
{
this.IsUnavailable = true;
}
return this.JobSummary = j;
}
/// <summary>
/// Copy the content of a cluster job.
/// </summary>
/// <param name="refresh">The value to copy from.</param>
internal void Copy(ClusterJobInformation refresh)
{
this.Name = refresh.Name;
this.Status = refresh.Status;
this.User = refresh.User;
this.JobSummary = refresh.JobSummary;
this.ClusterJobID = refresh.ClusterJobID;
this.Date = refresh.Date;
this.IsUnavailable = refresh.IsUnavailable;
this.VirtualCluster = refresh.VirtualCluster;
this.EstimatedRunningTime = refresh.EstimatedRunningTime;
}
/// <summary>
/// True if these two records represent the same job.
/// </summary>
/// <param name="other"></param>
/// <returns></returns>
public bool Equals(ClusterJobInformation other)
{
return
this.Cluster == other.Cluster &&
this.VirtualCluster == other.VirtualCluster &&
this.Name == other.Name &&
this.User == other.User &&
this.ClusterJobID == other.ClusterJobID &&
this.Date == other.Date;
}
}
}

24
JobBrowser/JOM/app.config
View File

@ -4,15 +4,15 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
@ -20,7 +20,23 @@
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" /> <bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly> </dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>

372
JobBrowser/JOM/clusteraccess.cs
View File

@ -19,14 +19,20 @@ limitations under the License.
*/ */
using System.Linq;
using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Diagnostics; using System.Diagnostics;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Azure;
using Microsoft.Research.Tools; using Microsoft.Research.Tools;
using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.Research.Peloponnese.Shared;
using Microsoft.Research.Peloponnese.WebHdfs;
using Microsoft.Research.Peloponnese.Hdfs;
namespace Microsoft.Research.JobObjectModel namespace Microsoft.Research.JobObjectModel
{ {
@ -43,7 +49,8 @@ namespace Microsoft.Research.JobObjectModel
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder. /// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary> /// </summary>
/// <returns>A stream that can be used to access the object contents.</returns> /// <returns>A stream that can be used to access the object contents.</returns>
ISharedStreamReader GetStream(); /// <param name="keepNewline">If true the streamReader will not strip the newlines.</param>
ISharedStreamReader GetStream(bool keepNewline);
/// <summary> /// <summary>
/// If the current object is a folder, it returns the contained objects. /// If the current object is a folder, it returns the contained objects.
/// </summary> /// </summary>
@ -203,12 +210,13 @@ namespace Microsoft.Research.JobObjectModel
/// A stream to the local cache, or null if the file is not cached. /// A stream to the local cache, or null if the file is not cached.
/// </summary> /// </summary>
/// <returns>A stream to access the file.</returns> /// <returns>A stream to access the file.</returns>
public virtual ISharedStreamReader GetStream() /// <param name="keepNewline">If true keep newlines.</param>
public virtual ISharedStreamReader GetStream(bool keepNewline)
{ {
if (this.LocalCachePath != null && File.Exists(this.LocalCachePath)) if (this.LocalCachePath != null && File.Exists(this.LocalCachePath))
{ {
CachedClusterResidentObject.Record(this); CachedClusterResidentObject.Record(this);
return new FileSharedStreamReader(this.LocalCachePath); return new FileSharedStreamReader(this.LocalCachePath, keepNewline);
} }
return null; return null;
} }
@ -355,14 +363,15 @@ namespace Microsoft.Research.JobObjectModel
/// The stream with the file contents. /// The stream with the file contents.
/// </summary> /// </summary>
/// <returns>A stream reder.</returns> /// <returns>A stream reder.</returns>
public override ISharedStreamReader GetStream() /// <param name="keepNewlines">If true keep the newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewlines)
{ {
try try
{ {
if (!this.RepresentsAFolder) if (!this.RepresentsAFolder)
{ {
//this.LocalCachePath = this.CachePath(this.Pathname); //this.LocalCachePath = this.CachePath(this.Pathname);
ISharedStreamReader baseStream = base.GetStream(); ISharedStreamReader baseStream = base.GetStream(keepNewlines);
if (baseStream != null) if (baseStream != null)
{ {
// file is cached // file is cached
@ -378,12 +387,12 @@ namespace Microsoft.Research.JobObjectModel
throw new ClusterException("Cannot cache folders"); throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream(); StreamWriter writer = this.CreateTempStream();
return new FileSharedStreamReader(this.Pathname.ToString(), writer, this.OnClose); return new FileSharedStreamReader(this.Pathname.ToString(), writer, keepNewlines, this.OnClose);
} }
else else
{ {
// dont cache it // dont cache it
return new FileSharedStreamReader(this.Pathname.ToString()); return new FileSharedStreamReader(this.Pathname.ToString(), keepNewlines);
} }
} }
catch (Exception ex) catch (Exception ex)
@ -650,9 +659,10 @@ namespace Microsoft.Research.JobObjectModel
/// The contents of the folder. /// The contents of the folder.
/// </summary> /// </summary>
/// <returns>The contents of the folder.</returns> /// <returns>The contents of the folder.</returns>
public override ISharedStreamReader GetStream() /// <param name="keepNewline">If true keep newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewline)
{ {
return this.OriginalFolder.GetStream(); return this.OriginalFolder.GetStream(keepNewline);
} }
/// <summary> /// <summary>
@ -720,9 +730,10 @@ namespace Microsoft.Research.JobObjectModel
/// A stream returning the contents. /// A stream returning the contents.
/// </summary> /// </summary>
/// <returns>The contents of this object.</returns> /// <returns>The contents of this object.</returns>
public ISharedStreamReader GetStream() /// <param name="keepNewlines">If true keep newlines.</param>
public ISharedStreamReader GetStream(bool keepNewlines)
{ {
return new StringIteratorStreamReader(this.contents); return new StringIteratorStreamReader(this.contents, keepNewlines);
} }
/// <summary> /// <summary>
@ -835,9 +846,10 @@ namespace Microsoft.Research.JobObjectModel
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder. /// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary> /// </summary>
/// <returns>A stream that can be used to access the object contents.</returns> /// <returns>A stream that can be used to access the object contents.</returns>
public ISharedStreamReader GetStream() /// <param name="keepNewlines">If true keep the newlines.</param>
public ISharedStreamReader GetStream(bool keepNewlines)
{ {
return new FileSharedStreamReader(this.path); return new FileSharedStreamReader(this.path, keepNewlines);
} }
/// <summary> /// <summary>
@ -969,8 +981,8 @@ namespace Microsoft.Research.JobObjectModel
this.RepresentsAFolder = isFolder; this.RepresentsAFolder = isFolder;
this.size = -1; this.size = -1;
if (!string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory)) if (!String.IsNullOrEmpty(CacheDirectory))
this.LocalCachePath = Path.Combine(CachedClusterResidentObject.CacheDirectory, this.path); this.LocalCachePath = Path.Combine(CacheDirectory, this.path);
} }
/// <summary> /// <summary>
@ -982,14 +994,46 @@ namespace Microsoft.Research.JobObjectModel
protected set; protected set;
} }
/// <summary>
/// Create a File URI from a file path.
/// </summary>
/// <param name="path">Path to file.</param>
/// <returns>The File uri.</returns>
/// <param name="config">Azure cluster configuration.</param>
public static Uri UriFromPath(AzureDfsClusterConfiguration config, string path)
{
if (path.StartsWith(config.Container))
throw new InvalidOperationException("Path contains container name");
var retval = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(config.AccountName, config.Container, path, null, config.AccountKey);
//Console.WriteLine("Uri {0}", retval);
return retval;
}
/// <summary>
/// Create a path from a URI.
/// </summary>
/// <param name="uri">URI of an HDFS file.</param>
/// <returns>The path to the file.</returns>
/// <param name="config">Azure cluster configuration.</param>
public static string PathFromUri(AzureDfsClusterConfiguration config, Uri uri)
{
string path = uri.LocalPath;
path = path.Trim('/');
if (path.StartsWith(config.Container))
path = path.Substring(config.Container.Length);
path = path.Trim('/');
return path;
}
/// <summary> /// <summary>
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder. /// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary> /// </summary>
/// <returns>A stream that can be used to access the object contents.</returns> /// <returns>A stream that can be used to access the object contents.</returns>
public override ISharedStreamReader GetStream() /// <param name="keepNewlines">If true keep the newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewlines)
{ {
ISharedStreamReader baseStream = base.GetStream(); ISharedStreamReader baseStream = base.GetStream(keepNewlines);
if (baseStream != null) if (baseStream != null)
{ {
// file is cached // file is cached
@ -1000,29 +1044,30 @@ namespace Microsoft.Research.JobObjectModel
Stream stream; Stream stream;
if (this.IsDfsStream) if (this.IsDfsStream)
{ {
var dfsFileStream = this.client.GetDfsFileStream(this.path); Uri uri = UriFromPath(this.Config as AzureDfsClusterConfiguration, this.path);
stream = dfsFileStream.Stream; stream = this.client.GetDfsStreamReader(uri);
} }
else else
{ {
string p = AzureDfsClusterStatus.GetBlobName(this.client.ContainerName,this.path);
stream = new AzureLogReaderStream( stream = new AzureLogReaderStream(
this.client.AccountName, this.client.AccountName,
this.client.AccountKey, this.client.AccountKey,
this.client.ContainerName, this.client.ContainerName,
this.path); p);
} }
long size = this.Size; long sz = this.Size;
int bufferSize = 1024*1024; int bufferSize = 1024*1024;
if (size >= 0) if (sz >= 0)
{ {
bufferSize = (int)(size/10); bufferSize = (int)(sz/10);
if (bufferSize < 1024*1024) if (bufferSize < 1024*1024)
bufferSize = 1024*1024; bufferSize = 1024*1024;
if (bufferSize > 20*1024*1024) if (bufferSize > 20*1024*1024)
bufferSize = 20*1024*1024; bufferSize = 20*1024*1024;
} }
StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, false, bufferSize); SimpleStreamReader reader = new SimpleStreamReader(stream, true, Encoding.UTF8, false, bufferSize);
if (this.ShouldCacheLocally && this.LocalCachePath != null) if (this.ShouldCacheLocally && this.LocalCachePath != null)
{ {
@ -1030,12 +1075,12 @@ namespace Microsoft.Research.JobObjectModel
if (this.RepresentsAFolder) if (this.RepresentsAFolder)
throw new ClusterException("Cannot cache folders"); throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream(); StreamWriter writer = this.CreateTempStream();
return new SharedStreamReader(reader, writer, this.OnClose); return new SharedStreamReader(reader, writer, keepNewlines, this.OnClose);
} }
else else
{ {
// dont cache it // dont cache it
return new SharedStreamReader(reader); return new SharedStreamReader(reader, keepNewlines);
} }
} }
@ -1060,7 +1105,7 @@ namespace Microsoft.Research.JobObjectModel
if (item is CloudBlockBlob) if (item is CloudBlockBlob)
{ {
CloudBlockBlob blob = (CloudBlockBlob)item; CloudBlockBlob blob = (CloudBlockBlob)item;
blocks.Add(blob.Name, blob.Properties.Length); this.blocks.Add(blob.Name, blob.Properties.Length);
} }
else if (item is CloudPageBlob) else if (item is CloudPageBlob)
{ {
@ -1072,8 +1117,8 @@ namespace Microsoft.Research.JobObjectModel
if (metadata.ContainsKey("writePosition")) if (metadata.ContainsKey("writePosition"))
{ {
long sz; long sz;
if (long.TryParse(metadata["writePosition"], out sz)) if (Int64.TryParse(metadata["writePosition"], out sz))
pages.Add(pageBlob.Name, sz); this.pages.Add(pageBlob.Name, sz);
} }
} }
else if (item is CloudBlobDirectory) else if (item is CloudBlobDirectory)
@ -1094,29 +1139,28 @@ namespace Microsoft.Research.JobObjectModel
this.PopulateCache(); this.PopulateCache();
long length = -1; long length = -1;
foreach (var child in this.client.EnumerateDirectory(this.path)) Uri uri = UriFromPath(this.Config as AzureDfsClusterConfiguration, this.path);
Console.WriteLine("AzureDfsFile.GetFileAndFolders({0}) -> {1}", this.path, uri);
foreach (var child in this.client.ExpandFileOrDirectory(uri))
{ {
Regex re = Utilities.RegexFromSearchPattern(match); Regex re = Utilities.RegexFromSearchPattern(match);
if (!re.IsMatch(child)) continue; if (!re.IsMatch(child.AbsolutePath)) continue;
bool isFolder = false; bool isFolder = false;
bool isDfsStream = false; bool isDfsStream = false;
if (blocks.ContainsKey(child)) if (this.blocks.ContainsKey(child.AbsolutePath))
{ {
isDfsStream = true; isDfsStream = true;
length = blocks[child]; length = this.blocks[child.AbsolutePath];
} }
else if (pages.ContainsKey(child)) else if (this.pages.ContainsKey(child.AbsolutePath))
{ {
isDfsStream = false; isDfsStream = false;
length = pages[child]; length = this.pages[child.AbsolutePath];
} }
else if (this.client != null)
// otherwise this information may be incorrect
isFolder = true;
var file = new AzureDfsFile(this.Config, this.Job, this.client, child, this.ShouldCacheLocally, isFolder); var file = new AzureDfsFile(this.Config, this.Job, this.client, PathFromUri(this.Config as AzureDfsClusterConfiguration, child), this.ShouldCacheLocally, isFolder);
file.IsDfsStream = isDfsStream; file.IsDfsStream = isDfsStream;
file.size = length; file.size = length;
@ -1158,30 +1202,24 @@ namespace Microsoft.Research.JobObjectModel
{ {
this.PopulateCache(); this.PopulateCache();
string filepath; string combined = Path.Combine(this.path, filename);
if (this.client != null) Uri filepath = UriFromPath(this.Config as AzureDfsClusterConfiguration, combined);
filepath = this.client.Combine(this.path, filename);
else
filepath = Path.Combine(this.path, filename);
bool isFolder = false; bool isFolder = false;
bool isDfsStream = false; bool isDfsStream = false;
long sz = -1; long sz = -1;
if (blocks.ContainsKey(filepath)) if (this.blocks.ContainsKey(combined))
{ {
isDfsStream = true; isDfsStream = true;
sz = blocks[filepath]; sz = this.blocks[filepath.AbsolutePath];
} }
else if (pages.ContainsKey(filepath)) else if (this.pages.ContainsKey(combined))
{ {
isDfsStream = false; isDfsStream = false;
sz = pages[filepath]; sz = this.pages[filepath.AbsolutePath];
} }
else if (this.client != null)
// if the client is null the information may be incorrect
isFolder = true;
var file = new AzureDfsFile(this.Config, this.Job, this.client, filepath, this.ShouldCacheLocally, isFolder); var file = new AzureDfsFile(this.Config, this.Job, this.client, combined, this.ShouldCacheLocally, isFolder);
file.IsDfsStream = isDfsStream; file.IsDfsStream = isDfsStream;
file.size = sz; file.size = sz;
return file; return file;
@ -1213,4 +1251,232 @@ namespace Microsoft.Research.JobObjectModel
return this.path; return this.path;
} }
} }
/// <summary>
/// A file residing on HDFS (accessed either using HDFS or WebHdfs).
/// </summary>
public class DfsFile : CachedClusterResidentObject
{
private Uri baseUri;
private Uri uri;
private HdfsClientBase client;
/// <summary>
/// A file with the specified path.
/// </summary>
/// <param name="path">Path to the file.</param>
/// <param name="client">Azure client.</param>
/// <param name="config">Cluster configuration.</param>
/// <param name="job">Job accessing this file.</param>
/// <param name="jobFolderUri">Uri to base folder.</param>
/// <param name="isFolder">If true this must be a folder.</param>
/// <param name="canCache">True if the file can be cached (it is immutable for sure).</param>
public DfsFile(ClusterConfiguration config, Uri jobFolderUri, DryadLinqJobSummary job, HdfsClientBase client, string path, bool canCache, bool isFolder)
: base(config, job)
{
this.client = client;
this.Exception = null;
this.baseUri = jobFolderUri;
this.uri = UriFromPath(jobFolderUri, path);
this.ShouldCacheLocally = canCache;
this.RepresentsAFolder = isFolder;
this.size = -1;
Console.WriteLine("DfsFile Uri={0}", this.uri);
if (!string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
this.LocalCachePath = Path.Combine(CachedClusterResidentObject.CacheDirectory, PathFromUri(this.baseUri, this.uri));
}
/// <summary>
/// Create a File URI from a file path.
/// </summary>
/// <param name="baseUri">Base URI point to jobs folder.</param>
/// <param name="path">Path to file.</param>
/// <returns>The File uri.</returns>
public static Uri UriFromPath(Uri baseUri, string path)
{
UriBuilder builder = new UriBuilder(baseUri);
builder.Path = builder.Path.TrimEnd('/') + "/" + path.TrimStart('/');
return builder.Uri;
}
/// <summary>
/// Create a path from a URI.
/// </summary>
/// <param name="uri">URI of an HDFS file.</param>
/// <param name="baseUri">Base URI (should be a prefix of the other uri).</param>
/// <returns>The path to the file.</returns>
public static string PathFromUri(Uri baseUri, Uri uri)
{
string path = uri.PathAndQuery;
if (path.StartsWith(baseUri.PathAndQuery))
path = path.Substring(baseUri.PathAndQuery.Length);
path = path.Trim('/');
return path;
}
private DfsFile(ClusterConfiguration config, DryadLinqJobSummary job, Exception ex)
: base(config, job)
{
this.Exception = ex;
}
/// <summary>
/// True if the object is a folder.
/// </summary>
public override bool RepresentsAFolder
{
get;
protected set;
}
/// <summary>
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary>
/// <returns>A stream that can be used to access the object contents.</returns>
/// <param name="keepNewlines">If true keep the newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewlines)
{
ISharedStreamReader baseStream = base.GetStream(keepNewlines);
if (baseStream != null)
{
// file is cached
Trace.TraceInformation("Reading from local cache {0}", baseStream);
return baseStream;
}
Stream stream = this.client.GetDfsStreamReader(this.uri);
long sz = this.Size;
int bufferSize = 1024 * 1024;
if (sz >= 0)
{
bufferSize = (int)(sz / 10);
if (bufferSize < 1024 * 1024)
bufferSize = 1024 * 1024;
if (bufferSize > 20 * 1024 * 1024)
bufferSize = 20 * 1024 * 1024;
}
SimpleStreamReader reader = new SimpleStreamReader(stream, true, Encoding.UTF8, false, bufferSize);
if (this.ShouldCacheLocally && this.LocalCachePath != null)
{
// cache it
if (this.RepresentsAFolder)
throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream();
return new SharedStreamReader(reader, writer, keepNewlines, this.OnClose);
}
else
{
// dont cache it
return new SharedStreamReader(reader, keepNewlines);
}
}
/// <summary>
/// If the current object is a folder, it returns the contained objects.
/// </summary>
/// <returns>An iterator over all contained objects that match the specified string.</returns>
/// <param name="match">A shell expression (similar to the argument of Directory.GetFiles()).</param>
public override IEnumerable<IClusterResidentObject> GetFilesAndFolders(string match)
{
HashSet<Uri> folders = new HashSet<Uri>();
foreach (var child in this.client.EnumerateSubdirectories(this.uri))
{
folders.Add(child);
}
Regex re = Utilities.RegexFromSearchPattern(match);
foreach (var child in this.client.ExpandFileOrDirectory(this.uri))
{
if (!re.IsMatch(PathFromUri(this.baseUri, child))) continue;
bool isFolder = folders.Contains(child);
var file = new DfsFile(this.Config, this.baseUri, this.Job, this.client, PathFromUri(this.baseUri, child), this.ShouldCacheLocally, isFolder);
long length;
long time;
this.client.GetFileStatus(child, out time, out length);
file.size = length;
file.CreationTime = TimeFromLong(time);
yield return file;
}
}
private long size;
/// <summary>
/// Size of the object in bytes (if not a folder). The size can be -1 when it is unknown.
/// </summary>
public override long Size
{
get { return this.size; }
}
/// <summary>
/// Short name of the object.
/// </summary>
public override string Name
{
get { return Path.GetFileName(PathFromUri(this.baseUri, this.uri)); }
}
private static DateTime origin = new DateTime(1970, 1, 1);
/// <summary>
/// Convert file status time into a DateTime.
/// </summary>
/// <param name="fileTime">File time obtained from client.</param>
/// <returns>A DateTime object.</returns>
public static DateTime TimeFromLong(long fileTime)
{
return origin + TimeSpan.FromMilliseconds(fileTime);
}
/// <summary>
/// Date when object was created.
/// </summary>
public override DateTime CreationTime
{
get { return DateTime.Now; }
}
/// <summary>
/// For a folder object, returns the contained file with the specified name.
/// </summary>
/// <param name="filename">File name within the folder.</param>
/// <returns>The file within the folder.</returns>
public override IClusterResidentObject GetFile(string filename)
{
var matchingFiles = this.GetFilesAndFolders(filename).ToList();
if (matchingFiles.Count == 1)
return matchingFiles[0];
return new DfsFile(this.Config, this.Job, new Exception("Ambiguous name " + filename + " in " + this));
}
/// <summary>
/// For a folder object, returns the contained folder with the specified name.
/// </summary>
/// <param name="foldername">Folder name within the folder.</param>
/// <returns>The subfolder within the folder.</returns>
public override IClusterResidentObject GetFolder(string foldername)
{
var file = this.GetFile(foldername);
if (!file.RepresentsAFolder) throw new InvalidOperationException(foldername + " is not a folder");
return file;
}
/// <summary>
/// Returns a string that represents the current object.
/// </summary>
/// <returns>
/// A string that represents the current object.
/// </returns>
/// <filterpriority>2</filterpriority>
public override string ToString()
{
return this.uri.ToString();
}
}
} }

47
JobBrowser/JOM/jobinfo.cs
View File

@ -19,8 +19,6 @@ limitations under the License.
*/ */
#undef USE_LINQ_TO_DRYAD
#undef USE_HPC
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
@ -660,6 +658,13 @@ namespace Microsoft.Research.JobObjectModel
return 0; return 0;
} }
/// <summary>
/// Sometimes the StreamReader can read a partial line only.
/// We cache here the previously read line and if it looks like the current line is just a fragment, then we concatentate it
/// to the previous line. The line is a fragment if it does not start with 'logtimelocal'.
/// </summary>
private string previousLine = "";
/// <summary> /// <summary>
/// New JM stdout parsing code, for YARN-based DryadLINQ. /// New JM stdout parsing code, for YARN-based DryadLINQ.
/// </summary> /// </summary>
@ -668,6 +673,17 @@ namespace Microsoft.Research.JobObjectModel
private bool ParseStdoutLineNew(string line) private bool ParseStdoutLineNew(string line)
{ {
if (string.IsNullOrWhiteSpace(line)) return true; if (string.IsNullOrWhiteSpace(line)) return true;
// The line should start with logtimelocal, otherwise it's probably a fragment of the previous line
this.previousLine += line;
if (!(line.EndsWith("\r") || line.EndsWith("\n")))
{
// line is incomplete, return now, parse later
return true;
}
line = this.previousLine.Trim();
this.previousLine = "";
Dictionary<string, string> kvp = Utilities.ParseCSVKVP(line); Dictionary<string, string> kvp = Utilities.ParseCSVKVP(line);
if (kvp == null) return false; if (kvp == null) return false;
@ -848,7 +864,7 @@ namespace Microsoft.Research.JobObjectModel
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version); ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
long totalRead = TryGetNumeric(kvp, "totalRead"); long totalRead = TryGetNumeric(kvp, "totalRead");
long tempRead = TryGetNumeric(kvp, "tempRead"); //long tempRead = TryGetNumeric(kvp, "tempRead");
long tempReadInRack = TryGetNumeric(kvp, "tempReadInRack"); long tempReadInRack = TryGetNumeric(kvp, "tempReadInRack");
long tempReadCrossRack = TryGetNumeric(kvp, "tempReadCrossRack"); long tempReadCrossRack = TryGetNumeric(kvp, "tempReadCrossRack");
long localRead = TryGetNumeric(kvp, "localRead"); long localRead = TryGetNumeric(kvp, "localRead");
@ -1456,7 +1472,7 @@ namespace Microsoft.Research.JobObjectModel
manager.Status(message, StatusKind.LongOp); manager.Status(message, StatusKind.LongOp);
if (this.cachedStdoutReader == null) if (this.cachedStdoutReader == null)
this.cachedStdoutReader = file.GetStream(); this.cachedStdoutReader = file.GetStream(true);
if (this.cachedStdoutReader.Exception != null) if (this.cachedStdoutReader.Exception != null)
{ {
manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error); manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
@ -1472,11 +1488,15 @@ namespace Microsoft.Research.JobObjectModel
while (true) while (true)
{ {
manager.Token.ThrowIfCancellationRequested(); manager.Token.ThrowIfCancellationRequested();
int startLine = currentLine;
bool completeLine = true; bool completeLine = true;
try try
{ {
completeLine = this.ParseStdoutLineNew(line); if (this.ClusterConfiguration is DfsClusterConfiguration)
completeLine = this.ParseStdoutLineNew(line);
else
{
this.ParseStdoutLine(line);
}
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -1486,9 +1506,8 @@ namespace Microsoft.Research.JobObjectModel
if (!completeLine) if (!completeLine)
{ {
if (this.cachedStdoutReader.EndOfStream) if (this.cachedStdoutReader.EndOfStream)
{ // no exception, the log may be truncated
throw new Exception("File ended while scanning for closing quote started on line " + startLine); break;
}
string extraline = this.cachedStdoutReader.ReadLine(); string extraline = this.cachedStdoutReader.ReadLine();
line += "\n" + extraline; line += "\n" + extraline;
@ -1599,7 +1618,7 @@ namespace Microsoft.Research.JobObjectModel
{ {
bool success = true; bool success = true;
ISharedStreamReader sr = logfile.GetStream(); ISharedStreamReader sr = logfile.GetStream(false);
if (sr.Exception != null) if (sr.Exception != null)
{ {
statusReporter("Exception while opening file " + logfile + ":" + sr.Exception.Message, StatusKind.Error); statusReporter("Exception while opening file " + logfile + ":" + sr.Exception.Message, StatusKind.Error);
@ -2042,7 +2061,7 @@ namespace Microsoft.Research.JobObjectModel
{ {
DryadJobStaticPlan retval; DryadJobStaticPlan retval;
{ {
retval = new DryadLinqJobStaticPlan(config, file.GetStream()); retval = new DryadLinqJobStaticPlan(config, file.GetStream(false));
} }
retval.ParseQueryPlan(manager); retval.ParseQueryPlan(manager);
return retval; return retval;
@ -2723,7 +2742,7 @@ namespace Microsoft.Research.JobObjectModel
{ {
uri = uri.Substring(0, option); uri = uri.Substring(0, option);
} }
this.LocalPath = uri.Substring(sepindex + 3); this.LocalPath = uri.Substring(sepindex + 3).Trim();
if (uripathprefix != null) { if (uripathprefix != null) {
// Unfortunately the uri is absolute, although it should be relative sometimes. We fix this here. // Unfortunately the uri is absolute, although it should be relative sometimes. We fix this here.
@ -3293,7 +3312,7 @@ namespace Microsoft.Research.JobObjectModel
if (this.InputChannels != null) if (this.InputChannels != null)
// skip discovery // skip discovery
inputs = false; inputs = false;
ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream(); ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream(false);
var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, manager); var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, manager);
if (channels == null) if (channels == null)
{ {
@ -3336,7 +3355,7 @@ namespace Microsoft.Research.JobObjectModel
manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error); manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error);
return false; return false;
} }
ISharedStreamReader sr = files.First().GetStream(); ISharedStreamReader sr = files.First().GetStream(false);
if (sr.Exception != null) if (sr.Exception != null)
{ {
manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error); manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);

38
JobBrowser/JOM/packages.config
View File

@ -1,21 +1,25 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" /> <package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" /> <package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" /> <package id="Microsoft.Bcl.Build" version="1.0.21" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" /> <package id="Microsoft.Hadoop.Client" version="1.3.2.0" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" /> <package id="Microsoft.Net.Http" version="2.2.28" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Client" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.WebHdfs" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.3.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.1.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.3.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Management.Storage" version="3.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" /> <package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" /> <package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" /> <package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
</packages> </packages>

2
JobBrowser/JOM/storage.cs
View File

@ -19,8 +19,6 @@ limitations under the License.
*/ */
#undef USE_DSC
#undef USE_TIDYFS
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;

40
JobBrowser/JobBrowser/ClusterBrowser.cs
View File

@ -19,6 +19,7 @@ limitations under the License.
*/ */
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
@ -142,6 +143,8 @@ namespace Microsoft.Research.DryadAnalysis
ClusterConfiguration config = ClusterConfiguration.KnownClusterByName(clusterName); ClusterConfiguration config = ClusterConfiguration.KnownClusterByName(clusterName);
this.Configuration = config; this.Configuration = config;
this.diagnoseToolStripMenuItem.Visible = config.SupportsDiagnosis;
this.diagnoseToolStripMenuItem1.Visible = config.SupportsDiagnosis;
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -602,12 +605,8 @@ namespace Microsoft.Research.DryadAnalysis
ToolStripItem strip = item.OwnerItem; ToolStripItem strip = item.OwnerItem;
string clus = strip.Text; string clus = strip.Text;
var config = this.EditCluster(clus); var conf = this.EditCluster(clus);
if (config != null) this.ConfigurationChanged(conf);
{
ClusterConfiguration.AddKnownCluster(config);
this.AddClusterNameToMenu(config.Name);
}
} }
void selItem_Click(object sender, EventArgs e) void selItem_Click(object sender, EventArgs e)
@ -620,9 +619,8 @@ namespace Microsoft.Research.DryadAnalysis
this.ClusterSelected(clus); this.ClusterSelected(clus);
} }
void AddNewCluster(object sender, EventArgs e) private void ConfigurationChanged(ClusterConfiguration conf)
{ {
ClusterConfiguration conf = this.EditCluster(null);
if (conf == null) return; if (conf == null) return;
// you cannot have two cache clusters at the same time // you cannot have two cache clusters at the same time
@ -650,6 +648,13 @@ namespace Microsoft.Research.DryadAnalysis
ClusterConfiguration.AddKnownCluster(conf); ClusterConfiguration.AddKnownCluster(conf);
this.AddClusterNameToMenu(conf.Name); this.AddClusterNameToMenu(conf.Name);
this.Status("Added cluster " + conf.Name, StatusKind.OK);
}
void AddNewCluster(object sender, EventArgs e)
{
ClusterConfiguration conf = this.EditCluster(null);
this.ConfigurationChanged(conf);
} }
/// <summary> /// <summary>
@ -661,10 +666,23 @@ namespace Microsoft.Research.DryadAnalysis
{ {
ClusterConfigEditor editor = new ClusterConfigEditor(); ClusterConfigEditor editor = new ClusterConfigEditor();
if (clusterName != null) try
{ {
var config = ClusterConfiguration.KnownClusterByName(clusterName); if (clusterName != null)
editor.SetConfigToEdit(config); {
var config = ClusterConfiguration.KnownClusterByName(clusterName);
editor.SetConfigToEdit(config);
}
else
{
editor.SetConfigToEdit(null);
}
}
catch (Exception)
{
// This can happen when the cluster serialization has changed
// and we can no longer read the saved properties
editor.SetConfigToEdit(null);
} }
DialogResult res = editor.ShowDialog(); DialogResult res = editor.ShowDialog();
if (res == System.Windows.Forms.DialogResult.OK) if (res == System.Windows.Forms.DialogResult.OK)

133
JobBrowser/JobBrowser/ClusterConfigEditor.Designer.cs generated
View File

@ -49,13 +49,14 @@ namespace Microsoft.Research.JobObjectModel
private void InitializeComponent() private void InitializeComponent()
{ {
this.tableLayoutPanel_buttons = new System.Windows.Forms.TableLayoutPanel(); this.tableLayoutPanel_buttons = new System.Windows.Forms.TableLayoutPanel();
this.buttonOK = new System.Windows.Forms.Button();
this.buttonCancel = new System.Windows.Forms.Button(); this.buttonCancel = new System.Windows.Forms.Button();
this.buttonOK = new System.Windows.Forms.Button();
this.tableLayoutPanel_properties = new System.Windows.Forms.TableLayoutPanel(); this.tableLayoutPanel_properties = new System.Windows.Forms.TableLayoutPanel();
this.label_name = new System.Windows.Forms.Label();
this.textBox_name = new System.Windows.Forms.TextBox(); this.textBox_name = new System.Windows.Forms.TextBox();
this.label1 = new System.Windows.Forms.Label(); this.label_type = new System.Windows.Forms.Label();
this.label2 = new System.Windows.Forms.Label();
this.comboBox_clusterType = new System.Windows.Forms.ComboBox(); this.comboBox_clusterType = new System.Windows.Forms.ComboBox();
this.label_description = new System.Windows.Forms.Label();
this.panel1 = new System.Windows.Forms.Panel(); this.panel1 = new System.Windows.Forms.Panel();
this.tableLayoutPanel_buttons.SuspendLayout(); this.tableLayoutPanel_buttons.SuspendLayout();
this.tableLayoutPanel_properties.SuspendLayout(); this.tableLayoutPanel_properties.SuspendLayout();
@ -69,14 +70,26 @@ namespace Microsoft.Research.JobObjectModel
this.tableLayoutPanel_buttons.Controls.Add(this.buttonCancel, 1, 0); this.tableLayoutPanel_buttons.Controls.Add(this.buttonCancel, 1, 0);
this.tableLayoutPanel_buttons.Controls.Add(this.buttonOK, 0, 0); this.tableLayoutPanel_buttons.Controls.Add(this.buttonOK, 0, 0);
this.tableLayoutPanel_buttons.Dock = System.Windows.Forms.DockStyle.Bottom; this.tableLayoutPanel_buttons.Dock = System.Windows.Forms.DockStyle.Bottom;
this.tableLayoutPanel_buttons.Location = new System.Drawing.Point(0, 143); this.tableLayoutPanel_buttons.Location = new System.Drawing.Point(0, 216);
this.tableLayoutPanel_buttons.Name = "tableLayoutPanel_buttons"; this.tableLayoutPanel_buttons.Name = "tableLayoutPanel_buttons";
this.tableLayoutPanel_buttons.RowCount = 1; this.tableLayoutPanel_buttons.RowCount = 1;
this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100F)); this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100F));
this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 20F)); this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 33F));
this.tableLayoutPanel_buttons.Size = new System.Drawing.Size(381, 33); this.tableLayoutPanel_buttons.Size = new System.Drawing.Size(389, 33);
this.tableLayoutPanel_buttons.TabIndex = 0; this.tableLayoutPanel_buttons.TabIndex = 0;
// //
// buttonCancel
//
this.buttonCancel.DialogResult = System.Windows.Forms.DialogResult.Cancel;
this.buttonCancel.Dock = System.Windows.Forms.DockStyle.Fill;
this.buttonCancel.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.buttonCancel.Location = new System.Drawing.Point(197, 3);
this.buttonCancel.Name = "buttonCancel";
this.buttonCancel.Size = new System.Drawing.Size(189, 27);
this.buttonCancel.TabIndex = 1;
this.buttonCancel.Text = "Cancel";
this.buttonCancel.UseVisualStyleBackColor = true;
//
// buttonOK // buttonOK
// //
this.buttonOK.DialogResult = System.Windows.Forms.DialogResult.OK; this.buttonOK.DialogResult = System.Windows.Forms.DialogResult.OK;
@ -84,101 +97,104 @@ namespace Microsoft.Research.JobObjectModel
this.buttonOK.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); this.buttonOK.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.buttonOK.Location = new System.Drawing.Point(3, 3); this.buttonOK.Location = new System.Drawing.Point(3, 3);
this.buttonOK.Name = "buttonOK"; this.buttonOK.Name = "buttonOK";
this.buttonOK.Size = new System.Drawing.Size(184, 27); this.buttonOK.Size = new System.Drawing.Size(188, 27);
this.buttonOK.TabIndex = 0; this.buttonOK.TabIndex = 0;
this.buttonOK.Text = "OK"; this.buttonOK.Text = "OK";
this.buttonOK.UseVisualStyleBackColor = true; this.buttonOK.UseVisualStyleBackColor = true;
// //
// buttonCancel
//
this.buttonCancel.DialogResult = System.Windows.Forms.DialogResult.Cancel;
this.buttonCancel.Dock = System.Windows.Forms.DockStyle.Fill;
this.buttonCancel.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.buttonCancel.Location = new System.Drawing.Point(193, 3);
this.buttonCancel.Name = "buttonCancel";
this.buttonCancel.Size = new System.Drawing.Size(185, 27);
this.buttonCancel.TabIndex = 1;
this.buttonCancel.Text = "Cancel";
this.buttonCancel.UseVisualStyleBackColor = true;
//
// tableLayoutPanel_properties // tableLayoutPanel_properties
// //
this.tableLayoutPanel_properties.AutoSize = true; this.tableLayoutPanel_properties.AutoSize = true;
this.tableLayoutPanel_properties.ColumnCount = 2; this.tableLayoutPanel_properties.ColumnCount = 2;
this.tableLayoutPanel_properties.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle()); this.tableLayoutPanel_properties.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle());
this.tableLayoutPanel_properties.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100F)); this.tableLayoutPanel_properties.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100F));
this.tableLayoutPanel_properties.Controls.Add(this.label2, 0, 0); this.tableLayoutPanel_properties.Controls.Add(this.comboBox_clusterType, 1, 2);
this.tableLayoutPanel_properties.Controls.Add(this.textBox_name, 1, 0); this.tableLayoutPanel_properties.Controls.Add(this.textBox_name, 1, 1);
this.tableLayoutPanel_properties.Controls.Add(this.label1, 0, 1); this.tableLayoutPanel_properties.Controls.Add(this.label_name, 0, 1);
this.tableLayoutPanel_properties.Controls.Add(this.comboBox_clusterType, 1, 1); this.tableLayoutPanel_properties.Controls.Add(this.label_description, 0, 0);
this.tableLayoutPanel_properties.Controls.Add(this.label_type, 0, 2);
this.tableLayoutPanel_properties.Dock = System.Windows.Forms.DockStyle.Top; this.tableLayoutPanel_properties.Dock = System.Windows.Forms.DockStyle.Top;
this.tableLayoutPanel_properties.Location = new System.Drawing.Point(0, 0); this.tableLayoutPanel_properties.Location = new System.Drawing.Point(0, 0);
this.tableLayoutPanel_properties.Name = "tableLayoutPanel_properties"; this.tableLayoutPanel_properties.Name = "tableLayoutPanel_properties";
this.tableLayoutPanel_properties.RowCount = 2; this.tableLayoutPanel_properties.RowCount = 3;
this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle()); this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle());
this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle()); this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle());
this.tableLayoutPanel_properties.Size = new System.Drawing.Size(381, 54); this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle());
this.tableLayoutPanel_properties.Size = new System.Drawing.Size(389, 66);
this.tableLayoutPanel_properties.TabIndex = 1; this.tableLayoutPanel_properties.TabIndex = 1;
// //
// label_name
//
this.label_name.AutoSize = true;
this.label_name.Dock = System.Windows.Forms.DockStyle.Fill;
this.label_name.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.label_name.Location = new System.Drawing.Point(3, 19);
this.label_name.Name = "label_name";
this.label_name.Size = new System.Drawing.Size(80, 27);
this.label_name.TabIndex = 5;
this.label_name.Text = "Cluster name";
this.label_name.TextAlign = System.Drawing.ContentAlignment.MiddleRight;
//
// textBox_name // textBox_name
// //
this.textBox_name.Dock = System.Windows.Forms.DockStyle.Fill; this.textBox_name.Dock = System.Windows.Forms.DockStyle.Fill;
this.textBox_name.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); this.textBox_name.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.textBox_name.Location = new System.Drawing.Point(89, 3); this.textBox_name.Location = new System.Drawing.Point(89, 22);
this.textBox_name.Name = "textBox_name"; this.textBox_name.Name = "textBox_name";
this.textBox_name.Size = new System.Drawing.Size(289, 21); this.textBox_name.Size = new System.Drawing.Size(297, 21);
this.textBox_name.TabIndex = 3; this.textBox_name.TabIndex = 3;
// //
// label1 // label_type
// //
this.label1.AutoSize = true; this.label_type.AutoSize = true;
this.label1.Dock = System.Windows.Forms.DockStyle.Fill; this.label_type.Dock = System.Windows.Forms.DockStyle.Fill;
this.label1.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); this.label_type.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.label1.Location = new System.Drawing.Point(3, 27); this.label_type.Location = new System.Drawing.Point(3, 46);
this.label1.Name = "label1"; this.label_type.Name = "label_type";
this.label1.Size = new System.Drawing.Size(80, 27); this.label_type.Size = new System.Drawing.Size(80, 20);
this.label1.TabIndex = 4; this.label_type.TabIndex = 4;
this.label1.Text = "Cluster type"; this.label_type.Text = "Cluster type";
this.label1.TextAlign = System.Drawing.ContentAlignment.MiddleRight; this.label_type.TextAlign = System.Drawing.ContentAlignment.MiddleRight;
//
// label2
//
this.label2.AutoSize = true;
this.label2.Dock = System.Windows.Forms.DockStyle.Fill;
this.label2.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.label2.Location = new System.Drawing.Point(3, 0);
this.label2.Name = "label2";
this.label2.Size = new System.Drawing.Size(80, 27);
this.label2.TabIndex = 5;
this.label2.Text = "Cluster name";
this.label2.TextAlign = System.Drawing.ContentAlignment.MiddleRight;
// //
// comboBox_clusterType // comboBox_clusterType
// //
this.comboBox_clusterType.Dock = System.Windows.Forms.DockStyle.Fill; this.comboBox_clusterType.Dock = System.Windows.Forms.DockStyle.Fill;
this.comboBox_clusterType.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); this.comboBox_clusterType.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.comboBox_clusterType.FormattingEnabled = true; this.comboBox_clusterType.IntegralHeight = false;
this.comboBox_clusterType.Location = new System.Drawing.Point(89, 30); this.comboBox_clusterType.Location = new System.Drawing.Point(89, 49);
this.comboBox_clusterType.Name = "comboBox_clusterType"; this.comboBox_clusterType.Name = "comboBox_clusterType";
this.comboBox_clusterType.Size = new System.Drawing.Size(289, 23); this.comboBox_clusterType.Size = new System.Drawing.Size(297, 23);
this.comboBox_clusterType.TabIndex = 6; this.comboBox_clusterType.TabIndex = 6;
this.comboBox_clusterType.SelectedIndexChanged += new System.EventHandler(this.comboBox_clusterType_SelectedIndexChanged); this.comboBox_clusterType.SelectedIndexChanged += new System.EventHandler(this.comboBox_clusterType_SelectedIndexChanged);
// //
// label_description
//
this.label_description.AutoSize = true;
this.tableLayoutPanel_properties.SetColumnSpan(this.label_description, 2);
this.label_description.Dock = System.Windows.Forms.DockStyle.Fill;
this.label_description.Location = new System.Drawing.Point(3, 3);
this.label_description.Margin = new System.Windows.Forms.Padding(3);
this.label_description.Name = "label_description";
this.label_description.Size = new System.Drawing.Size(383, 13);
this.label_description.TabIndex = 7;
this.label_description.Text = "Description";
this.label_description.TextAlign = System.Drawing.ContentAlignment.MiddleCenter;
//
// panel1 // panel1
// //
this.panel1.Dock = System.Windows.Forms.DockStyle.Fill; this.panel1.Dock = System.Windows.Forms.DockStyle.Fill;
this.panel1.Location = new System.Drawing.Point(0, 54); this.panel1.Location = new System.Drawing.Point(0, 0);
this.panel1.Name = "panel1"; this.panel1.Name = "panel1";
this.panel1.Size = new System.Drawing.Size(381, 89); this.panel1.Size = new System.Drawing.Size(389, 216);
this.panel1.TabIndex = 2; this.panel1.TabIndex = 2;
// //
// ClusterConfigEditor // ClusterConfigEditor
// //
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(381, 176); this.ClientSize = new System.Drawing.Size(389, 249);
this.Controls.Add(this.panel1);
this.Controls.Add(this.tableLayoutPanel_properties); this.Controls.Add(this.tableLayoutPanel_properties);
this.Controls.Add(this.panel1);
this.Controls.Add(this.tableLayoutPanel_buttons); this.Controls.Add(this.tableLayoutPanel_buttons);
this.Name = "ClusterConfigEditor"; this.Name = "ClusterConfigEditor";
this.Text = "Cluster Editor"; this.Text = "Cluster Editor";
@ -196,11 +212,12 @@ namespace Microsoft.Research.JobObjectModel
private System.Windows.Forms.Button buttonCancel; private System.Windows.Forms.Button buttonCancel;
private System.Windows.Forms.Button buttonOK; private System.Windows.Forms.Button buttonOK;
private System.Windows.Forms.TableLayoutPanel tableLayoutPanel_properties; private System.Windows.Forms.TableLayoutPanel tableLayoutPanel_properties;
private System.Windows.Forms.Label label2; private System.Windows.Forms.Label label_name;
private System.Windows.Forms.TextBox textBox_name; private System.Windows.Forms.TextBox textBox_name;
private System.Windows.Forms.Label label1; private System.Windows.Forms.Label label_type;
private System.Windows.Forms.ComboBox comboBox_clusterType; private System.Windows.Forms.ComboBox comboBox_clusterType;
private System.Windows.Forms.Panel panel1; private System.Windows.Forms.Panel panel1;
private System.Windows.Forms.Label label_description;
} }
} }

18
JobBrowser/JobBrowser/ClusterConfigEditor.cs
View File

@ -23,7 +23,6 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Drawing; using System.Drawing;
using System.Windows.Forms; using System.Windows.Forms;
using Microsoft.Research.Tools;
namespace Microsoft.Research.JobObjectModel namespace Microsoft.Research.JobObjectModel
{ {
@ -43,9 +42,9 @@ namespace Microsoft.Research.JobObjectModel
this.addedControls = new List<Control>(); this.addedControls = new List<Control>();
this.propEditor = new Dictionary<string, TextBox>(); this.propEditor = new Dictionary<string, TextBox>();
for (var i = ClusterConfiguration.ClusterType.Unknown+1; i < ClusterConfiguration.ClusterType.MaxUnused; i++) foreach (var type in ClusterConfiguration.Available)
{ {
this.comboBox_clusterType.Items.Add(i.ToString()); this.comboBox_clusterType.Items.Add(type.ToString());
} }
} }
@ -72,6 +71,7 @@ namespace Microsoft.Research.JobObjectModel
this.textBox_name.Text = props.Name; this.textBox_name.Text = props.Name;
this.comboBox_clusterType.SelectedItem = props.Type.ToString(); this.comboBox_clusterType.SelectedItem = props.Type.ToString();
this.propEditor.Clear();
this.AddPropertiesToEdit(props.Properties); this.AddPropertiesToEdit(props.Properties);
} }
@ -84,6 +84,8 @@ namespace Microsoft.Research.JobObjectModel
/// <param name="props">Properties to add controls for.</param> /// <param name="props">Properties to add controls for.</param>
private void AddPropertiesToEdit(List<PropertySetting> props) private void AddPropertiesToEdit(List<PropertySetting> props)
{ {
this.label_description.Text = this.config.Description;
foreach (var prop in props) foreach (var prop in props)
{ {
Label labl = new Label(); Label labl = new Label();
@ -125,7 +127,7 @@ namespace Microsoft.Research.JobObjectModel
{ {
var type = (ClusterConfiguration.ClusterType)Enum.Parse(typeof(ClusterConfiguration.ClusterType), this.comboBox_clusterType.Text); var type = (ClusterConfiguration.ClusterType)Enum.Parse(typeof(ClusterConfiguration.ClusterType), this.comboBox_clusterType.Text);
ClusterConfigurationSerialization ser = new ClusterConfigurationSerialization() ClusterConfigurationSerialization ser = new ClusterConfigurationSerialization
{ {
Name = this.textBox_name.Text, Name = this.textBox_name.Text,
Type = type, Type = type,
@ -150,13 +152,15 @@ namespace Microsoft.Research.JobObjectModel
private void comboBox_clusterType_SelectedIndexChanged(object sender, EventArgs e) private void comboBox_clusterType_SelectedIndexChanged(object sender, EventArgs e)
{ {
// initial setting for an existing configuration if (!this.canChangeType)
if (this.config != null) return; // we are just editing the cluster properties
return;
this.RemoveAddedProperties(); this.RemoveAddedProperties();
var type = (ClusterConfiguration.ClusterType)Enum.Parse(typeof(ClusterConfiguration.ClusterType), this.comboBox_clusterType.Text); var type = (ClusterConfiguration.ClusterType)Enum.Parse(typeof(ClusterConfiguration.ClusterType), this.comboBox_clusterType.Text);
this.config = ClusterConfiguration.CreateConfiguration(type); this.config = ClusterConfiguration.CreateConfiguration(type);
this.AddPropertiesToEdit(this.config.ExtractData().Properties); var serialization = this.config.ExtractData();
this.AddPropertiesToEdit(serialization.Properties);
} }
} }
} }

Some files were not shown because too many files have changed in this diff Show More