Compare commits
15 Commits
| Author | SHA1 | Date |
|---|---|---|
misard
|
ad28579dd8 | |
|
misard
|
6980be9895 | |
|
misard
|
e72fef34c6 | |
|
Michael Isard
|
04ecff4353 | |
|
misard
|
5e9dde8ab6 | |
|
Michael Isard
|
47d0131083 | |
|
misard
|
b31fadd55e | |
|
misard
|
d3643fc5fb | |
|
misard
|
efc79b749b | |
|
Michael Isard
|
b9cda36f69 | |
Dennis Fetterly
|
5c260ffe0b | |
|
Michael Isard
|
93be3e4b66 | |
|
Michael Isard
|
ab56207d72 | |
|
Michael Isard
|
86242db082 | |
|
Michael Isard
|
5ad103cba7 |
|
|
@ -19,3 +19,5 @@ x64/
|
|||
build/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
/Calypso/packages
|
||||
/DryadLinqTests/packages
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
|
|
@ -9,10 +10,11 @@
|
|||
<OutputType>Library</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>Microsoft.Research.Dryad.ClusterInterface</RootNamespace>
|
||||
<AssemblyName>DryadHttpClusterInterface</AssemblyName>
|
||||
<AssemblyName>Microsoft.Research.Dryad.HttpClusterInterface</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<TargetFrameworkProfile />
|
||||
<NuGetPackageImportStamp>0025dfe5</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
|
|
@ -33,60 +35,12 @@
|
|||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Data.Services.Client" />
|
||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
|
|
@ -95,24 +49,34 @@
|
|||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\SharedAssemblyInfo.cs">
|
||||
<Link>Properties\SharedAssemblyInfo.cs</Link>
|
||||
</Compile>
|
||||
<Compile Include="Interfaces.cs" />
|
||||
<Compile Include="HttpScheduler.cs" />
|
||||
<Compile Include="Process.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="app.config" />
|
||||
<None Include="packages.config" />
|
||||
<None Include="app.config">
|
||||
<SubType>Designer</SubType>
|
||||
</None>
|
||||
<None Include="packages.config">
|
||||
<SubType>Designer</SubType>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
|
|
|
|||
|
|
@ -90,10 +90,10 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
return fileServer.Uri.AbsoluteUri;
|
||||
}
|
||||
|
||||
public IProcess NewProcess(IProcessWatcher watcher, string commandLine)
|
||||
public IProcess NewProcess(IProcessWatcher watcher, string commandLine, string commandLineArguments)
|
||||
{
|
||||
ISchedulerProcess process = scheduler.NewProcess();
|
||||
return new Process(process, watcher, commandLine, logger);
|
||||
return new Process(process, watcher, commandLine, commandLineArguments, logger);
|
||||
}
|
||||
|
||||
public void ScheduleProcess(IProcess ip, List<Affinity> affinities)
|
||||
|
|
|
|||
|
|
@ -368,8 +368,9 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
/// the application on the process' lifecycle</param>
|
||||
/// <param name="commandLine">the command line to use to start the process on the remote
|
||||
/// computer</param>
|
||||
/// <param name="commandLineArguments">arguments to provide to the remote process</param>
|
||||
/// <returns>a handle to the new process</returns>
|
||||
IProcess NewProcess(IProcessWatcher watcher, string commandLine);
|
||||
IProcess NewProcess(IProcessWatcher watcher, string commandLine, string commandLineArguments);
|
||||
|
||||
/// <summary>
|
||||
/// request that a Process object, return from NewProcess, be scheduled according
|
||||
|
|
|
|||
|
|
@ -56,11 +56,6 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
/// </summary>
|
||||
private State state;
|
||||
|
||||
/// <summary>
|
||||
/// string used to start the remote process
|
||||
/// </summary>
|
||||
private string commandLine;
|
||||
|
||||
/// <summary>
|
||||
/// this is the handle that the scheduler supplies that is used to refer
|
||||
/// to the process
|
||||
|
|
@ -114,11 +109,12 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
/// <summary>
|
||||
/// construct a new object to represent the lifecycle of a process being scheduled
|
||||
/// </summary>
|
||||
public Process(ISchedulerProcess p, IProcessWatcher w, string cmd, ILogger l)
|
||||
public Process(ISchedulerProcess p, IProcessWatcher w, string cmd, string cmdLineArgs, ILogger l)
|
||||
{
|
||||
schedulerProcess = p;
|
||||
state = State.Initializing;
|
||||
commandLine = cmd;
|
||||
CommandLine = cmd;
|
||||
CommandLineArguments = cmdLineArgs;
|
||||
watcher = w;
|
||||
cancelTask = new TaskCompletionSource<bool>();
|
||||
directory = null;
|
||||
|
|
@ -137,7 +133,12 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
/// <summary>
|
||||
/// the string used to start the remote process
|
||||
/// </summary>
|
||||
public string CommandLine { get { return commandLine; } }
|
||||
public string CommandLine { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// arguments provided when starting the remote process
|
||||
/// </summary>
|
||||
public string CommandLineArguments { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// the local directory of the process at the daemon's host computer
|
||||
|
|
@ -199,8 +200,12 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
|
||||
ToMatched(computer, DateTime.Now.ToFileTimeUtc());
|
||||
|
||||
StringBuilder payload = new StringBuilder();
|
||||
payload.AppendLine(CommandLine);
|
||||
payload.AppendLine(CommandLineArguments);
|
||||
|
||||
Task<string> bail = interrupt.ContinueWith((t) => "");
|
||||
Task<string> upload = PostRequest(computer, directory + "?op=create", Encoding.UTF8.GetBytes(commandLine));
|
||||
Task<string> upload = PostRequest(computer, directory + "?op=create", Encoding.UTF8.GetBytes(payload.ToString()));
|
||||
Task<string> completed = await Task.WhenAny(bail, upload);
|
||||
|
||||
if (completed == bail)
|
||||
|
|
|
|||
|
|
@ -24,32 +24,7 @@ using System.Runtime.InteropServices;
|
|||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("DryadHttpClusterInterface")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("Microsoft Corporation")]
|
||||
[assembly: AssemblyProduct("DryadHttpClusterInterface")]
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.HttpClusterInterface")]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("9d262d5f-f6c5-4f9c-a1cc-7568cb399bf9")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("0.1.0.0")]
|
||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
||||
|
|
|
|||
|
|
@ -4,24 +4,40 @@
|
|||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
@ -1,13 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
|
||||
</packages>
|
||||
30
Dryad.sln
30
Dryad.sln
|
|
@ -1,6 +1,8 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 2012
|
||||
# Visual Studio 2013
|
||||
VisualStudioVersion = 12.0.30723.0
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DryadVertex", "DryadVertex", "{99F5E7FE-ADD4-4E9D-BA90-5D3D3409BEB3}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "VertexHost", "VertexHost", "{37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D}"
|
||||
|
|
@ -27,7 +29,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dprocess", "DryadVertex\Ver
|
|||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ManagedWrapperVertex", "DryadVertex\VertexHost\vertex\managedwrappervertex\ManagedWrapperVertex.vcxproj", "{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}"
|
||||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHost", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}"
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHostBody", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}"
|
||||
ProjectSection(ProjectDependencies) = postProject
|
||||
{AA529122-F51C-48D7-A8C1-C0B24F570885} = {AA529122-F51C-48D7-A8C1-C0B24F570885}
|
||||
{482E0741-E244-4974-97D4-3A7167581E91} = {482E0741-E244-4974-97D4-3A7167581E91}
|
||||
|
|
@ -78,7 +80,18 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalScheduler", "LocalSche
|
|||
{A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6} = {A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6}
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadLinqTests", "DryadLinqTests\DryadLinqTests.csproj", "{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}"
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VertexHostMain", "DryadVertex\VertexHost\vertex\VertexHostMain\VertexHostMain.csproj", "{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "NuSpec", "NuSpec", "{35121ABA-5FD2-4E1D-9C05-033D78D824F3}"
|
||||
ProjectSection(SolutionItems) = preProject
|
||||
Microsoft.Research.Dryad.nuspec = Microsoft.Research.Dryad.nuspec
|
||||
Microsoft.Research.Dryad.targets = Microsoft.Research.Dryad.targets
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Misc", "Misc", "{D361F025-2F7F-4C1A-8FBC-4AD826CC656D}"
|
||||
ProjectSection(SolutionItems) = preProject
|
||||
SharedAssemblyInfo.cs = SharedAssemblyInfo.cs
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
|
|
@ -146,10 +159,10 @@ Global
|
|||
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Debug|x64.Build.0 = Debug|x64
|
||||
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.ActiveCfg = Release|x64
|
||||
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.Build.0 = Release|x64
|
||||
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Debug|x64.Build.0 = Debug|x64
|
||||
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Release|x64.ActiveCfg = Release|x64
|
||||
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Release|x64.Build.0 = Release|x64
|
||||
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Debug|x64.Build.0 = Debug|x64
|
||||
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Release|x64.ActiveCfg = Release|x64
|
||||
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Release|x64.Build.0 = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
@ -162,11 +175,12 @@ Global
|
|||
{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
|
||||
{57663B94-E11B-431E-BE4B-E2C61112DEC5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
|
||||
{AA529122-F51C-48D7-A8C1-C0B24F570885} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
|
||||
{A0033286-9C5F-4113-BAA5-58A1274F95C5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
|
||||
{BDEDD3BB-C7E2-498F-A212-F99786C8E23C} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
{AB9EA66C-5811-49A7-B002-24203AEB9083} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
{3EE0920C-0607-4569-9EC3-5C12BB6EF244} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
{89C5654B-02E4-478D-A7E6-50D79F638B4F} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
{A0033286-9C5F-4113-BAA5-58A1274F95C5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
|
||||
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
|
|
@ -10,7 +13,7 @@
|
|||
<OutputType>Exe</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>Microsoft.Research.Dryad.GraphManager</RootNamespace>
|
||||
<AssemblyName>DryadLinqGraphManager</AssemblyName>
|
||||
<AssemblyName>Microsoft.Research.Dryad.GraphManager</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<FileUpgradeFlags>
|
||||
|
|
@ -33,6 +36,7 @@
|
|||
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
|
||||
<UseApplicationTrust>false</UseApplicationTrust>
|
||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||
<NuGetPackageImportStamp>5c14de70</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
|
|
@ -53,52 +57,28 @@
|
|||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core">
|
||||
|
|
@ -108,9 +88,12 @@
|
|||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data.Services.Client" />
|
||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
<HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq">
|
||||
|
|
@ -123,6 +106,9 @@
|
|||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\SharedAssemblyInfo.cs">
|
||||
<Link>Properties\SharedAssemblyInfo.cs</Link>
|
||||
</Compile>
|
||||
<Compile Include="DryadLinqApplication.cs" />
|
||||
<Compile Include="GraphBuilder.cs" />
|
||||
<Compile Include="LinqToDryadException.cs" />
|
||||
|
|
@ -168,10 +154,19 @@
|
|||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath);
|
||||
if (!SUCCEEDED(err))
|
||||
{
|
||||
string msg = String.Format("Could not read DSC input file {0}", info.sources[0]);
|
||||
string msg = String.Format("Could not read Partitioned file input {0}", info.sources[0]);
|
||||
throw new LinqToDryadException(msg, err);
|
||||
}
|
||||
|
||||
|
|
@ -127,7 +127,9 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
|
||||
DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");
|
||||
|
||||
err = input.Open(app.GetUniverse(), info.sources[0]);
|
||||
Uri srcUri = new Uri(info.sources[0]);
|
||||
|
||||
err = input.Open(app.GetUniverse(), srcUri.GetLeftPart(UriPartial.Path), info.recordType);
|
||||
if (!SUCCEEDED(err))
|
||||
{
|
||||
string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError());
|
||||
|
|
@ -182,10 +184,16 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
if ( info.ioType == VertexInfo.IOType.PARTITIONEDFILE )
|
||||
{
|
||||
DrPartitionOutputStream output = new DrPartitionOutputStream();
|
||||
int err = output.Open(new Uri(info.sources[0]).AbsolutePath, info.partitionUncPath);
|
||||
Uri sourceUri = new Uri(info.sources[0]);
|
||||
string sourcePath = sourceUri.AbsolutePath;
|
||||
if (!String.IsNullOrEmpty(sourceUri.Host))
|
||||
{
|
||||
sourcePath = @"\\" + sourceUri.Host + sourcePath;
|
||||
}
|
||||
int err = output.Open(sourcePath, info.partitionUncPath);
|
||||
if (!SUCCEEDED(err))
|
||||
{
|
||||
string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]);
|
||||
string msg = String.Format("Could not open output fileset {0}", sourcePath);
|
||||
throw new LinqToDryadException(msg, err);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -73,12 +73,12 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
|
||||
public static Uri AddDfsJobDirectoryFromArgs(string arg)
|
||||
{
|
||||
string jobId = Microsoft.Research.Peloponnese.Storage.AzureUtils.ApplicationIdFromEnvironment();
|
||||
string jobDirectoryBase = Microsoft.Research.Peloponnese.Storage.AzureUtils.CmdLineDecode(arg.Substring(arg.IndexOf('=') + 1));
|
||||
string jobId = Microsoft.Research.Peloponnese.Yarn.Utils.ApplicationIdFromEnvironment();
|
||||
string jobDirectoryBase = Microsoft.Research.Peloponnese.Utils.CmdLineDecode(arg.Substring(arg.IndexOf('=') + 1));
|
||||
string jobDirectorySpecific = jobDirectoryBase.Replace("_JOBID_", jobId);
|
||||
|
||||
UriBuilder logLocation = new UriBuilder(jobDirectorySpecific);
|
||||
logLocation.Path = logLocation.Path + "calypso.log";
|
||||
logLocation.Path = logLocation.Path.TrimEnd('/') + "/calypso.log";
|
||||
|
||||
DebugHelper.AddReporter(new DrCalypsoReporter(logLocation.Uri.AbsoluteUri));
|
||||
|
||||
|
|
@ -94,7 +94,7 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
{
|
||||
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
|
||||
{
|
||||
string dfsPath = dfsDirectory.AbsolutePath + Path.GetFileName(localPath);
|
||||
string dfsPath = dfsDirectory.AbsolutePath.TrimEnd('/') + "/" + Path.GetFileName(localPath);
|
||||
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
|
||||
hdfs.UploadAll(localPath, dfsPath);
|
||||
}
|
||||
|
|
@ -102,11 +102,12 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
else if (dfsDirectory.Scheme == "azureblob")
|
||||
{
|
||||
string account, key, container, blob;
|
||||
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
|
||||
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container);
|
||||
string dfsPath = blob + Path.GetFileName(localPath);
|
||||
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
|
||||
azure.PutDfsFile(localPath, dfsPath);
|
||||
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
|
||||
var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
|
||||
string dfsPath = blob.TrimEnd('/') + "/" + Path.GetFileName(localPath);
|
||||
Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
|
||||
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsUri.AbsoluteUri);
|
||||
azure.PutDfsFile(dfsUri, localPath);
|
||||
}
|
||||
else if (dfsDirectory.Scheme == "file")
|
||||
{
|
||||
|
|
@ -138,10 +139,11 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
else if (dfsDirectory.Scheme == "azureblob")
|
||||
{
|
||||
string account, key, container, blob;
|
||||
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
|
||||
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container);
|
||||
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
|
||||
var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
|
||||
string dfsPath = blob + dfsName;
|
||||
azure.PutDfsFile(payloadBytes, dfsPath);
|
||||
Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
|
||||
azure.PutDfsFile(dfsUri, payloadBytes);
|
||||
}
|
||||
else if (dfsDirectory.Scheme == "file")
|
||||
{
|
||||
|
|
@ -207,7 +209,39 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
logDir = logDir.Split(',').First().Trim();
|
||||
DrLogging.Initialize(Path.Combine(logDir, "graphmanager"), false);
|
||||
}
|
||||
|
||||
string logLevel = Environment.GetEnvironmentVariable("DRYAD_LOGGING_LEVEL");
|
||||
if (logLevel == null)
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Debug);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (logLevel == "OFF")
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Off);
|
||||
}
|
||||
else if (logLevel == "CRITICAL")
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Assert);
|
||||
}
|
||||
else if (logLevel == "ERROR")
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Error);
|
||||
}
|
||||
else if (logLevel == "WARN")
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Warning);
|
||||
}
|
||||
else if (logLevel == "INFO")
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Info);
|
||||
}
|
||||
else
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Debug);
|
||||
}
|
||||
}
|
||||
|
||||
// Report start time to Artemis - must come after
|
||||
// DrLogging is initialized so stdout is redirected
|
||||
|
|
|
|||
|
|
@ -28,6 +28,13 @@ using Microsoft.Research.Dryad;
|
|||
|
||||
namespace Microsoft.Research.Dryad.GraphManager
|
||||
{
|
||||
internal class ConfigDependency : Microsoft.Research.Peloponnese.Shared.AssemblyDependencyAttribute
|
||||
{
|
||||
public ConfigDependency() : base("Microsoft.Research.Dryad.GraphManager.exe.config", false)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
class Program
|
||||
{
|
||||
static Uri dfsDirectory;
|
||||
|
|
|
|||
|
|
@ -22,34 +22,12 @@ using System.Reflection;
|
|||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
[assembly: Microsoft.Research.Dryad.GraphManager.ConfigDependency()]
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("DryadLinqGraphManager")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
|
||||
[assembly: AssemblyProduct("DryadLinqGraphManager")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.GraphManager")]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("55c7f4b4-02ab-4309-b276-3c9ab5a927e0")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("0.1.0.0")]
|
||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
||||
|
|
|
|||
|
|
@ -237,6 +237,12 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
vertex.info = new VertexInfo();
|
||||
vertex.info.ioType = GetIoType(ioType);
|
||||
|
||||
XmlNode recordType = storageSet.SelectSingleNode("RecordType");
|
||||
if (recordType != null)
|
||||
{
|
||||
vertex.info.recordType = recordType.InnerXml;
|
||||
}
|
||||
|
||||
XmlNodeList storageUris = storageSet.SelectNodes("SourceURI");
|
||||
vertex.info.sources = new string[storageUris.Count];
|
||||
for (int indexStorageUri=0; indexStorageUri<storageUris.Count; indexStorageUri++)
|
||||
|
|
|
|||
|
|
@ -7,24 +7,40 @@
|
|||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.2" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -5,6 +5,14 @@
|
|||
</startup>
|
||||
<runtime>
|
||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
|
|
@ -18,12 +26,20 @@
|
|||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,469 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class ApplyAndForkTests
|
||||
{
|
||||
public static void Run(DryadLinqContext context, string matchPattern)
|
||||
{
|
||||
TestLog.Message(" **********************");
|
||||
TestLog.Message(" ApplyAndForkTests ");
|
||||
TestLog.Message(" **********************");
|
||||
|
||||
var tests = new Dictionary<string, Action>()
|
||||
{
|
||||
{"NonHomomorphicUnaryApply", () => NonHomomorphicUnaryApply(context) },
|
||||
{"HomomorphicUnaryApply", () => HomomorphicUnaryApply(context) },
|
||||
{"NonHomomorphicBinaryApply", () => NonHomomorphicBinaryApply(context) },
|
||||
{"LeftHomomorphicBinaryApply", () => LeftHomomorphicBinaryApply(context) },
|
||||
{"FullHomomorphicBinaryApply_DifferentDataSets", () => FullHomomorphicBinaryApply_DifferentDataSets(context) },
|
||||
{"FullHomomorphicBinaryApply_IdenticalDataSets", () => FullHomomorphicBinaryApply_IdenticalDataSets(context) },
|
||||
{"Aggregate_WithCombiner", () => Aggregate_WithCombiner(context) },
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
{
|
||||
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
|
||||
{
|
||||
test.Value.Invoke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static IEnumerable<int> NonHomomorphic_Unary_Func(IEnumerable<int> input)
|
||||
{
|
||||
return input;
|
||||
}
|
||||
|
||||
// [DistributiveOverConcat]
|
||||
public static IEnumerable<int> Homomorphic_Unary_Func(IEnumerable<int> input)
|
||||
{
|
||||
return input;
|
||||
}
|
||||
|
||||
public static IEnumerable<int> NonHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
|
||||
{
|
||||
return left;
|
||||
}
|
||||
|
||||
// [LeftDistributiveOverConcat]
|
||||
public static IEnumerable<int> LeftHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
|
||||
{
|
||||
return left;
|
||||
}
|
||||
|
||||
// Note: an apply function must only consume each enumerable once, and it must produce an enumerable
|
||||
// So for a simple pass-through function that does a little work, we must enumerate only once.
|
||||
// Else we get the error: "An HpcLinq channel can't be read more than once."
|
||||
// [DistributiveOverConcat]
|
||||
public static IEnumerable<int> FullHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
|
||||
{
|
||||
long cLeft = 0;
|
||||
foreach (int x in left)
|
||||
{
|
||||
cLeft++;
|
||||
yield return x;
|
||||
}
|
||||
|
||||
long cRight = 0;
|
||||
foreach (int x in right)
|
||||
{
|
||||
cRight++;
|
||||
yield return x;
|
||||
}
|
||||
|
||||
if (cLeft == 0)
|
||||
throw new Exception("a node received empty left-data");
|
||||
|
||||
if (cRight == 0)
|
||||
throw new Exception("a node received empty right-data");
|
||||
}
|
||||
|
||||
public static bool NonHomomorphicUnaryApply(DryadLinqContext context)
|
||||
{
|
||||
string testName = "NonHomomorphicUnaryApply";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/NonHomomorphicUnaryApply";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool HomomorphicUnaryApply(DryadLinqContext context)
|
||||
{
|
||||
string testName = "HomomorphicUnaryApply";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/HomomorphicUnaryApply";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool NonHomomorphicBinaryApply(DryadLinqContext context)
|
||||
{
|
||||
string testName = "NonHomomorphicBinaryApply";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/NonHomomorphicBinaryApply";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool LeftHomomorphicBinaryApply(DryadLinqContext context)
|
||||
{
|
||||
string testName = "LeftHomomorphicBinaryApply";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/LeftHomomorphicBinaryApply";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool FullHomomorphicBinaryApply_DifferentDataSets(DryadLinqContext context)
|
||||
{
|
||||
string testName = "FullHomomorphicBinaryApply_DifferentDataSets";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/FullHomomorphicBinaryApply_DifferentDataSets";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool FullHomomorphicBinaryApply_IdenticalDataSets(DryadLinqContext context)
|
||||
{
|
||||
string testName = "FullHomomorphicBinaryApply_IdenticalDataSets";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/FullHomomorphicBinaryApply_2";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
[Associative(typeof(AssociativeRecursive1))]
|
||||
public static string IntToStringCSVAggregator(string agg, int next)
|
||||
{
|
||||
return agg + "," + next.ToString();
|
||||
}
|
||||
public class AssociativeRecursive1 : IAssociative<string>
|
||||
{
|
||||
public string Seed()
|
||||
{
|
||||
return "";
|
||||
}
|
||||
public string RecursiveAccumulate(string first, string second)
|
||||
{
|
||||
return first + second;
|
||||
}
|
||||
}
|
||||
|
||||
public static bool Aggregate_WithCombiner(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Aggregate_WithCombiner";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
|
||||
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,53 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<configuration>
|
||||
<startup>
|
||||
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
|
||||
</startup>
|
||||
<runtime>
|
||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="FSharp.Core" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
|
||||
<bindingRedirect oldVersion="4.0.0.0" newVersion="4.3.0.0" />
|
||||
<bindingRedirect oldVersion="2.3.5.0" newVersion="4.3.0.0" />
|
||||
<bindingRedirect oldVersion="2.0.0.0" newVersion="4.3.0.0" />
|
||||
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,219 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<ProjectGuid>d0153361-1b5d-4715-8ee3-aa824f89cbc3</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<RootNamespace>DryadLinqFSharpTests</RootNamespace>
|
||||
<AssemblyName>DryadLinqFSharpTests</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<Name>DryadLinqFSharpTests</Name>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<Tailcalls>false</Tailcalls>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<WarningLevel>3</WarningLevel>
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DocumentationFile>bin\Debug\DryadLinqFSharpTests.XML</DocumentationFile>
|
||||
<Prefer32Bit>true</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<Tailcalls>true</Tailcalls>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<WarningLevel>3</WarningLevel>
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DocumentationFile>bin\Release\DryadLinqFSharpTests.XML</DocumentationFile>
|
||||
<Prefer32Bit>true</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm">
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData">
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client">
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl">
|
||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl.Drawing">
|
||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl.GraphViewerGdi">
|
||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.DryadLinq">
|
||||
<HintPath>..\packages\Microsoft.Research.Dryad.0.1.4-beta001\lib\net45\Microsoft.Research.DryadLinq.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge">
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils">
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks">
|
||||
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks.Extensions">
|
||||
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
|
||||
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Common">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage">
|
||||
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="mscorlib" />
|
||||
<Reference Include="FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json">
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
<Reference Include="System.Data.Linq" />
|
||||
<Reference Include="System.Drawing" />
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.Extensions">
|
||||
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="System.Net.Http.Primitives">
|
||||
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Numerics" />
|
||||
<Reference Include="System.Runtime.Serialization" />
|
||||
<Reference Include="System.ServiceModel" />
|
||||
<Reference Include="System.ServiceModel.Discovery" />
|
||||
<Reference Include="System.Spatial">
|
||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="App.config" />
|
||||
<Compile Include="Program.fs" />
|
||||
<None Include="packages.config" />
|
||||
</ItemGroup>
|
||||
<PropertyGroup>
|
||||
<MinimumVisualStudioVersion Condition="'$(MinimumVisualStudioVersion)' == ''">11</MinimumVisualStudioVersion>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<Tailcalls>false</Tailcalls>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<WarningLevel>3</WarningLevel>
|
||||
<DocumentationFile>bin\Debug\DryadLinqFSharpTests.XML</DocumentationFile>
|
||||
<Prefer32Bit>true</Prefer32Bit>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<Tailcalls>true</Tailcalls>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<WarningLevel>3</WarningLevel>
|
||||
<DocumentationFile>bin\Release\DryadLinqFSharpTests.XML</DocumentationFile>
|
||||
<Prefer32Bit>true</Prefer32Bit>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets" Condition=" Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')" />
|
||||
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
|
||||
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
|
||||
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
|
||||
</Target>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets" Condition="Exists('..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
// Learn more about F# at http://fsharp.net
|
||||
// See the 'F# Tutorial' project for more help.
|
||||
|
||||
open System
|
||||
open System.Collections.Generic
|
||||
open System.Linq
|
||||
open System.IO
|
||||
open Microsoft.Research.DryadLinq
|
||||
|
||||
let context = new DryadLinqContext(1)
|
||||
let inputUri = @"partfile:///d:/DryadLinqTemp/PartFiles/foo.pt"
|
||||
let input = context.FromStore<LineRecord>(inputUri)
|
||||
|
||||
type Pair = { word : string; count : int }
|
||||
|
||||
let Histogram (source : IQueryable<LineRecord>) (k : int) =
|
||||
let words = source.SelectMany(fun (x : LineRecord) -> x.Line.Split(' ') |> Seq.ofArray)
|
||||
let groups = words.GroupBy(fun x -> x)
|
||||
let counts = groups.Select(fun (x : IGrouping<string, string>) -> { word = x.Key; count = x.Count() })
|
||||
let ordered = counts.OrderByDescending(fun x -> x.count)
|
||||
ordered.Take(k)
|
||||
|
||||
let Histogram1 (source : IQueryable<LineRecord>) (k : int) =
|
||||
query {
|
||||
for ws in source do
|
||||
for w in ws.Line.Split(' ') do
|
||||
groupBy w into g
|
||||
select { word = g.Key; count = g.Count() } into winfo
|
||||
sortByDescending winfo.count
|
||||
take k
|
||||
}
|
||||
|
||||
let Histogram2 =
|
||||
query {
|
||||
for ws in input do
|
||||
for w in ws.Line.Split(' ') do
|
||||
groupBy w into g
|
||||
select { word = g.Key; count = g.Count() } into winfo
|
||||
sortByDescending winfo.count
|
||||
take 10
|
||||
}
|
||||
|
||||
for x in Histogram1 input 10 do System.Console.WriteLine(x.word + " : " + x.count.ToString())
|
||||
|
||||
//let foo = Histogram input 10
|
||||
//foo.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/xxx.pt").SubmitAndWait()
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
|
||||
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Dryad" version="0.1.4-beta001" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}</ProjectGuid>
|
||||
<ProjectGuid>{4EA15885-003F-4B4D-A7DB-C172C2F23161}</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>DryadLinqTests</RootNamespace>
|
||||
|
|
@ -30,6 +30,25 @@
|
|||
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
|
||||
<RestorePackages>true</RestorePackages>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||
|
|
@ -53,50 +72,110 @@
|
|||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl">
|
||||
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl.Drawing">
|
||||
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl.GraphViewerGdi">
|
||||
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.DryadLinq, Version=0.1.3.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Research.Dryad.0.1.4-beta001\lib\net45\Microsoft.Research.DryadLinq.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge">
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils">
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json">
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Data.Services.Client" />
|
||||
<Reference Include="System.Data.Linq" />
|
||||
<Reference Include="System.Drawing" />
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Runtime.Serialization" />
|
||||
<Reference Include="System.ServiceModel" />
|
||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
<HintPath>packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
|
|
@ -106,8 +185,20 @@
|
|||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="ApplyAndForkTests.cs" />
|
||||
<Compile Include="BasicAPITests.cs" />
|
||||
<Compile Include="GroupByReduceTests.cs" />
|
||||
<Compile Include="MiscBugFixTests.cs" />
|
||||
<Compile Include="Program.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
<Compile Include="RangePartitionAPICoverageTests.cs" />
|
||||
<Compile Include="SerializationTests.cs" />
|
||||
<Compile Include="SerializationTestTypes.cs" />
|
||||
<Compile Include="SimpleTests.cs" />
|
||||
<Compile Include="TestLog.cs" />
|
||||
<Compile Include="TypesInQueryTests.cs" />
|
||||
<Compile Include="Utils.cs" />
|
||||
<Compile Include="WordCount.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="App.config" />
|
||||
|
|
@ -137,14 +228,21 @@
|
|||
</BootstrapperPackage>
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<Import Project="packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
|
||||
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
|
||||
<Error Condition="!Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
|
||||
<Error Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
|
||||
</Target>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets" Condition="Exists('packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 2012
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadLinqTests", "DryadLinqTests.csproj", "{4EA15885-003F-4B4D-A7DB-C172C2F23161}"
|
||||
EndProject
|
||||
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "DryadLinqFSharpTests", "DryadLinqFSharpTests\DryadLinqFSharpTests.fsproj", "{D0153361-1B5D-4715-8EE3-AA824F89CBC3}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Debug|Mixed Platforms = Debug|Mixed Platforms
|
||||
Debug|x64 = Debug|x64
|
||||
Release|Any CPU = Release|Any CPU
|
||||
Release|Mixed Platforms = Release|Mixed Platforms
|
||||
Release|x64 = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|x64.Build.0 = Debug|x64
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Mixed Platforms.ActiveCfg = Release|x64
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Mixed Platforms.Build.0 = Release|x64
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|x64.ActiveCfg = Release|x64
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|x64.Build.0 = Release|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|x64.Build.0 = Debug|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Mixed Platforms.ActiveCfg = Release|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Mixed Platforms.Build.0 = Release|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|x64.ActiveCfg = Release|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|x64.Build.0 = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,967 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public static class MiscBugFixTests
|
||||
{
|
||||
public static void Run(DryadLinqContext context, string matchPattern)
|
||||
{
|
||||
TestLog.Message(" **********************");
|
||||
TestLog.Message(" MiscBugFixTests ");
|
||||
TestLog.Message(" **********************");
|
||||
|
||||
var tests = new Dictionary<string, Action>()
|
||||
{
|
||||
{"Bug11447_GroupByWithComparer", () => Bug11447_GroupByWithComparer(context) },
|
||||
{"Bug12584_HashPartitionOutputCount", () => Bug12584_HashPartitionOutputCount(context) },
|
||||
{"Bug13108_DisableSequenceEquals", () => Bug13108_DisableSequenceEquals(context) },
|
||||
{"Bug13529_and_Bug13593_IndexedOperatorCompilation", () => Bug13529_and_Bug13593_IndexedOperatorCompilation(context) },
|
||||
{"Bug13130_ReverseOperator", () => Bug13130_ReverseOperator(context) },
|
||||
{"Bug13736_IndexedTakeWhile", () => Bug13736_IndexedTakeWhile(context) },
|
||||
{"Bug13534_HashPartitionNegIndexIsError", () => Bug13534_HashPartitionNegIndexIsError(context) },
|
||||
{"Bug13474_and_Bug13483_FromDscOnBadFileSet", () => Bug13474_and_Bug13483_FromDscOnBadFileSet(context) },
|
||||
{"Bug13637_EmptyFilesInFilesets", () => Bug13637_EmptyFilesInFilesets(context) },
|
||||
{"Bug13637_LocalDebugProducingZeroRecords", () => Bug13637_LocalDebugProducingZeroRecords(context) },
|
||||
{"Bug13245_FromDsc_Submit_throws", () => Bug13245_FromDsc_Submit_throws(context) },
|
||||
{"Bug13245_QueryUsingNonHpcLinqOperator", () => Bug13245_QueryUsingNonHpcLinqOperator(context) },
|
||||
{"Bug13302_ConfigEnvironmentCleanup", () => Bug13302_ConfigEnvironmentCleanup(context) },
|
||||
{"Bug13970_MismatchedDataTypes", () => Bug13970_MismatchedDataTypes(context) },
|
||||
{"Bug14010_AlreadyDisposedContext", () => Bug14010_AlreadyDisposedContext(context) },
|
||||
{"Bug14256_LeaseOnTempDscFileset", () => Bug14256_LeaseOnTempDscFileset(context) },
|
||||
{"Bug14189_OrderPreservation", () => Bug14189_OrderPreservation(context) },
|
||||
{"Bug14190_MergeJoin_DecreasingOrder", () => Bug14190_MergeJoin_DecreasingOrder(context) },
|
||||
{"Bug14192_MultiApplySubExpressionReuse", () => Bug14192_MultiApplySubExpressionReuse(context) },
|
||||
{"Bug14870_LongIndexTakeWhile", () => Bug14870_LongIndexTakeWhile(context) },
|
||||
{"Bug15159_NotOperatorForNullableBool", () => Bug15159_NotOperatorForNullableBool(context) },
|
||||
{"Bug15371_NoDataMembersForSerialization", () => Bug15371_NoDataMembersForSerialization(context) },
|
||||
{"Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes", () => Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(context) },
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
{
|
||||
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
|
||||
{
|
||||
test.Value.Invoke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static bool Bug11447_GroupByWithComparer(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug11447_GroupByWithComparer";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
|
||||
result[0] = a;
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
|
||||
result[1] = a;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug12584_HashPartitionOutputCount(DryadLinqContext context) // ToDo
|
||||
{
|
||||
string testName = "Bug12584_HashPartitionOutputCount";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/Bug12584_HashPartitionOutputCount";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var info = pt1.HashPartition(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
info.Wait();
|
||||
|
||||
//// partition verification are only valid for cluster execution..
|
||||
|
||||
//// check that nOutputPartitions == nInputPartitions.
|
||||
//// Note: this is today's behavior, but we don't strictly guarantee this and the rules may change.
|
||||
//var inFS = context.DscService.GetFileSet(DataGenerators.SIMPLE_FILESET_NAME);
|
||||
//var inPCount = inFS.GetFiles().Count();
|
||||
//var outFS = context.DscService.GetFileSet("DevUnitTest/Bug12584_out");
|
||||
//var outPCount = outFS.GetFiles().Count();
|
||||
//passed &= TestUtils.Assert(outPCount == inPCount, "Output nPartitions should be equal to input nPartitions.");
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13108_DisableSequenceEquals(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13108_DisableSequenceEquals";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
bool result = pt1.Select(x => x).SequenceEqual(pt2.Select(x => x));
|
||||
passed &= false; // NotSupportedException should have been thrown
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
}
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13529_and_Bug13593_IndexedOperatorCompilation(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13529_and_Bug13593_IndexedOperatorCompilation";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
result[0] = pt1.Select((x, i) => x)
|
||||
.LongSelect((x, i) => x)
|
||||
.LongSelectMany((x, i) => new[] { x })
|
||||
.SelectMany((x, i) => new[] { x })
|
||||
.Where((x, i) => true)
|
||||
.LongWhere((x, i) => true)
|
||||
.TakeWhile((x, i) => true)
|
||||
.LongTakeWhile((x, i) => true)
|
||||
.SkipWhile((x, i) => false)
|
||||
.LongSkipWhile((x, i) => false)
|
||||
.ToArray();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
result[1] = pt1.Select((x, i) => x)
|
||||
.LongSelect((x, i) => x)
|
||||
.LongSelectMany((x, i) => new[] { x })
|
||||
.SelectMany((x, i) => new[] { x })
|
||||
.Where((x, i) => true)
|
||||
.LongWhere((x, i) => true)
|
||||
.TakeWhile((x, i) => true)
|
||||
.LongTakeWhile((x, i) => true)
|
||||
.SkipWhile((x, i) => false)
|
||||
.LongSkipWhile((x, i) => false)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13130_ReverseOperator(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13130_ReverseOperator";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
//data set #1
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
result[0] = pt1.Reverse().ToArray();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
result[1] = pt1.Reverse().ToArray();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
//data set #2 ToDo
|
||||
//data set #3 ToDo
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13736_IndexedTakeWhile(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13736_IndexedTakeWhile";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
// dataset #1
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var pt2 = pt1.TakeWhile((x, i) => i < 6).ToArray();
|
||||
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 6).ToArray();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var pt2 = pt1.TakeWhile((x, i) => i < 6).ToArray();
|
||||
result[1] = pt1.AsEnumerable().TakeWhile((x, i) => i < 6).ToArray();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
// dataset #2
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var pt2 = pt1.TakeWhile((x, i) => i < 125).ToArray();
|
||||
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 125).ToArray();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetGroupByReduceDataSet(context);
|
||||
var pt2 = pt1.TakeWhile((x, i) => i < 125).ToArray();
|
||||
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 125).ToArray();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13534_HashPartitionNegIndexIsError(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13534_HashPartitionNegIndexIsError";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
pt1.HashPartition(x => x, 0); // exception should be thrown
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
pt1.HashPartition(x => x, -1); // exception should be thrown
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13474_and_Bug13483_FromDscOnBadFileSet(DryadLinqContext context)
|
||||
{
|
||||
// ToDo
|
||||
return false;
|
||||
}
|
||||
public static bool Bug13637_EmptyFilesInFilesets(DryadLinqContext context)
|
||||
{
|
||||
// ToDo
|
||||
return false;
|
||||
}
|
||||
|
||||
public static bool Bug13637_LocalDebugProducingZeroRecords(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13637_LocalDebugProducingZeroRecords";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/Bug13637_LocalDebugProducingZeroRecords";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> pt2 = pt1.Where(x => false).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var queryInfo2 = pt2.Submit();
|
||||
queryInfo2.Wait();
|
||||
int[] data2 = pt2.ToArray();
|
||||
passed &= (data2.Length == 0);
|
||||
|
||||
// query producing no records -> anonymous output
|
||||
IQueryable<int> pt3 = pt1.Where(x => false);
|
||||
var queryInfo3 = pt3.Submit();
|
||||
queryInfo3.Wait();
|
||||
|
||||
int[] data3 = pt3.ToArray();
|
||||
passed &= (data3.Length == 0);
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13245_FromDsc_Submit_throws(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13245_FromDsc_Submit_throws";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var info = pt1.Submit();
|
||||
passed &= false; // should throw ArgEx as the input isn't well formed
|
||||
}
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static IQueryable<int> Blah(this IQueryable<int> source)
|
||||
{
|
||||
return Enumerable.Range(1, 1).AsQueryable();
|
||||
}
|
||||
public static bool Bug13245_QueryUsingNonHpcLinqOperator(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13245_QueryUsingNonHpcLinqOperator";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var info = pt1.Blah().Submit();
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static IEnumerable<int> CheckEnvVar(IEnumerable<int> en)
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable("DummyEnvVar") == null)
|
||||
throw new Exception("the expected environment variable is not defined");
|
||||
return en;
|
||||
}
|
||||
public static bool Bug13302_ConfigEnvironmentCleanup(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13302_ConfigEnvironmentCleanup";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var info = pt1.Apply((en) => CheckEnvVar(en)).Submit();
|
||||
info.Wait();
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13970_MismatchedDataTypes(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13970_MismatchedDataTypes";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetGroupByReduceDataSet(context);
|
||||
double? output = pt1.AverageAsQuery().Single();
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= true; // expected
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14010_AlreadyDisposedContext(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug14010_AlreadyDisposedContext";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
context.LocalDebug = false;
|
||||
try
|
||||
{
|
||||
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
|
||||
ctx.Dispose();
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
|
||||
int output = pt1.Select(x => x).First();
|
||||
passed &= false;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
|
||||
ctx.Dispose();
|
||||
int output = pt1.Select(x => x).First();
|
||||
passed &= false;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
|
||||
ctx.Dispose();
|
||||
IQueryable<int> query = pt1.Select(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, "abc"), true);
|
||||
var info = DryadLinqQueryable.Submit(query);
|
||||
passed &= false;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= true;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14256_LeaseOnTempDscFileset(DryadLinqContext context) // TODO
|
||||
{
|
||||
string testName = "Bug14256_LeaseOnTempDscFileset";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14189_OrderPreservation(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug14189_OrderPreservation";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IGrouping<int, int>[] clusterSorted, localSorted;
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
var output = pt1.OrderBy(x => x % 4)
|
||||
.Select(x => x).Select(x => x).Where(x => true) // this pipeline was not preserving order correctly.
|
||||
.GroupBy(x => x % 4)
|
||||
.ToArray();
|
||||
|
||||
passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
|
||||
// sort back on the key for deterministic output.
|
||||
clusterSorted = output.OrderBy(x => x.Key).ToArray();
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
var output = pt1.OrderBy(x => x % 4)
|
||||
.Select(x => x).Select(x => x).Where(x => true) // this pipeline was not preserving order correctly.
|
||||
.GroupBy(x => x % 4)
|
||||
.ToArray();
|
||||
|
||||
passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
|
||||
// sort back on the key for deterministic output.
|
||||
localSorted = output.OrderBy(x => x.Key).ToArray();
|
||||
}
|
||||
|
||||
// check that each group of output has the same elements as the LINQ groups.
|
||||
for (int i = 0; i < 4; i++)
|
||||
{
|
||||
var a = clusterSorted[i].OrderBy(x => x);
|
||||
var b = localSorted[i].OrderBy(x => x);
|
||||
|
||||
passed &= a.SequenceEqual(b); //the output should match linq. Error for group: + i);
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14190_MergeJoin_DecreasingOrder(DryadLinqContext context) // TODO
|
||||
{
|
||||
string testName = "Bug14190_MergeJoin_DecreasingOrder";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/Bug14190_MergeJoin_DecreasingOrder";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14192_MultiApplySubExpressionReuse(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug14192_MultiApplySubExpressionReuse";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/Bug14192_MultiApplySubExpressionReuse";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var data = pt1.Select(x => x);
|
||||
|
||||
var info = pt1.Apply(new[] { pt1, pt1 }, (sources) => new int[] { 1, 2, 3 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
info.Wait();
|
||||
|
||||
// ToDo
|
||||
//string queryPlan = TestUtils.GetRecentQueryXmlAsText();
|
||||
//int nVerticesInPlan = TestUtils.GetVertexStageCount(queryPlan);
|
||||
|
||||
//passed &= (nVerticesInPlan == 7); // "Only seven vertices should appear (before bug, there were 10 of which the last three were extraneous.");
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14870_LongIndexTakeWhile(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug14870_LongIndexTakeWhile";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
|
||||
// ToDo - move to data generator
|
||||
int[][] data = new[] {
|
||||
new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { }, new int[] { } };
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.LongTakeWhile((x, i) => i < 8).ToArray();
|
||||
passed &= (output.Length == 8); // "Only eight items should be returned."
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug15159_NotOperatorForNullableBool(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug15159_NotOperatorForNullableBool";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<bool?>[] result = new IEnumerable<bool?>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
|
||||
result[0] = output.ToArray();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
|
||||
result[1] = output.ToArray();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public class NoDataMembersClass
|
||||
{
|
||||
|
||||
}
|
||||
public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug15371_NoDataMembersForSerialization";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
|
||||
var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
var result = context.FromStore<NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (DryadLinqException Ex)
|
||||
{
|
||||
passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
|
||||
Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
// use an anonymous type with nullable fields, and make sure we at least have some null values in the stream
|
||||
var output = pt1.Select(x => new { FirstField = (x % 2 == 0) ? new int?(x) : default(int?), SecondField = x.ToString() })
|
||||
.GroupBy(x => x.FirstField, y => y.SecondField); // use of GB ensures we exercise the emitted GetHashCode() overload
|
||||
passed &= (output.Count() != 0); // "Query return 0 length output"
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -17,13 +17,15 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
|||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using System.Reflection;
|
||||
using Microsoft.Research.DryadLinq;
|
||||
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
|
||||
namespace DryadLinqTests
|
||||
|
|
@ -32,64 +34,38 @@ namespace DryadLinqTests
|
|||
{
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
//Test1(args);
|
||||
//Test2(args);
|
||||
//Test3(args);
|
||||
//Test4(args);
|
||||
//Test5(args);
|
||||
SimpleTests.Run(args);
|
||||
|
||||
/*
|
||||
*
|
||||
Config conf = new Config("dryad-temp", "main1", @"d:\temp\TestLog\");
|
||||
DryadLinqContext context = new DryadLinqContext(Config.cluster);
|
||||
|
||||
string matchPattern = @"";
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "BasicAPITests.txt");
|
||||
BasicAPITests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "ApplyAndForkTests.txt");
|
||||
ApplyAndForkTests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "GroupByReduceTests.txt");
|
||||
GroupByReduceTests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "RangePartitionAPICoverageTests.txt");
|
||||
RangePartitionAPICoverageTests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "TypesInQueryTests.txt");
|
||||
TypesInQueryTests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "SerializationTests.txt");
|
||||
SerializationTests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "MiscBugFixTests.txt");
|
||||
MiscBugFixTests.Run(context, matchPattern);
|
||||
*/
|
||||
|
||||
}
|
||||
|
||||
public static void Test1(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile");
|
||||
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var lines = input.Where(x => x.Line.Contains("white"));
|
||||
var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
|
||||
result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test2(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile:///d:/DryadLinqTemp/PartFiles");
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var q1 = input.Where(x => x.Line.Contains("white"));
|
||||
var q2 = input.Where(x => x.Line.Contains("the"));
|
||||
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
|
||||
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
|
||||
DryadLinqQueryable.SubmitAndWait(res1, res2);
|
||||
}
|
||||
|
||||
public static void Test3(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile:///d:/DryadLinqTemp/PartFiles");
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var words = input.SelectMany(x => x.Line.Split(' '));
|
||||
var groups = words.GroupBy(x => x);
|
||||
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
|
||||
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
|
||||
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
|
||||
result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test4(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext("svc-d2-01");
|
||||
var input = context.FromStore<LineRecord>("hdfs://svc-d2-01:8033/user/misard/foo.txt");
|
||||
var lines = input.Where(x => x.Line.Contains("white"));
|
||||
var result = lines.ToStore("hdfs://svc-d2-01:8033/user/yuanbyu/foo.txt", true);
|
||||
result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test5(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1);
|
||||
Uri dataUri = AzureUtils.ToAzureUri("msrsvc", "I4JPlk0bZ6YWypg+RJamyq0us1b+kCcuoeKlPhfiHTcVW7P4xvuzURvlRShSo1O3UDhcL2LiY4kMaarD+p1lKg==", "test", "testwrite");
|
||||
IEnumerable<LineRecord> lines = DataProvider.ReadData<LineRecord>(context, dataUri);
|
||||
foreach (var x in lines)
|
||||
{
|
||||
Console.WriteLine(x);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ using System.Runtime.InteropServices;
|
|||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("")]
|
||||
[assembly: AssemblyProduct("DryadLinqTests")]
|
||||
[assembly: AssemblyProduct("Dryad")]
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
|
@ -39,7 +39,7 @@ using System.Runtime.InteropServices;
|
|||
[assembly: ComVisible(false)]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("1fce146c-8597-4cc2-92f6-ebb655cd99ed")]
|
||||
[assembly: Guid("49b725be-1187-4056-9890-7a0cb4c064a4")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
|
|
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
|
|||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("0.1.0.0")]
|
||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
||||
[assembly: AssemblyVersion("0.1.5.0")]
|
||||
[assembly: AssemblyFileVersion("0.1.5.0")]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,842 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.DryadLinq.Internal;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class RangePartitionAPICoverageTests
|
||||
{
|
||||
public static void Run(DryadLinqContext context, string matchPattern)
|
||||
{
|
||||
TestLog.Message(" **********************");
|
||||
TestLog.Message(" RangePartitionAPICoverageTests ");
|
||||
TestLog.Message(" **********************");
|
||||
|
||||
var tests = new Dictionary<string, Action>()
|
||||
{
|
||||
{"RP_keySelector", () => RP_keySelector(context) },
|
||||
{"RP_keySelector_pcount", () => RP_keySelector_pcount(context) },
|
||||
{"RP_keySelector_isDescending", () => RP_keySelector_isDescending(context) },
|
||||
{"RP_keySelector_rangeKeys", () => RP_keySelector_rangeKeys(context) },
|
||||
{"RP_keySelector_isDescending_pcount", () => RP_keySelector_isDescending_pcount(context) },
|
||||
{"RP_keySelector_keyComparer_isDescending", () => RP_keySelector_keyComparer_isDescending(context) },
|
||||
{"RP_keySelector_rangeKeys_keyComparer", () => RP_keySelector_rangeKeys_keyComparer(context) },
|
||||
{"RP_keySelector_keyComparer_isDescending_pcount", () => RP_keySelector_keyComparer_isDescending_pcount(context) },
|
||||
{"RP_keySelector_rangeKeys_keyComparer_isDescending", () => RP_keySelector_rangeKeys_keyComparer_isDescending(context) },
|
||||
{"RP_singlePartition_autoSeparators", () => RP_singlePartition_autoSeparators(context) },
|
||||
{"RP_rangeSeparators_customComparer", () => RP_rangeSeparators_customComparer(context) },
|
||||
{"RP_rangeSeparators_nullCustomComparer", () => RP_rangeSeparators_nullCustomComparer(context) },
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
{
|
||||
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
|
||||
{
|
||||
test.Value.Invoke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static IEnumerable<int> CountPartitions<T>(IEnumerable<T> pt)
|
||||
{
|
||||
yield return 1;
|
||||
}
|
||||
|
||||
public static IEnumerable<T> MaxInPartition<T>(IEnumerable<T> pt)
|
||||
{
|
||||
yield return pt.Max();
|
||||
}
|
||||
|
||||
public static IEnumerable<T> MinInPartition<T>(IEnumerable<T> pt)
|
||||
{
|
||||
yield return pt.Min();
|
||||
}
|
||||
|
||||
public static IEnumerable<T2> MaxCompare<T1, T2>(IEnumerable<T1> pt, int i, IEnumerable<T2> maxes)
|
||||
{
|
||||
if (0 == i)
|
||||
{
|
||||
yield return (T2)Convert.ChangeType(1, typeof(int));
|
||||
}
|
||||
|
||||
int priorMax = (int)Convert.ChangeType(maxes.ToArray()[i - 1], typeof(T2));
|
||||
foreach (T1 item in pt)
|
||||
{
|
||||
int val = (int)Convert.ChangeType(item, typeof(T1));
|
||||
if (val <= priorMax)
|
||||
{
|
||||
yield return (T2)Convert.ChangeType(1, typeof(int));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static bool RP_keySelector(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 3, false);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_pcount(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_pcount";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_pcount";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, 2).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 2, false);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, 2);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_isDescending(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_isDescending";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_isDescending";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 3, true);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, true);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_rangeKeys(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_rangeKeys";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
IEnumerable<int>[] result2 = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_rangeKeys_1";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
//increasing ranges keys.
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 4, false);
|
||||
result[0] = pt2.ToList();
|
||||
|
||||
|
||||
string outFile2 = "unittest/output/RP_keySelector_rangeKeys_2";
|
||||
IQueryable<int> pt1_2 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
//decreasing ranges keys.
|
||||
IQueryable<int> pt2_2 = pt1_2.RangePartition(x => x, new[] { 8, 5, 2 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile2), true);
|
||||
var jobInfo2 = pt2_2.Submit();
|
||||
jobInfo2.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2_2, 4, true);
|
||||
result2[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
//increasing ranges keys.
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 });
|
||||
result[1] = pt2.ToList();
|
||||
|
||||
IQueryable<int> pt1_2 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
//decreasing ranges keys.
|
||||
IQueryable<int> pt2_2 = pt1_2.RangePartition(x => x, new[] { 8, 5, 2 });
|
||||
result2[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
Validate.Check(result2);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_isDescending_pcount(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_isDescending_pcount";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_isDescending_pcount";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, true, 2).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 2, true);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, true, 2);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_keyComparer_isDescending(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_keyComparer_isDescending";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_keyComparer_isDescending";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 3, true);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_rangeKeys_keyComparer(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_rangeKeys_keyComparer";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_rangeKeys_keyComparer";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }, Comparer<int>.Default)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 4, false);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }, Comparer<int>.Default);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_keyComparer_isDescending_pcount(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_keyComparer_isDescending_pcount";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_keyComparer_isDescending_pcount";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true, 2)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 2, true);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true, 2);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_rangeKeys_keyComparer_isDescending(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_rangeKeys_keyComparer_isDescending";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_rangeKeys_keyComparer_isDescending";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer<int>.Default, true)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 4, true);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer<int>.Default, true);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_singlePartition_autoSeparators(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_singlePartition_autoSeparators";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_singlePartition_autoSeparators";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.Apply(x => x) // force a merge
|
||||
.RangePartition(x => x)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 1, false);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.Apply(x => x) // force a merge
|
||||
.RangePartition(x => x);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_rangeSeparators_customComparer(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_rangeSeparators_customComparer";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_rangeSeparators_customComparer";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
|
||||
//these keys should be considered not-sorted
|
||||
try
|
||||
{
|
||||
var results = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, new WeirdIntComparer(), false).ToArray();
|
||||
passed &= false; // "an exception should have been thrown (non-sorted separators)."
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
//expected
|
||||
}
|
||||
|
||||
//these keys should also be considered not-sorted
|
||||
try
|
||||
{
|
||||
var results = pt1.RangePartition(x => x, new int[] { 4, 3, 2, 1 }, new WeirdIntComparer(), false).ToArray();
|
||||
passed &= false; // "an exception should have been thrown (non-sorted separators)."
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
//expected
|
||||
}
|
||||
|
||||
//these keys should work
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new int[] { 6, 6, 3, 1 }, new WeirdIntComparer(), false)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 5, new WeirdIntComparer(), false);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
string outFile = "unittest/output/RP_rangeSeparators_customComparer";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
|
||||
//these keys should be considered not-sorted
|
||||
try
|
||||
{
|
||||
var results = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, new WeirdIntComparer(), false).ToArray();
|
||||
passed &= false; // "an exception should have been thrown (non-sorted separators)."
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
//expected
|
||||
}
|
||||
|
||||
//these keys should also be considered not-sorted
|
||||
try
|
||||
{
|
||||
var results = pt1.RangePartition(x => x, new int[] { 4, 3, 2, 1 }, new WeirdIntComparer(), false).ToArray();
|
||||
passed &= false; // "an exception should have been thrown (non-sorted separators)."
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
//expected
|
||||
}
|
||||
|
||||
//these keys should work
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new int[] { 6, 6, 3, 1 }, new WeirdIntComparer(), false)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool TestRangePartitioned(IQueryable<int> pt, int expectedPCount, bool expectedIsDescending)
|
||||
{
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int> ptCount = pt.ApplyPerPartition(x => CountPartitions(x));
|
||||
int cPartitions = ptCount.Count();
|
||||
|
||||
passed &= (expectedPCount == cPartitions);
|
||||
|
||||
if (passed)
|
||||
{
|
||||
int[] ptMax = pt.ApplyPerPartition(x => MaxInPartition(x)).ToArray();
|
||||
int[] ptMin = pt.ApplyPerPartition(x => MinInPartition(x)).ToArray();
|
||||
|
||||
// compare mins and maxs of each partition
|
||||
for (int i = 1; i < cPartitions; i++)
|
||||
{
|
||||
if (false == expectedIsDescending)
|
||||
passed &= (ptMin[i] > ptMax[i - 1]);
|
||||
else
|
||||
passed &= (ptMax[i] < ptMin[i - 1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
return passed;
|
||||
}
|
||||
|
||||
|
||||
[Serializable]
|
||||
public class WeirdIntComparer : IComparer<int>
|
||||
{
|
||||
public int Compare(int x, int y)
|
||||
{
|
||||
int xx = x % 2;
|
||||
int yy = y % 2;
|
||||
return xx.CompareTo(yy);
|
||||
}
|
||||
}
|
||||
|
||||
public static bool RP_rangeSeparators_nullCustomComparer(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_rangeSeparators_nullCustomComparer";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
var results1 = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null, false).ToArray();
|
||||
var results2 = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null).ToArray();
|
||||
//passing is not throwing.
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool TestRangePartitioned(IQueryable<int> pt, int expectedPCount, IComparer<int> comparer, bool expectedIsDescending)
|
||||
{
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int> ptCount = pt.ApplyPerPartition(x => CountPartitions(x));
|
||||
int cPartitions = ptCount.Count();
|
||||
|
||||
passed &= (expectedPCount == cPartitions);
|
||||
|
||||
if (passed)
|
||||
{
|
||||
int[] ptMax = pt.ApplyPerPartition(x => MaxInPartition(x)).ToArray();
|
||||
int[] ptMin = pt.ApplyPerPartition(x => MinInPartition(x)).ToArray();
|
||||
|
||||
// compare mins and maxs of each partition
|
||||
for (int i = 1; i < cPartitions; i++)
|
||||
{
|
||||
passed &= (comparer.Compare(ptMin[i], ptMax[i - 1]) == 0); // ToDo
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
return passed;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,595 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Decorated and undecorated types
|
||||
//
|
||||
|
||||
// A UDT with no attribute shouldn't autoserialize without an explicit [Serializable] attribute
|
||||
public class UDT_Undecorated
|
||||
{
|
||||
public UDT_Undecorated(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
}
|
||||
|
||||
// UDT which is marked as serializable. This should be autoserialized
|
||||
[Serializable]
|
||||
public class UDT_MarkedSerializable
|
||||
{
|
||||
public UDT_MarkedSerializable(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
}
|
||||
|
||||
// UDT that has the old style static serialization methods. This should be rejected.
|
||||
public class UDT_StaticSerializer
|
||||
{
|
||||
public UDT_StaticSerializer(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
|
||||
public static UDT_StaticSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
var val = new UDT_StaticSerializer(0);
|
||||
val.m_field1 = reader.ReadInt32();
|
||||
val.m_field2 = reader.ReadInt32();
|
||||
|
||||
return val;
|
||||
}
|
||||
|
||||
public static void Write(DryadLinqBinaryWriter writer, UDT_StaticSerializer val)
|
||||
{
|
||||
writer.Write(val.m_field1);
|
||||
writer.Write(val.m_field2);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// UDTs with custom serializers
|
||||
//
|
||||
|
||||
// UDT that has an attribute declaring itself as its custom serializer
|
||||
[CustomDryadLinqSerializer(typeof(UDT_SelfCustomSerializer))]
|
||||
public struct UDT_SelfCustomSerializer : IDryadLinqSerializer<UDT_SelfCustomSerializer>
|
||||
{
|
||||
public UDT_SelfCustomSerializer(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
|
||||
public UDT_SelfCustomSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
var val = new UDT_SelfCustomSerializer(0);
|
||||
val.m_field1 = reader.ReadInt32();
|
||||
val.m_field2 = reader.ReadInt32();
|
||||
|
||||
return val;
|
||||
}
|
||||
|
||||
public void Write(DryadLinqBinaryWriter writer, UDT_SelfCustomSerializer val)
|
||||
{
|
||||
writer.Write(val.m_field1);
|
||||
writer.Write(val.m_field2);
|
||||
}
|
||||
}
|
||||
|
||||
// UDT that declares another type as its custom serializer
|
||||
[CustomDryadLinqSerializer(typeof(CustomUDTSerializer))]
|
||||
public struct UDT_ExternalCustomSerializer
|
||||
{
|
||||
public UDT_ExternalCustomSerializer(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
}
|
||||
|
||||
// this is the custom serializer for UDT_ExternalCustomSerializer
|
||||
public class CustomUDTSerializer : IDryadLinqSerializer<UDT_ExternalCustomSerializer>
|
||||
{
|
||||
public UDT_ExternalCustomSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
var val = new UDT_ExternalCustomSerializer(0);
|
||||
val.m_field1 = reader.ReadInt32();
|
||||
val.m_field2 = reader.ReadInt32();
|
||||
|
||||
return val;
|
||||
}
|
||||
|
||||
public void Write(DryadLinqBinaryWriter writer, UDT_ExternalCustomSerializer val)
|
||||
{
|
||||
writer.Write(val.m_field1);
|
||||
writer.Write(val.m_field2);
|
||||
}
|
||||
}
|
||||
|
||||
// UDT with a CustomHpcSerializer attribute that points to an invalid type for the serializer
|
||||
[CustomDryadLinqSerializer(typeof(int))]
|
||||
public struct UDT_BadCustomSerializerType1
|
||||
{
|
||||
public UDT_BadCustomSerializerType1(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
}
|
||||
|
||||
// UDT with a CustomHpcSerializer attribute that points to a serializer type that targets a different UDT
|
||||
[CustomDryadLinqSerializer(typeof(CustomUDTSerializer))]
|
||||
public struct UDT_BadCustomSerializerType2
|
||||
{
|
||||
public UDT_BadCustomSerializerType2(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
}
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Inheritance
|
||||
//
|
||||
|
||||
// A UDT that has sub types. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_BaseType
|
||||
{
|
||||
public UDT_BaseType(int val)
|
||||
{
|
||||
m_baseTypeField = val + 42;
|
||||
}
|
||||
|
||||
public int m_baseTypeField;
|
||||
}
|
||||
|
||||
// A UDT that has derives from a type other than object. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_DerivedType : UDT_BaseType
|
||||
{
|
||||
public UDT_DerivedType(int val)
|
||||
: base(val)
|
||||
{
|
||||
m_derivedTypeField = val + 84;
|
||||
}
|
||||
|
||||
public int m_derivedTypeField;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Field types
|
||||
//
|
||||
|
||||
// A UDT with no data. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_EmptyType
|
||||
{
|
||||
public UDT_EmptyType(int val)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
// A UDT with a public field of type System.Object. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_ObjectField
|
||||
{
|
||||
public UDT_ObjectField(int val)
|
||||
{
|
||||
m_intField = val + 84;
|
||||
m_objectField = null;
|
||||
}
|
||||
|
||||
public int m_intField;
|
||||
public object m_objectField;
|
||||
}
|
||||
|
||||
// A UDT with a public field of type System.Object[]. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_ObjectArrayField
|
||||
{
|
||||
public UDT_ObjectArrayField(int val)
|
||||
{
|
||||
m_intField = val + 84;
|
||||
m_objectArrayField = new object[1];
|
||||
m_objectArrayField[0] = null;
|
||||
}
|
||||
|
||||
public int m_intField;
|
||||
public object[] m_objectArrayField;
|
||||
}
|
||||
|
||||
// A UDT with a public field of type List<System.Object>. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_ObjectListField
|
||||
{
|
||||
public UDT_ObjectListField(int val)
|
||||
{
|
||||
m_intField = val + 84;
|
||||
m_objectListField = new List<object>();
|
||||
m_objectListField.Add(null);
|
||||
}
|
||||
|
||||
public int m_intField;
|
||||
public List<object> m_objectListField;
|
||||
}
|
||||
|
||||
// A UDT with an object field, and a custom serializer. This should not be rejected
|
||||
[CustomDryadLinqSerializer(typeof(UDT_ObjectFieldAndCustomSerializer))]
|
||||
public class UDT_ObjectFieldAndCustomSerializer : IDryadLinqSerializer<UDT_ObjectFieldAndCustomSerializer>
|
||||
{
|
||||
private object m_objectRecord = (object)"";
|
||||
|
||||
// Required by CustomHpcSerializer
|
||||
public UDT_ObjectFieldAndCustomSerializer() { }
|
||||
|
||||
public UDT_ObjectFieldAndCustomSerializer(int val)
|
||||
{
|
||||
m_objectRecord = (object)String.Format("{0}", val);
|
||||
}
|
||||
|
||||
#region IHpcSerializer implementation
|
||||
|
||||
public UDT_ObjectFieldAndCustomSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
string tmp = reader.ReadString();
|
||||
int val = Int32.Parse(tmp);
|
||||
|
||||
return new UDT_ObjectFieldAndCustomSerializer(val);
|
||||
}
|
||||
|
||||
public void Write(DryadLinqBinaryWriter writer, UDT_ObjectFieldAndCustomSerializer record)
|
||||
{
|
||||
writer.Write((string)record.m_objectRecord);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// An empty UDT with a custom serializer. This should be rejected, because even though the user has control over serialization
|
||||
// we will encounter runtime problems if the custom serializer reads/writer 0 bytes. This behavior is simply to discourage empty CS code.
|
||||
[CustomDryadLinqSerializer(typeof(UDT_EmptyTypeWithCustomSerializer))]
|
||||
public class UDT_EmptyTypeWithCustomSerializer : IDryadLinqSerializer<UDT_EmptyTypeWithCustomSerializer>
|
||||
{
|
||||
// Required by CustomHpcSerializer
|
||||
public UDT_EmptyTypeWithCustomSerializer() { }
|
||||
|
||||
public UDT_EmptyTypeWithCustomSerializer(int val)
|
||||
{
|
||||
}
|
||||
|
||||
#region IHpcSerializer implementation
|
||||
|
||||
public UDT_EmptyTypeWithCustomSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
return new UDT_EmptyTypeWithCustomSerializer(0);
|
||||
}
|
||||
|
||||
public void Write(DryadLinqBinaryWriter writer, UDT_EmptyTypeWithCustomSerializer record)
|
||||
{
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Visibility
|
||||
//
|
||||
|
||||
// UDT with a field of non-public type. We cannot handle these
|
||||
[Serializable]
|
||||
public class UDT_FieldOfNonPublicType
|
||||
{
|
||||
private enum SecretCodeLevel
|
||||
{
|
||||
Secret,
|
||||
SuperSecret,
|
||||
}
|
||||
|
||||
public UDT_FieldOfNonPublicType(int val)
|
||||
{
|
||||
m_field = (SecretCodeLevel)val;
|
||||
}
|
||||
|
||||
private SecretCodeLevel m_field;
|
||||
}
|
||||
|
||||
// UDT with a private field of a public type. We do handle these using emitted IL code.
|
||||
[Serializable]
|
||||
public class UDT_PrivateFieldOfPublicType
|
||||
{
|
||||
public UDT_PrivateFieldOfPublicType(int val)
|
||||
{
|
||||
m_field = val + 1;
|
||||
}
|
||||
|
||||
private int m_field;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Nesting
|
||||
//
|
||||
|
||||
[Serializable]
|
||||
public class UDT_Nested_InnerAndOuterSerializable
|
||||
{
|
||||
public UDT_Nested_InnerAndOuterSerializable(int val)
|
||||
{
|
||||
m_field = val + 1;
|
||||
m_field2 = new NestedSerializable(val);
|
||||
}
|
||||
|
||||
private int m_field;
|
||||
private NestedSerializable m_field2;
|
||||
|
||||
[Serializable]
|
||||
public class NestedSerializable
|
||||
{
|
||||
public NestedSerializable(int val)
|
||||
{
|
||||
m_field = val / 2.0;
|
||||
}
|
||||
|
||||
private double m_field;
|
||||
}
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class UDT_Nested_InnerEnum_InnerAndOuterSerializable
|
||||
{
|
||||
public UDT_Nested_InnerEnum_InnerAndOuterSerializable(int val)
|
||||
{
|
||||
m_field = val + 1;
|
||||
m_field2 = (NestedEnum)(val % 3);
|
||||
}
|
||||
|
||||
private int m_field;
|
||||
private NestedEnum m_field2;
|
||||
|
||||
//[Serializable]
|
||||
public enum NestedEnum
|
||||
{
|
||||
Foo = 0,
|
||||
Bar = 1,
|
||||
Baz = 2,
|
||||
}
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class UDT_Nested_OuterSerializableInnerNotSerializable
|
||||
{
|
||||
public UDT_Nested_OuterSerializableInnerNotSerializable(int val)
|
||||
{
|
||||
m_field = val + 1;
|
||||
m_field2 = new NestedNotSerializable(val);
|
||||
}
|
||||
|
||||
private int m_field;
|
||||
private NestedNotSerializable m_field2;
|
||||
|
||||
public class NestedNotSerializable
|
||||
{
|
||||
public NestedNotSerializable(int val)
|
||||
{
|
||||
m_field = val / 2.0;
|
||||
}
|
||||
|
||||
private double m_field;
|
||||
}
|
||||
}
|
||||
|
||||
public class UDT_Nested_OuterNotSerializableInnerSerializable
|
||||
{
|
||||
public UDT_Nested_OuterNotSerializableInnerSerializable(int val)
|
||||
{
|
||||
m_field = val + 1;
|
||||
m_field2 = new NestedSerializable(val);
|
||||
}
|
||||
|
||||
private int m_field;
|
||||
private NestedSerializable m_field2;
|
||||
|
||||
[Serializable]
|
||||
public class NestedSerializable
|
||||
{
|
||||
public NestedSerializable(int val)
|
||||
{
|
||||
m_field = val / 2.0;
|
||||
}
|
||||
|
||||
private double m_field;
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Self reference
|
||||
//
|
||||
|
||||
// First level circular type
|
||||
[Serializable]
|
||||
public class UDT_FirstLevelCircular
|
||||
{
|
||||
public UDT_FirstLevelCircular(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
m_circularRef = null;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
|
||||
public UDT_FirstLevelCircular m_circularRef;
|
||||
}
|
||||
|
||||
// First level circular type with an array reference to self
|
||||
[Serializable]
|
||||
public class UDT_FirstLevelCircularArrayRef
|
||||
{
|
||||
public UDT_FirstLevelCircularArrayRef(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
m_circularRefArray = new UDT_FirstLevelCircular[5];
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
|
||||
public UDT_FirstLevelCircular[] m_circularRefArray;
|
||||
}
|
||||
|
||||
// Second level circular type
|
||||
[Serializable]
|
||||
public class UDT_SecondLevelCircular
|
||||
{
|
||||
public UDT_SecondLevelCircular(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
m_child = new UDT_CircularRefChild(this);
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
|
||||
public UDT_CircularRefChild m_child;
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class UDT_CircularRefChild
|
||||
{
|
||||
public UDT_CircularRefChild(UDT_SecondLevelCircular parent)
|
||||
{
|
||||
m_parent = parent;
|
||||
}
|
||||
|
||||
public UDT_SecondLevelCircular m_parent;
|
||||
}
|
||||
|
||||
// Circular type with custom serializer. Should not be rejected
|
||||
[CustomDryadLinqSerializer(typeof(UDT_CircularTypeWithCustomSerializer))]
|
||||
public class UDT_CircularTypeWithCustomSerializer : IDryadLinqSerializer<UDT_CircularTypeWithCustomSerializer>
|
||||
{
|
||||
public UDT_CircularTypeWithCustomSerializer() { }
|
||||
public UDT_CircularTypeWithCustomSerializer(int val)
|
||||
{
|
||||
m_field1 = val;
|
||||
|
||||
// create each new object with log2(val) self references hanging off of m_next
|
||||
if (val == 0)
|
||||
m_next = null;
|
||||
else
|
||||
m_next = new UDT_CircularTypeWithCustomSerializer(val / 2);
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public UDT_CircularTypeWithCustomSerializer m_next;
|
||||
|
||||
//
|
||||
// sample recursive custom serializer
|
||||
//
|
||||
public UDT_CircularTypeWithCustomSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
UDT_CircularTypeWithCustomSerializer obj = new UDT_CircularTypeWithCustomSerializer();
|
||||
|
||||
bool bHasValidNext = reader.ReadBool();
|
||||
obj.m_field1 = reader.ReadInt32();
|
||||
|
||||
if (bHasValidNext)
|
||||
{
|
||||
obj.m_next = this.Read(reader); // recursively read the next
|
||||
}
|
||||
else
|
||||
{
|
||||
obj.m_next = null; // terminate recursion
|
||||
}
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
public void Write(DryadLinqBinaryWriter writer, UDT_CircularTypeWithCustomSerializer x)
|
||||
{
|
||||
if (x.m_next != null)
|
||||
{
|
||||
writer.Write(true); // bHasValidNext for the reader side
|
||||
writer.Write(x.m_field1);
|
||||
this.Write(writer, x.m_next); // write out recursively
|
||||
}
|
||||
else
|
||||
{
|
||||
writer.Write(false); // bHasValidNext = false for the reader side, makes sure recursive reads stop
|
||||
writer.Write(x.m_field1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This type itself isn't circular, but contains a field of a circular type that is custom serialized. Autoserialization should work for this guy
|
||||
[Serializable]
|
||||
public class UDT_TypeContainingCustomSerializedCircularType
|
||||
{
|
||||
public UDT_TypeContainingCustomSerializedCircularType(int val)
|
||||
{
|
||||
m_field1 = val;
|
||||
m_circularType = new UDT_CircularTypeWithCustomSerializer(val);
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public UDT_CircularTypeWithCustomSerializer m_circularType;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,380 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class SerializationTests
|
||||
{
|
||||
public static void Run(DryadLinqContext context, string matchPattern)
|
||||
{
|
||||
TestLog.Message(" **********************");
|
||||
TestLog.Message(" SerializationTests ");
|
||||
TestLog.Message(" **********************");
|
||||
|
||||
context.LocalDebug = false;
|
||||
|
||||
IQueryable<int> source = DataGenerator.GetSimpleFileSets(context);
|
||||
|
||||
var tests = new Dictionary<string, Action>()
|
||||
{
|
||||
{"UDT_Undecorated", () =>
|
||||
SerializationTests.TestUDT<UDT_Undecorated>(context, source, "Basic UDT with no attribute. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
|
||||
|
||||
{"UDT_MarkedSerializable", () =>
|
||||
SerializationTests.TestUDT<UDT_MarkedSerializable>(context, source, "Basic UDT which is marked as serializable.", null, 0)},
|
||||
|
||||
{"UDT_StaticSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_StaticSerializer>(context, source, "Basic UDT with old stype static serializers. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
|
||||
|
||||
{"UDT_SelfCustomSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_SelfCustomSerializer>(context, source, "UDT implementing custom serializer for itself.", null, 0)},
|
||||
|
||||
{"UDT_ExternalCustomSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_ExternalCustomSerializer>(context, source, "UDT with a custom serializer", null, 0)},
|
||||
|
||||
{"UDT_BadCustomSerializerType1", () =>
|
||||
SerializationTests.TestUDT<UDT_BadCustomSerializerType1>(context, source, "UDT decorated with an invalid custom serializer type.", typeof(DryadLinqException), 0)},
|
||||
|
||||
{"UDT_BadCustomSerializerType2", () =>
|
||||
SerializationTests.TestUDT<UDT_BadCustomSerializerType2>(context, source, "UDT decorated with another UDT's custom serializer.", typeof(DryadLinqException), 0)},
|
||||
|
||||
{"UDT_BaseType", () =>
|
||||
SerializationTests.TestUDT<UDT_BaseType>(context, source, "UDT with subtypes. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleSubtypes"))},
|
||||
|
||||
{"UDT_DerivedType", () =>
|
||||
SerializationTests.TestUDT<UDT_DerivedType>(context, source, "UDT with a base type. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleDerivedtypes"))},
|
||||
|
||||
{"UDT_FieldOfNonPublicType", () =>
|
||||
SerializationTests.TestUDT<UDT_FieldOfNonPublicType>(context, source, "UDT with a field of a non-public type. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("UDTHasFieldOfNonPublicType"))},
|
||||
|
||||
{"UDT_Nested_OuterSerializableInnerNotSerializable", () =>
|
||||
SerializationTests.TestUDT<UDT_Nested_OuterSerializableInnerNotSerializable>(context, source, "Nested UDT, outer type Serializable, inner type not Serializable. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable"))},
|
||||
|
||||
{"UDT_Nested_OuterNotSerializableInnerSerializable", () =>
|
||||
SerializationTests.TestUDT<UDT_Nested_OuterNotSerializableInnerSerializable>(context, source, "Nested UDT, outer type not Serializable, inner type Serializable. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable"))},
|
||||
|
||||
{"UDT_Nested_InnerAndOuterSerializable", () =>
|
||||
SerializationTests.TestUDT<UDT_Nested_InnerAndOuterSerializable>(context, source, "Nested UDT, both outer and innter types marked Serializable.", null, 0)},
|
||||
|
||||
{"UDT_Nested_InnerEnum_InnerAndOuterSerializable", () =>
|
||||
SerializationTests.TestUDT<UDT_Nested_InnerEnum_InnerAndOuterSerializable>(context, source, "Nested UDT, inner enum, both outer and innter types marked Serializable.", null, 0)},
|
||||
|
||||
{"int?", () =>
|
||||
SerializationTests.TestUDT<int?>(context, source, "Nullable type as the UDT. Should be autoserialized.", null, 0)},
|
||||
|
||||
{"UDT_FirstLevelCircular", () =>
|
||||
SerializationTests.TestUDT<UDT_FirstLevelCircular>(context, source, "Circular UDT. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
|
||||
|
||||
{"UDT_FirstLevelCircularArrayRef", () =>
|
||||
SerializationTests.TestUDT<UDT_FirstLevelCircularArrayRef>(context, source, "Circular UDT with array reference to self. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
|
||||
|
||||
{"UDT_SecondLevelCircular", () =>
|
||||
SerializationTests.TestUDT<UDT_SecondLevelCircular>(context, source, "Circular UDT with indirect reference to self. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
|
||||
|
||||
{"UDT_CircularTypeWithCustomSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_CircularTypeWithCustomSerializer>(context, source, "Circular UDT with a custom serializer. Should not be rejected.", null, 0)},
|
||||
|
||||
{"UDT_TypeContainingCustomSerializedCircularType", () =>
|
||||
SerializationTests.TestUDT<UDT_TypeContainingCustomSerializedCircularType>(context, source, "An UDT containing a field of a circular UDT with a custom serializer. Should be autoserialized.", null, 0)},
|
||||
|
||||
{"UDT_ObjectField", () =>
|
||||
SerializationTests.TestUDT<UDT_ObjectField>(context, source, "UDT with a field of type System.Object. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"UDT_ObjectArrayField", () =>
|
||||
SerializationTests.TestUDT<UDT_ObjectArrayField>(context, source, "UDT with a field of type System.Object[]. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"UDT_ObjectListField", () =>
|
||||
SerializationTests.TestUDT<UDT_ObjectListField>(context, source, "UDT with a field of type List<object>. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"object", () =>
|
||||
SerializationTests.TestUDT<object>(context, source, "UDT = System.Object. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"object[]", () =>
|
||||
SerializationTests.TestUDT<object[]>(context, source, "UDT = System.Object[]. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"List<object>", () =>
|
||||
SerializationTests.TestUDT<List<object>>(context, source, "UDT = List<object>. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"UDT_EmptyType", () =>
|
||||
SerializationTests.TestUDT<UDT_EmptyType>(context, source, "Empty UDT. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers"))},
|
||||
|
||||
{"UDT_ObjectFieldAndCustomSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_ObjectFieldAndCustomSerializer>(context, source, "UDT with an Object field, and a custom serializer. Should not be rejected.", null, 0)},
|
||||
|
||||
{"UDT_EmptyTypeWithCustomSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_EmptyTypeWithCustomSerializer>(context, source, "Empty UDT with a custom serializer. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers"))},
|
||||
|
||||
{"UDT_PrivateFieldOfPublicType", () =>
|
||||
SerializationTests.TestUDT<UDT_PrivateFieldOfPublicType>(context, source, "UDT with a private field of a public type. Should be autoserialized.", null, 0)},
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
{
|
||||
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
|
||||
{
|
||||
test.Value.Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// the generic entry method used by most tests in this file
|
||||
public static bool TestUDT<TRecord>(DryadLinqContext context, IQueryable<int> source, string testMessage, Type expectedExceptionType, int expectedErrorCode)
|
||||
{
|
||||
string testName = testMessage;
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
// first make sure SelectFunc knows how to handle the type.
|
||||
try
|
||||
{
|
||||
object o = SelectFunc<TRecord>(0);
|
||||
}
|
||||
catch
|
||||
{
|
||||
TestLog.Message(String.Format("SelectFunc doesn't support type {0}", typeof(TRecord).Name));
|
||||
passed &= false;
|
||||
goto Done;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var results = source.Select(x => SelectFunc<TRecord>(x));
|
||||
|
||||
int count = 0;
|
||||
foreach (var r in results) count++;
|
||||
|
||||
if (count == 0)
|
||||
{
|
||||
TestLog.Message("...FAILED! No elements returned from query");
|
||||
passed &= false;
|
||||
goto Done;
|
||||
}
|
||||
}
|
||||
catch (Exception exp)
|
||||
{
|
||||
if (expectedExceptionType == null)
|
||||
{
|
||||
TestLog.Message(String.Format("...FAILED! Caught {0} while none was expected: ", exp.GetType().Name));
|
||||
passed &= false;
|
||||
goto Done;
|
||||
}
|
||||
else if (expectedExceptionType != exp.GetType())
|
||||
{
|
||||
TestLog.Message(String.Format("...FAILED! Caught {0} while while {1} was expected: {0}", exp.GetType().Name, expectedExceptionType.Name));
|
||||
passed &= false;
|
||||
goto Done;
|
||||
}
|
||||
else if (exp is DryadLinqException && expectedErrorCode != 0 && ((DryadLinqException)exp).ErrorCode != expectedErrorCode)
|
||||
{
|
||||
TestLog.Message("...FAILED! Caught DryadLinqException but fault code is wrong.");
|
||||
passed &= false;
|
||||
goto Done;
|
||||
}
|
||||
|
||||
TestLog.Message("....PASSED (caught correct exception)");
|
||||
goto Done;
|
||||
}
|
||||
|
||||
if (expectedExceptionType == null)
|
||||
{
|
||||
TestLog.Message(".....PASSED");
|
||||
}
|
||||
else
|
||||
{
|
||||
TestLog.Message("...FAILED! No exception was thrown from query.");
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
Done:
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static TRecord SelectFunc<TRecord>(int x)
|
||||
{
|
||||
if (typeof(TRecord) == typeof(object))
|
||||
{
|
||||
return (TRecord)(new object());
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(object[]))
|
||||
{
|
||||
return (TRecord)((object)new object[0]);
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(List<object>))
|
||||
{
|
||||
return (TRecord)((object)new List<object>());
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_Undecorated))
|
||||
{
|
||||
return (TRecord)((object)new UDT_Undecorated(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_SelfCustomSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_SelfCustomSerializer(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_MarkedSerializable))
|
||||
{
|
||||
return (TRecord)((object)new UDT_MarkedSerializable(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_StaticSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_StaticSerializer(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_ExternalCustomSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_ExternalCustomSerializer(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_BadCustomSerializerType1))
|
||||
{
|
||||
return (TRecord)((object)new UDT_BadCustomSerializerType1(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_BadCustomSerializerType2))
|
||||
{
|
||||
return (TRecord)((object)new UDT_BadCustomSerializerType2(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_BaseType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_BaseType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_DerivedType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_DerivedType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_DerivedType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_DerivedType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_FieldOfNonPublicType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_FieldOfNonPublicType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_PrivateFieldOfPublicType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_PrivateFieldOfPublicType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_Nested_OuterSerializableInnerNotSerializable))
|
||||
{
|
||||
return (TRecord)((object)new UDT_Nested_OuterSerializableInnerNotSerializable(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_Nested_OuterNotSerializableInnerSerializable))
|
||||
{
|
||||
return (TRecord)((object)new UDT_Nested_OuterNotSerializableInnerSerializable(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_Nested_InnerAndOuterSerializable))
|
||||
{
|
||||
return (TRecord)((object)new UDT_Nested_InnerAndOuterSerializable(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_Nested_InnerEnum_InnerAndOuterSerializable))
|
||||
{
|
||||
return (TRecord)((object)new UDT_Nested_InnerEnum_InnerAndOuterSerializable(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(int?))
|
||||
{
|
||||
return (TRecord)((object)((int?)x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_FirstLevelCircular))
|
||||
{
|
||||
return (TRecord)((object)new UDT_FirstLevelCircular(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_FirstLevelCircularArrayRef))
|
||||
{
|
||||
return (TRecord)((object)new UDT_FirstLevelCircularArrayRef(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_SecondLevelCircular))
|
||||
{
|
||||
return (TRecord)((object)new UDT_SecondLevelCircular(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_CircularTypeWithCustomSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_CircularTypeWithCustomSerializer(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_TypeContainingCustomSerializedCircularType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_TypeContainingCustomSerializedCircularType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_ObjectField))
|
||||
{
|
||||
return (TRecord)((object)new UDT_ObjectField(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_ObjectArrayField))
|
||||
{
|
||||
return (TRecord)((object)new UDT_ObjectArrayField(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_ObjectListField))
|
||||
{
|
||||
return (TRecord)((object)new UDT_ObjectListField(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_ObjectFieldAndCustomSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_ObjectFieldAndCustomSerializer(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_EmptyType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_EmptyType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_EmptyTypeWithCustomSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_EmptyTypeWithCustomSerializer(x));
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Unrecognized TRecord");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class C1
|
||||
{
|
||||
public int x;
|
||||
public float y;
|
||||
|
||||
public C1(int a, float b)
|
||||
{
|
||||
this.x = a;
|
||||
this.y = b;
|
||||
}
|
||||
}
|
||||
|
||||
public class SimpleTests
|
||||
{
|
||||
public static void Run(string[] args)
|
||||
{
|
||||
Test1(args);
|
||||
//Test2(args);
|
||||
//Test3(args);
|
||||
//Test4(args);
|
||||
//Test5(args);
|
||||
}
|
||||
|
||||
public static void Test1(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile");
|
||||
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var lines = input.Where(x => x.Line.Contains("white"));
|
||||
foreach (var x in lines) Console.WriteLine(x);
|
||||
//var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
|
||||
//result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test2(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile");
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var q1 = input.Where(x => x.Line.Contains("white"));
|
||||
var q2 = input.Where(x => x.Line.Contains("the"));
|
||||
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
|
||||
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
|
||||
DryadLinqQueryable.SubmitAndWait(res1, res2);
|
||||
}
|
||||
|
||||
public static void Test3(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile");
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var words = input.SelectMany(x => x.Line.Split(' '));
|
||||
var groups = words.GroupBy(x => x);
|
||||
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
|
||||
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
|
||||
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
|
||||
result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test4(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext("svc-d2-01");
|
||||
var input = context.FromStore<LineRecord>("hdfs://svc-d2-01:8033/user/misard/foo.txt");
|
||||
var lines = input.Where(x => x.Line.Contains("white"));
|
||||
var result = lines.ToStore("hdfs://svc-d2-01:8033/user/yuanbyu/foo.txt", true);
|
||||
result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test5(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1);
|
||||
Uri dataUri = AzureUtils.ToAzureUri("msrsvc", "test", "testwrite");
|
||||
IEnumerable<LineRecord> lines = DataProvider.ReadData<LineRecord>(context, dataUri);
|
||||
foreach (var x in lines)
|
||||
{
|
||||
Console.WriteLine(x);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class TestResult
|
||||
{
|
||||
public DryadLinqContext Context { get; private set; }
|
||||
public string TestName { get; private set; }
|
||||
public bool Passed { get; private set; }
|
||||
//public string Error { get; private set; }
|
||||
|
||||
public TestResult(string testName, DryadLinqContext context, bool passed)
|
||||
{
|
||||
TestName = testName;
|
||||
Passed = passed;
|
||||
Context = context;
|
||||
}
|
||||
}
|
||||
|
||||
public class TestLog
|
||||
{
|
||||
static internal int nTestsRun;
|
||||
static internal int nTestsPassed;
|
||||
static internal string fileName = "";
|
||||
|
||||
internal static void LogInit(string path)
|
||||
{
|
||||
fileName = path;
|
||||
|
||||
// create empty file
|
||||
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName))
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: QA test harnesses may rely on the formatting
|
||||
internal static void TestStart(string testName)
|
||||
{
|
||||
Message(" --- Starting: [" + testName + "] --- ");
|
||||
}
|
||||
|
||||
// NOTE: QA tests may rely on the formatting
|
||||
internal static void LogResult(TestResult result)
|
||||
{
|
||||
nTestsRun++;
|
||||
if (result.Passed)
|
||||
{
|
||||
nTestsPassed++;
|
||||
}
|
||||
|
||||
Message(" * " + (result.Passed ? "Pass" : "FAIL") );
|
||||
Message(" --- Completed: [" + result.TestName + "] --- ");
|
||||
NewLine();
|
||||
}
|
||||
internal static void LogResult(bool result)
|
||||
{
|
||||
nTestsRun++;
|
||||
if (result)
|
||||
{
|
||||
nTestsPassed++;
|
||||
}
|
||||
NewLine();
|
||||
}
|
||||
|
||||
internal static void NewLine()
|
||||
{
|
||||
if (fileName.Length > 0)
|
||||
{
|
||||
// append data
|
||||
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName, true))
|
||||
{
|
||||
file.WriteLine();
|
||||
}
|
||||
}
|
||||
Console.WriteLine();
|
||||
}
|
||||
internal static void Message(string msg)
|
||||
{
|
||||
if (fileName.Length > 0)
|
||||
{
|
||||
// append data
|
||||
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName, true))
|
||||
{
|
||||
file.WriteLine(msg);
|
||||
}
|
||||
}
|
||||
Console.WriteLine(msg);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,515 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class TypesInQueryTests
|
||||
{
|
||||
public static void Run(DryadLinqContext context, string matchPattern)
|
||||
{
|
||||
TestLog.Message(" **********************");
|
||||
TestLog.Message(" TypesInQueryTests ");
|
||||
TestLog.Message(" **********************");
|
||||
|
||||
var tests = new Dictionary<string, Action>()
|
||||
{
|
||||
{"NonSealedTypeRecords", () => NonSealedTypeRecords(context) },
|
||||
{"DerivedTypeRecords", () => DerivedTypeRecords(context) },
|
||||
{"ObjectRecords", () => ObjectRecords(context) },
|
||||
{"BadRecordsNotSerialized", () => BadRecordsNotSerialized(context) },
|
||||
{"GroupByWithAnonymousTypes_Bug15675", () => GroupByWithAnonymousTypes_Bug15675(context) },
|
||||
{"GroupByWithAnonymousTypes_Pipeline", () => GroupByWithAnonymousTypes_Pipeline(context) },
|
||||
{"GroupByWithAnonymousTypes_MultipleAnonymousTypes", () => GroupByWithAnonymousTypes_MultipleAnonymousTypes(context) },
|
||||
{"GroupByWithAnonymousTypes_GenericWithAnonTypeParam", () => GroupByWithAnonymousTypes_GenericWithAnonTypeParam(context) },
|
||||
{"GroupByWithAnonymousTypes_ArrayOfAnon", () => GroupByWithAnonymousTypes_ArrayOfAnon(context) },
|
||||
{"GroupByWithAnonymousTypes_NestedAnonTypes", () => GroupByWithAnonymousTypes_NestedAnonTypes(context) },
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
{
|
||||
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
|
||||
{
|
||||
test.Value.Invoke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static bool NonSealedTypeRecords(DryadLinqContext context)
|
||||
{
|
||||
string testName = "NonSealedTypeRecords";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<NonSealedClass>[] result = new IEnumerable<NonSealedClass>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => new NonSealedClass()).ToArray();
|
||||
result[0] = output;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => new NonSealedClass()).ToArray();
|
||||
result[1] = output;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool DerivedTypeRecords(DryadLinqContext context)
|
||||
{
|
||||
string testName = "DerivedTypeRecords";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<ParentClass>[] result1 = new IEnumerable<ParentClass>[2];
|
||||
IEnumerable<ChildClass>[] result2 = new IEnumerable<ChildClass>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output1 = pt1.Select(x => new ParentClass()).ToArray();
|
||||
var output2 = pt1.Select(x => new ChildClass()).ToArray();
|
||||
result1[0] = output1;
|
||||
result2[0] = output2;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output1 = pt1.Select(x => new ParentClass()).ToArray();
|
||||
var output2 = pt1.Select(x => new ChildClass()).ToArray();
|
||||
result1[1] = output1;
|
||||
result2[1] = output2;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result1);
|
||||
Validate.Check(result2);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool ObjectRecords(DryadLinqContext context)
|
||||
{
|
||||
string testName = "ObjectRecords";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<object>[] result = new IEnumerable<object>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => new object()).ToArray();
|
||||
result[0] = output;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => new object()).ToArray();
|
||||
result[1] = output;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool BadRecordsNotSerialized(DryadLinqContext context)
|
||||
{
|
||||
string testName = "BadRecordsNotSerialized";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<string>[] result = new IEnumerable<string>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => (x % 2 == 0 ? new ChildClass() : new ParentClass())).Select(x => x is ChildClass ? "child" : "parent").ToArray();
|
||||
result[0] = output;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => (x % 2 == 0 ? new ChildClass() : new ParentClass())).Select(x => x is ChildClass ? "child" : "parent").ToArray();
|
||||
result[1] = output;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
//Bug 15675 -- GroupBy creates a DryadOrderByNode which is confused by the anonymous type.
|
||||
// specifically, the call to DryadSort incorrectly referenced both the anonymous type and the code-gen proxy for the anonymous type.
|
||||
// -- the fix was to clean up how & when we process the types involved in the query so that only the proxy would be referenced.
|
||||
public static bool GroupByWithAnonymousTypes_Bug15675(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_Bug15675";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => new { Num = i % 10 })
|
||||
.GroupBy(x => x.Num, x => x.Num)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => new { Num = i % 10 })
|
||||
.GroupBy(x => x.Num, x => x.Num)
|
||||
.ToArray();
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool GroupByWithAnonymousTypes_Pipeline(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_Pipeline";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => new { Num = i % 10 })
|
||||
.Where(x => true)
|
||||
.GroupBy(x => x.Num, x => x.Num)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => new { Num = i % 10 })
|
||||
.Where(x => true)
|
||||
.GroupBy(x => x.Num, x => x.Num)
|
||||
.ToArray();
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool GroupByWithAnonymousTypes_MultipleAnonymousTypes(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_MultipleAnonymousTypes";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => new { Num = i % 10 })
|
||||
.Where(x => true)
|
||||
.Select(i => new { Num2 = i.Num })
|
||||
.GroupBy(x => x.Num2, x => x.Num2)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => new { Num = i % 10 })
|
||||
.Where(x => true)
|
||||
.Select(i => new { Num2 = i.Num })
|
||||
.GroupBy(x => x.Num2, x => x.Num2)
|
||||
.ToArray();
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool GroupByWithAnonymousTypes_GenericWithAnonTypeParam(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_GenericWithAnonTypeParam";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => MakeNewMyGenericType(new { Num = i % 10 }))
|
||||
.GroupBy(x => x.Field.Num, x => x.Field.Num)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => MakeNewMyGenericType(new { Num = i % 10 }))
|
||||
.GroupBy(x => x.Field.Num, x => x.Field.Num)
|
||||
.ToArray();
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool GroupByWithAnonymousTypes_ArrayOfAnon(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_ArrayOfAnon";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => new[] { new { Num = i % 10 } })
|
||||
.GroupBy(x => x[0].Num, x => x[0].Num)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => new[] { new { Num = i % 10 } })
|
||||
.GroupBy(x => x[0].Num, x => x[0].Num)
|
||||
.ToArray();
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool GroupByWithAnonymousTypes_NestedAnonTypes(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_NestedAnonTypes";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => new { Num = new { NumInner = new int?(i % 10) } }) // nullable-fields present particular challenge.
|
||||
.GroupBy(x => x.Num.NumInner, x => x.Num.NumInner)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => new { Num = new { NumInner = new int?(i % 10) } }) // nullable-fields present particular challenge.
|
||||
.GroupBy(x => x.Num.NumInner, x => x.Num.NumInner)
|
||||
.ToArray();
|
||||
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
// This is a helper method to allow 'naming the anonymous type'
|
||||
// -- there is no name to give 'anonymous' inline to the original query,
|
||||
// but a generic-method only has to call it by pseudonym T, which is a useful workaround.
|
||||
public static MyGenericType<T> MakeNewMyGenericType<T>(T item)
|
||||
{
|
||||
return new MyGenericType<T>(item);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class MyGenericType<T>
|
||||
{
|
||||
public T Field;
|
||||
|
||||
public MyGenericType(T data)
|
||||
{
|
||||
Field = data;
|
||||
}
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class NonSealedClass
|
||||
{
|
||||
public int X;
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class ParentClass
|
||||
{
|
||||
public int X;
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class ChildClass : ParentClass
|
||||
{
|
||||
public int Y;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,410 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.DryadLinq.Internal;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using Microsoft.WindowsAzure.Storage;
|
||||
using Microsoft.WindowsAzure.Storage.Blob;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class Config
|
||||
{
|
||||
public static string accountName = @"";
|
||||
public static string storageKey = @"";
|
||||
public static string containerName = @"";
|
||||
public static string cluster = @"";
|
||||
|
||||
public static string testLogPath = @"";
|
||||
|
||||
public Config(string clusterName, string container, string logPath)
|
||||
{
|
||||
cluster = clusterName;
|
||||
|
||||
AzureSubscriptions azs = new AzureSubscriptions();
|
||||
Task<AzureCluster> clusterTask = azs.GetClusterAsync(clusterName);
|
||||
clusterTask.Wait();
|
||||
|
||||
accountName = clusterTask.Result.StorageAccount;
|
||||
storageKey = clusterTask.Result.StorageKey;
|
||||
containerName = container;
|
||||
testLogPath = logPath;
|
||||
}
|
||||
}
|
||||
|
||||
public class DataGenerator
|
||||
{
|
||||
public DataGenerator()
|
||||
{
|
||||
}
|
||||
|
||||
public static IQueryable<int> CreateSimpleFileSetsEx()
|
||||
{
|
||||
int[] input = { 0, 1, 2 };
|
||||
IEnumerable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
|
||||
IEnumerable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
|
||||
IEnumerable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 4, 4));
|
||||
|
||||
return rangePartition.AsQueryable();
|
||||
}
|
||||
|
||||
public static IEnumerable<IEnumerable<int>> CreateSimpleFileSets()
|
||||
{
|
||||
IEnumerable<IEnumerable<int>> data = new int[][]
|
||||
{
|
||||
new[] { 1, 2, 3, 4 },
|
||||
new[] { 5, 6, 7, 8 },
|
||||
new[] { 9, 10, 11, 12 },
|
||||
};
|
||||
return data;
|
||||
}
|
||||
|
||||
public static IEnumerable<IEnumerable<int>> CreateGroupByReduceDataSet()
|
||||
{
|
||||
// we need quite a few elements to ensure the combiner will be activated in Stage#1 groupBy.
|
||||
// 33 elements per partition should suffice, but 100 per partition is safer.
|
||||
IEnumerable<IEnumerable<int>> data = new int[][]
|
||||
{
|
||||
Enumerable.Range(1,100).ToArray(),
|
||||
Enumerable.Range(101,100).ToArray(),
|
||||
};
|
||||
return data;
|
||||
}
|
||||
|
||||
public static IEnumerable<IEnumerable<int>> CreateRangePartitionDataSet()
|
||||
{
|
||||
// we need a lot of data to ensure sampler will get some data.
|
||||
// A few thousand should suffice.
|
||||
IEnumerable<IEnumerable<int>> data = new int[][]
|
||||
{
|
||||
Enumerable.Range(1,1000).ToArray(),
|
||||
Enumerable.Range(20000,2000).ToArray(),
|
||||
Enumerable.Range(40000,5000).ToArray(),
|
||||
};
|
||||
return data;
|
||||
}
|
||||
|
||||
public static IQueryable<int> GetSimpleFileSets(DryadLinqContext context)
|
||||
{
|
||||
//IEnumerable<IEnumerable<int>> data = new int[][]
|
||||
// {
|
||||
// new[] { 0, 1, 2, 3 },
|
||||
// new[] { 4, 5, 6, 7 },
|
||||
// new[] { 8, 9, 10, 11},
|
||||
// };
|
||||
|
||||
//IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
|
||||
// "unittest/inputdata/SimpleFile.txt"));
|
||||
IQueryable<int> input = context.FromEnumerable(new int[1]);
|
||||
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
|
||||
IQueryable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
|
||||
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 4, 4));
|
||||
//IQueryable<int> store = rangePartition.ToStore(@"unittest/inputdata/SimpleFile.txt");
|
||||
return rangePartition;
|
||||
}
|
||||
|
||||
public static IQueryable<int> GetGroupByReduceDataSet(DryadLinqContext context)
|
||||
{
|
||||
//IEnumerable<IEnumerable<int>> data = new int[][] {
|
||||
// Enumerable.Range(1,100).ToArray(),
|
||||
// Enumerable.Range(101,100).ToArray(),
|
||||
// };
|
||||
|
||||
IQueryable<int> input = context.FromEnumerable(new int[1]);
|
||||
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 2)); // {0, 1}
|
||||
IQueryable<int> partitions = range.HashPartition(x => x, 2); // create 2 partitions
|
||||
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 100 + 1, 100));
|
||||
return rangePartition;
|
||||
}
|
||||
|
||||
public static IQueryable<int> GetRangePartitionDataSet(DryadLinqContext context)
|
||||
{
|
||||
// we need a lot of data to ensure sampler will get some data.
|
||||
// A few thousand should suffice.
|
||||
//IEnumerable<IEnumerable<int>> data = new int[][] {
|
||||
// Enumerable.Range(1,1000).ToArray(),
|
||||
// Enumerable.Range(20000,2000).ToArray(),
|
||||
// Enumerable.Range(40000,5000).ToArray(),
|
||||
// };
|
||||
|
||||
IQueryable<int> input = context.FromEnumerable(new int[1]);
|
||||
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
|
||||
IQueryable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
|
||||
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 20000 + 1, 1000));
|
||||
return rangePartition;
|
||||
}
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class ReverseComparer<T> : IComparer<T>
|
||||
{
|
||||
IComparer<T> _originalComparer;
|
||||
public ReverseComparer(IComparer<T> originalComparer)
|
||||
{
|
||||
_originalComparer = originalComparer ?? Comparer<T>.Default;
|
||||
}
|
||||
|
||||
public int Compare(T x, T y)
|
||||
{
|
||||
return (_originalComparer.Compare(y, x)); //note reversed order of operands
|
||||
}
|
||||
|
||||
public override bool Equals(object obj)
|
||||
{
|
||||
ReverseComparer<T> objTyped = obj as ReverseComparer<T>;
|
||||
|
||||
return objTyped != null && _originalComparer.Equals(objTyped._originalComparer);
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
// Modify the hash code so that it differs from the hash code for the underlying comparer.
|
||||
// It would also probably be good enough to just return _originalComparer.GetHashCode().
|
||||
return unchecked((_originalComparer.GetHashCode() + 123457) * 10007);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public class Utils
|
||||
{
|
||||
public static bool DeleteFile(string accountName, string accountKey, string containerName, string fileName, bool delSubDirs)
|
||||
{
|
||||
try
|
||||
{
|
||||
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + accountName + ";AccountKey=" + accountKey);
|
||||
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
|
||||
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
|
||||
container.CreateIfNotExists();
|
||||
BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
|
||||
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
|
||||
container.SetPermissions(containerPermissions);
|
||||
|
||||
if (false == delSubDirs)
|
||||
{
|
||||
CloudBlockBlob remoteFile = container.GetBlockBlobReference(fileName);
|
||||
remoteFile.DeleteIfExists();
|
||||
}
|
||||
|
||||
if (true == delSubDirs)
|
||||
{
|
||||
foreach (IListBlobItem item in container.ListBlobs(fileName, true))
|
||||
{
|
||||
CloudBlockBlob blob = (CloudBlockBlob)item;
|
||||
blob.DeleteIfExists();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
public static bool FileExists(string accountName, string accountKey, string containerName, string fileName)
|
||||
{
|
||||
try
|
||||
{
|
||||
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + accountName + ";AccountKey=" + accountKey);
|
||||
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
|
||||
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
|
||||
container.CreateIfNotExists();
|
||||
BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
|
||||
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
|
||||
container.SetPermissions(containerPermissions);
|
||||
|
||||
IEnumerable<IListBlobItem> files = container.ListBlobs(fileName, true);
|
||||
return (files.Count() > 0);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
internal static DryadLinqContext MakeBasicConfig(string cluster) //???
|
||||
{
|
||||
var context = new DryadLinqContext(cluster);
|
||||
try
|
||||
{
|
||||
context.JobFriendlyName = "DryadLinq_DevUnitTests";
|
||||
context.CompileForVertexDebugging = true;
|
||||
context.JobEnvironmentVariables.Add("DummyEnvVar", "hello"); //note: this is consumed by a unit-test.
|
||||
|
||||
if (File.Exists("Microsoft.Hpc.Linq.pdb")) // TODO: fix references
|
||||
{
|
||||
context.ResourcesToAdd.Add("Microsoft.Hpc.Linq.pdb");
|
||||
}
|
||||
|
||||
if (File.Exists("Microsoft.Hpc.Dsc.Client.pdb")) // TODO: fix references
|
||||
{
|
||||
context.ResourcesToAdd.Add("Microsoft.Hpc.Dsc.Client.pdb");
|
||||
}
|
||||
|
||||
// To prevent job from running forever, and blocking other test
|
||||
context.JobRuntimeLimit = (int)TimeSpan.FromMinutes(30).TotalSeconds;
|
||||
|
||||
|
||||
//config.AllowConcurrentUserDelegatesInSingleProcess = false;
|
||||
|
||||
// If we are on Azure, we have to set the nodeGroup to "NodeRole" so that the default of "ComputeNodes" is not used
|
||||
// This fixes "FromEnumerableTests" on Azure which queries the active node-group.
|
||||
// Note also, the headnode for an azure deployment defaults to "HPCCluster" (at least from James' script)
|
||||
int onAzureInt = 0;
|
||||
string onAzureString = Environment.GetEnvironmentVariable("CCP_SCHEDULERONAZURE");
|
||||
if (onAzureString != null)
|
||||
{
|
||||
int.TryParse(onAzureString, out onAzureInt);
|
||||
}
|
||||
|
||||
if (onAzureInt == 1)
|
||||
{
|
||||
context.NodeGroup = "NodeRole";
|
||||
}
|
||||
|
||||
}
|
||||
catch (DryadLinqException)
|
||||
{
|
||||
}
|
||||
return context;
|
||||
}
|
||||
|
||||
internal static DryadLinqRecordReader<TRecord> MakeDryadRecordReader<TRecord>(DryadLinqContext context, string readPath)
|
||||
{
|
||||
DryadLinqFactory<TRecord> factory = (DryadLinqFactory<TRecord>)DryadLinqCodeGen.GetFactory(context, typeof(TRecord));
|
||||
NativeBlockStream nativeStream = ReflectionHelper.CreateDryadLinqFileStream(readPath, FileMode.Open, FileAccess.Read);
|
||||
// ??? NativeBlockStream nativeStream = ReflectionHelper.CreateDryadLinqFileStream(readPath, FileMode.Open, FileAccess.Read, DscCompressionScheme.None);
|
||||
DryadLinqRecordReader<TRecord> reader = factory.MakeReader(nativeStream);
|
||||
return reader;
|
||||
}
|
||||
}
|
||||
|
||||
public class Validate
|
||||
{
|
||||
public static void
|
||||
Check<T>(
|
||||
IEnumerable<T>[] ss,
|
||||
IComparer<T> comparer = null,
|
||||
bool sort = true,
|
||||
bool verbose = false,
|
||||
IComparer<T> sortcomparer = null
|
||||
)
|
||||
{
|
||||
|
||||
if (ss.Length == 0) return;
|
||||
|
||||
if (comparer == null)
|
||||
{
|
||||
comparer = Comparer<T>.Default;
|
||||
if (comparer == null)
|
||||
{
|
||||
throw new ArgumentNullException("Can't not be null.");
|
||||
}
|
||||
}
|
||||
if (sortcomparer == null)
|
||||
sortcomparer = comparer;
|
||||
|
||||
T[][] aa = new T[ss.Length][];
|
||||
for (int i = 0; i < aa.Length; i++)
|
||||
{
|
||||
aa[i] = ss[i].ToArray();
|
||||
if (sort) Array.Sort(aa[i], sortcomparer);
|
||||
}
|
||||
int len = aa[0].Length;
|
||||
for (int i = 1; i < aa.Length; i++)
|
||||
{
|
||||
if (aa[i].Length != len)
|
||||
{
|
||||
throw new Exception("Wrong number of elements.");
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < len; i++)
|
||||
{
|
||||
T elem = aa[0][i];
|
||||
for (int j = 1; j < aa.Length; j++)
|
||||
{
|
||||
if (verbose)
|
||||
{
|
||||
//TestOutput.WriteLine("Comparing {0} to {1}", elem.ToString(), aa[j][i].ToString());
|
||||
}
|
||||
if (comparer.Compare(elem, aa[j][i]) != 0)
|
||||
{
|
||||
throw new Exception("Elements failed to match: " + elem + " != " + aa[j][i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal static bool outFileExists(string outFile)
|
||||
{
|
||||
try
|
||||
{
|
||||
return Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static class ReflectionHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// Create DryadLinqFileStream object via reflection
|
||||
/// </summary>
|
||||
///<param name="parameters"></param>
|
||||
/// <returns></returns>
|
||||
public static NativeBlockStream CreateDryadLinqFileStream(params object[] parameters)
|
||||
{
|
||||
return Assembly.LoadWithPartialName("Microsoft.Hpc.Linq").GetType("Microsoft.Hpc.Linq.Internal.HpcLinqFileStream") //???
|
||||
.GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, parameters.Select(p => p.GetType()).ToArray(), null)
|
||||
.Invoke(parameters) as NativeBlockStream;
|
||||
}
|
||||
|
||||
private static Type s_errorCodeType = null;
|
||||
public static int GetDryadLinqErrorCode(string name)
|
||||
{
|
||||
if (s_errorCodeType == null)
|
||||
{
|
||||
Assembly asm = Assembly.Load("Microsoft.Research.DryadLinq");
|
||||
Type[] types = asm.GetTypes();
|
||||
foreach (var t in types)
|
||||
{
|
||||
if (t.Name == "DryadLinqErrorCode")
|
||||
{
|
||||
s_errorCodeType = t;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
var finfo = s_errorCodeType.GetField(name);
|
||||
return (int)finfo.GetValue(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
// comment the following line to run on Azure
|
||||
#define local
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class WordCount
|
||||
{
|
||||
public static void WordCountExample()
|
||||
{
|
||||
#if local
|
||||
// This overload runs the computation on your local computer using a single worker
|
||||
var config = new DryadLinqContext(1);
|
||||
|
||||
var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
|
||||
// You can create inputs from any IEnumerable source using this method
|
||||
var input = config.FromEnumerable(lines);
|
||||
#else
|
||||
string clusterName = "Replace with your HDInsight 3.0 cluster name";
|
||||
// to use the davinci.txt example input below, select your cluster's default
|
||||
// storage account and container, which automatically includes the sample text
|
||||
string accountName = "Replace with a storage account name";
|
||||
string containerName = "Replace with a storage container name";
|
||||
|
||||
// This overload creates an Azure-based computation
|
||||
var config = new DryadLinqContext(clusterName);
|
||||
config.JobFriendlyName = "DryadLINQ Sample Wordcount";
|
||||
|
||||
// plain text files should be read as type LineRecord
|
||||
var input = config.FromStore<LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
|
||||
"example/data/gutenberg/davinci.txt"));
|
||||
#endif
|
||||
|
||||
var words = input.SelectMany(x => x.Line.Split(' '));
|
||||
var groups = words.GroupBy(x => x);
|
||||
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
|
||||
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
|
||||
|
||||
#if local
|
||||
// any collection computed by the query can be materialized back at the client,
|
||||
// not just the 'output' collection. For large collections this is expensive!
|
||||
foreach (LineRecord line in toOutput)
|
||||
{
|
||||
Console.WriteLine(line.Line);
|
||||
}
|
||||
#else
|
||||
// the 'true' parameter to ToStore means the output will be over-written if you run
|
||||
// the job more than once
|
||||
var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
|
||||
"wc-out.txt"), true).SubmitAndWait();
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +1,23 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
|
||||
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Dryad" version="0.1.4-beta001" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -24,13 +25,13 @@
|
|||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -54,6 +55,13 @@
|
|||
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<NuGetPackageImportStamp>b0b5ea03</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<TargetName>$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<TargetName>$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Midl>
|
||||
|
|
@ -61,7 +69,7 @@
|
|||
</Midl>
|
||||
<ClCompile>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\..\peloponnese\HadoopBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_LIB;WIN32_LEAN_AND_MEAN;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<MinimalRebuild>false</MinimalRebuild>
|
||||
<BasicRuntimeChecks>Default</BasicRuntimeChecks>
|
||||
|
|
@ -72,10 +80,6 @@
|
|||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
|
||||
</ClCompile>
|
||||
<Lib>
|
||||
<AdditionalDependencies>Microsoft.Research.Peloponnese.HadoopBridge.lib</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>..\..\..\..\..\peloponnese\bin\$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
|
||||
</Lib>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<Midl>
|
||||
|
|
@ -88,13 +92,9 @@
|
|||
</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\..\peloponnese\HadoopBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
|
||||
</ClCompile>
|
||||
<Lib>
|
||||
<AdditionalDependencies>Microsoft.Research.Peloponnese.HadoopBridge.lib</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>..\..\..\..\..\peloponnese\bin\$(Configuration)</AdditionalLibraryDirectories>
|
||||
</Lib>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="include\channelbuffer.h" />
|
||||
|
|
@ -149,13 +149,16 @@
|
|||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
|
||||
</ImportGroup>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
|
||||
</packages>
|
||||
|
|
@ -929,6 +929,37 @@ RChannelBufferHdfsWriter::RChannelBufferHdfsWriter(const char* uri)
|
|||
Hdfs::InstanceAccessor ia(bridge);
|
||||
ia.Discard();
|
||||
}
|
||||
|
||||
char envUserName[128];
|
||||
DWORD ret = ::GetEnvironmentVariableA("USER", envUserName, sizeof(envUserName));
|
||||
if (ret > 0 && ret < sizeof(envUserName)-1)
|
||||
{
|
||||
m_user.Set(envUserName);
|
||||
DrLogI("Hdfs writer set user to %s from environment", m_user.GetString());
|
||||
}
|
||||
else
|
||||
{
|
||||
if (ret == 0 && GetLastError() != ERROR_ENVVAR_NOT_FOUND)
|
||||
{
|
||||
DrLogW("Can't get USER environment variable: %s", DRERRORSTRING(DrGetLastError()));
|
||||
}
|
||||
if (ret >= sizeof(envUserName)-1)
|
||||
{
|
||||
DrLogW("USER environment is too long: %d", ret);
|
||||
}
|
||||
|
||||
DWORD bufSize = sizeof(envUserName);
|
||||
BOOL bRet = ::GetUserNameA(envUserName, &bufSize);
|
||||
if (bRet == 0)
|
||||
{
|
||||
DrLogW("Can't get username: %s", DRERRORSTRING(DrGetLastError()));
|
||||
}
|
||||
else
|
||||
{
|
||||
m_user.Set(envUserName);
|
||||
DrLogI("Hdfs writer set user to %s from username", m_user.GetString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1021,7 +1052,7 @@ bool RChannelBufferHdfsWriter::Open(Hdfs::Instance** pInstance,
|
|||
Hdfs::InstanceAccessor ia(instance);
|
||||
|
||||
Hdfs::Writer* writer;
|
||||
bool openedWriter = ia.OpenWriter(filePath.GetString(), false, &writer);
|
||||
bool openedWriter = ia.OpenCreate(filePath.GetString(), 1024 * 1024, -1, &writer);
|
||||
if (!openedWriter)
|
||||
{
|
||||
char* errorMsg = ia.GetExceptionMessage();
|
||||
|
|
@ -1036,6 +1067,21 @@ bool RChannelBufferHdfsWriter::Open(Hdfs::Instance** pInstance,
|
|||
ia.Discard();
|
||||
return false;
|
||||
}
|
||||
bool setPermissions = ia.SetOwnerAndPermission(filePath.GetString(), m_user.GetString(), NULL, 0644);
|
||||
if (!setPermissions)
|
||||
{
|
||||
char* errorMsg = ia.GetExceptionMessage();
|
||||
DrStr64 description;
|
||||
description.SetF("Can't set HDFS file permissions '%s': %s",
|
||||
m_uri.GetString(), errorMsg);
|
||||
HadoopNative::DisposeString(errorMsg);
|
||||
|
||||
m_completionItem.Attach(MakeErrorItem(DryadError_ChannelOpenError,
|
||||
description.GetString()));
|
||||
|
||||
ia.Discard();
|
||||
return false;
|
||||
}
|
||||
|
||||
DrLogI("Opened Hdfs writer for %s", m_uri.GetString());
|
||||
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ private:
|
|||
bool Open(Hdfs::Instance** pInstance, Hdfs::Writer** pWriter);
|
||||
bool AddToQueue(WriteEntry* entry);
|
||||
|
||||
DrStr64 m_user;
|
||||
DrStr64 m_uri;
|
||||
DryadBList<WriteEntry> m_queue;
|
||||
UInt32 m_queueLength;
|
||||
|
|
|
|||
|
|
@ -2205,7 +2205,7 @@ void RChannelBufferWriterNativeFile::EagerCloseFile()
|
|||
}
|
||||
else
|
||||
{
|
||||
DrLogI( "Set final file pointer");
|
||||
DrLogI( "Set final file pointer. Length %I64u", finalLength.QuadPart);
|
||||
}
|
||||
|
||||
ok = SetEndOfFile(m_bufferedFileHandle);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -21,13 +21,13 @@
|
|||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>false</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>false</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -19,14 +19,14 @@
|
|||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
<CLRSupport>false</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -22,7 +24,7 @@
|
|||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
</PropertyGroup>
|
||||
|
|
@ -30,7 +32,7 @@
|
|||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -54,6 +56,13 @@
|
|||
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<NuGetPackageImportStamp>7f5f48b5</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<TargetName>$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<TargetName>$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Midl>
|
||||
|
|
@ -72,10 +81,6 @@
|
|||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
|
||||
</ClCompile>
|
||||
<Lib>
|
||||
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
|
||||
</Lib>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<Midl>
|
||||
|
|
@ -91,10 +96,6 @@
|
|||
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;..\channel\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
|
||||
</ClCompile>
|
||||
<Lib>
|
||||
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
|
||||
</Lib>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="include\dryadvertex.h" />
|
||||
|
|
@ -123,13 +124,19 @@
|
|||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
</ImportGroup>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="Native" />
|
||||
</packages>
|
||||
|
|
@ -277,7 +277,6 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
|
|||
DryadVertexProgramRef* pProgram)
|
||||
{
|
||||
DryadVertexProgramBase* program;
|
||||
|
||||
//
|
||||
// If factory is not supplied, try to get it from factory registry.
|
||||
// If still unable, fail with error.
|
||||
|
|
@ -295,12 +294,14 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
|
|||
return DryadError_VertexInitialization;
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Get vertex factory. If one cannot be found, report initialization error
|
||||
//
|
||||
factory = LookUpFactory(argumentList[0]);
|
||||
if (factory == NULL)
|
||||
{
|
||||
DrLogW("Factory Registry called with unknown factory UID %s.", argumentList[0].GetString());
|
||||
DrStr128 errorString;
|
||||
errorString.SetF("Factory Registry called with unknown factory UID %s",
|
||||
argumentList[0].GetString());
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ using System.Runtime.InteropServices;
|
|||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using Microsoft.Research.Peloponnese.Azure;
|
||||
|
||||
namespace Microsoft.Research.Dryad.Channel
|
||||
{
|
||||
|
|
@ -74,7 +74,7 @@ namespace Microsoft.Research.Dryad.Channel
|
|||
{
|
||||
Log.LogInformation("Opening read for " + source.AbsoluteUri);
|
||||
string account, key, container, blobName;
|
||||
AzureUtils.FromAzureUri(source, out account, out key, out container, out blobName);
|
||||
Utils.FromAzureUri(source, out account, out key, out container, out blobName);
|
||||
client = new AzureDfsClient(account, key, container, false, new PeloponneseLogger(Log.Logger));
|
||||
client.SetParallelThreadCount(4);
|
||||
|
||||
|
|
@ -94,7 +94,7 @@ namespace Microsoft.Research.Dryad.Channel
|
|||
private async Task OpenBlob()
|
||||
{
|
||||
string account, key, container, blobName;
|
||||
AzureUtils.FromAzureUri(source, out account, out key, out container, out blobName);
|
||||
Utils.FromAzureUri(source, out account, out key, out container, out blobName);
|
||||
|
||||
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(source.Query);
|
||||
|
||||
|
|
@ -105,9 +105,11 @@ namespace Microsoft.Research.Dryad.Channel
|
|||
++blobIndex;
|
||||
}
|
||||
|
||||
Log.LogInformation("Opening read for blob " + blobName);
|
||||
Uri blobUri = Utils.ToAzureUri(account, container, blobName, null, key);
|
||||
|
||||
readStream = (await client.GetDfsFileStreamAsync(blobName, ExecutionTimeout, new PeloponneseLogger(Log.Logger))).Stream;
|
||||
Log.LogInformation("Opening read for blob " + blobUri.AbsoluteUri);
|
||||
|
||||
readStream = (await client.GetDfsStreamReaderAsync(blobUri, ExecutionTimeout, new PeloponneseLogger(Log.Logger)));
|
||||
|
||||
long offset = -1;
|
||||
if (query["offset"] != null)
|
||||
|
|
@ -215,7 +217,7 @@ namespace Microsoft.Research.Dryad.Channel
|
|||
int toRead = (int)Math.Min((long)managedBuffer.Length, bytesToRead);
|
||||
|
||||
Log.LogInformation("About to read buffer length " + toRead);
|
||||
readData.nRead = await AzureUtils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
|
||||
readData.nRead = await Utils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
|
||||
{
|
||||
return await readStream.ReadAsync(managedBuffer, 0, toRead);
|
||||
});
|
||||
|
|
@ -301,7 +303,7 @@ namespace Microsoft.Research.Dryad.Channel
|
|||
{
|
||||
int toRead = (int)Math.Min(buffer.LongLength, endOffset - currentOffset);
|
||||
|
||||
int nRead = await AzureUtils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
|
||||
int nRead = await Utils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
|
||||
{
|
||||
return await readStream.ReadAsync(buffer, 0, toRead);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ using System.Linq;
|
|||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using Microsoft.Research.Peloponnese.Azure;
|
||||
|
||||
namespace Microsoft.Research.Dryad.Channel
|
||||
{
|
||||
|
|
|
|||
|
|
@ -24,32 +24,8 @@ using System.Runtime.InteropServices;
|
|||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("managedchannel")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("")]
|
||||
[assembly: AssemblyProduct("managedchannel")]
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.ManagedChannel")]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("f806a093-c5ae-4752-82d3-26bcc2341b29")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("0.1.0.0")]
|
||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
||||
|
|
|
|||
|
|
@ -4,24 +4,44 @@
|
|||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.HDInsight.Threading" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.0" newVersion="1.2.1.0" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
|
|
@ -9,11 +11,12 @@
|
|||
<OutputType>Library</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>Microsoft.Research.Dryad.Channel</RootNamespace>
|
||||
<AssemblyName>DryadManagedChannel</AssemblyName>
|
||||
<AssemblyName>Microsoft.Research.Dryad.ManagedChannel</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir>
|
||||
<RestorePackages>true</RestorePackages>
|
||||
<NuGetPackageImportStamp>72e7670e</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
|
|
@ -35,59 +38,38 @@
|
|||
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Data.Services.Client" />
|
||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
|
|
@ -97,6 +79,9 @@
|
|||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\..\..\..\SharedAssemblyInfo.cs">
|
||||
<Link>Properties\SharedAssemblyInfo.cs</Link>
|
||||
</Compile>
|
||||
<Compile Include="AzureReader.cs" />
|
||||
<Compile Include="AzureWriter.cs" />
|
||||
<Compile Include="FileWriter.cs" />
|
||||
|
|
@ -116,10 +101,16 @@
|
|||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.2" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -17,32 +17,27 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
|||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace Microsoft.Research.DryadLinq
|
||||
namespace VertexHostMain
|
||||
{
|
||||
internal class YarnScheduler : IScheduler
|
||||
internal class ConfigDependency : Microsoft.Research.Peloponnese.Shared.AssemblyDependencyAttribute
|
||||
{
|
||||
public string m_headNode;
|
||||
|
||||
public void Connect(string headNode)
|
||||
public
|
||||
ConfigDependency()
|
||||
: base("Microsoft.Research.Dryad.VertexHost.exe.config", false)
|
||||
{
|
||||
this.m_headNode = headNode;
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
class Program
|
||||
{
|
||||
this.m_headNode = null;
|
||||
}
|
||||
|
||||
public IServerVersion GetServerVersion()
|
||||
static int Main(string[] args)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
return VertexHost.Main(args);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
[assembly: VertexHostMain.ConfigDependency()]
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.VertexHost")]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("5534b858-8472-4dc3-8f24-5b59f8805156")]
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>VertexHostMain</RootNamespace>
|
||||
<AssemblyName>Microsoft.Research.Dryad.VertexHost</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<TargetFrameworkProfile />
|
||||
<NuGetPackageImportStamp>fb5afcd2</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<OutputPath>..\..\..\..\bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<DebugType>full</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
|
||||
<OutputPath>..\..\..\..\bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<Optimize>true</Optimize>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
<Reference Include="Microsoft.CSharp" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\..\..\..\SharedAssemblyInfo.cs">
|
||||
<Link>Properties\SharedAssemblyInfo.cs</Link>
|
||||
</Compile>
|
||||
<Compile Include="Program.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\vertexHost\VertexHost.vcxproj">
|
||||
<Project>{0cf3d1d5-9bbe-4175-979b-ec6138ef4f37}</Project>
|
||||
<Name>VertexHostBody</Name>
|
||||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="app.config" />
|
||||
<None Include="packages.config" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
<?xml version="1.0"?>
|
||||
<configuration>
|
||||
<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/></startup>
|
||||
<runtime>
|
||||
<gcServer enabled="true"></gcServer>
|
||||
<gcConcurrent enabled="false"></gcConcurrent>
|
||||
<generatePublisherEvidence enabled="false"/>
|
||||
<gcAllowVeryLargeObjects enabled="true" />
|
||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -21,13 +21,13 @@
|
|||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
LIBRARY "DryadLINQNativeChannels.dll"
|
||||
LIBRARY "Microsoft.Research.Dryad.DryadLinq.NativeWrapper.dll"
|
||||
|
||||
EXPORTS
|
||||
GetTotalLength
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -19,14 +19,14 @@
|
|||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -52,8 +52,8 @@
|
|||
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">DryadLINQNativeChannels</TargetName>
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">DryadLINQNativeChannels</TargetName>
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad.DryadLinq.NativeWrapper</TargetName>
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad.DryadLinq.NativeWrapper</TargetName>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Midl>
|
||||
|
|
@ -111,6 +111,7 @@
|
|||
<ClInclude Include="stdafx.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="version.cpp" />
|
||||
<ClCompile Include="wrappernativeinfostubs.cpp" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@
|
|||
<ClCompile Include="wrappernativeinfostubs.cpp">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="version.cpp">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="DryadLINQNativeChannels.def">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
#ifdef _MANAGED
|
||||
#pragma managed
|
||||
|
||||
using namespace System::Reflection;
|
||||
using namespace System::Runtime::InteropServices;
|
||||
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.DryadLinq.NativeWrapper")];
|
||||
[assembly: AssemblyDescription("")];
|
||||
[assembly: AssemblyConfiguration("")];
|
||||
[assembly: AssemblyCompany("Microsoft Corporation")];
|
||||
[assembly: AssemblyProduct("Microsoft.Research.Dryad.DryadLinq.NativeWrapper")];
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
|
||||
[assembly: AssemblyTrademark("")];
|
||||
[assembly: AssemblyCulture("")];
|
||||
|
||||
[assembly: ComVisible(false)];
|
||||
|
||||
[assembly: AssemblyVersion("0.2.1.0")];
|
||||
[assembly: AssemblyFileVersion("0.2.1.0")];
|
||||
#endif
|
||||
|
|
@ -28,7 +28,6 @@ limitations under the License.
|
|||
#include <mscoree.h>
|
||||
#include <wrappernativeinfo.h>
|
||||
|
||||
|
||||
#pragma managed
|
||||
|
||||
#pragma warning(disable:4947) // so that we can use Assembly::LoadWithPartialName()
|
||||
|
|
@ -198,18 +197,25 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
|
|||
logName = System::IO::Path::Combine(logDirectory, logName);
|
||||
}
|
||||
|
||||
int threadsPerWorker = 1;
|
||||
System::String^ threadsPerWorkerStr = System::Environment::GetEnvironmentVariable("DRYAD_THREADS_PER_WORKER");
|
||||
if (threadsPerWorkerStr != nullptr)
|
||||
{
|
||||
threadsPerWorker = Int32::Parse(threadsPerWorkerStr);
|
||||
}
|
||||
DrLogI("ManagedWrapperVertex: threadsPerWorker %u", threadsPerWorker);
|
||||
|
||||
DrLogI("ManagedWrapperVertex: %p %u %u", nativeInfo, numberOfInputChannels, numberOfOutputChannels);
|
||||
DrLogI("ManagedWrapperVertex: Calling %s.%s", GetArgument(2), GetArgument(3));
|
||||
DrLogging::FlushLog();
|
||||
|
||||
|
||||
DrStr128 errorMsg;
|
||||
DrError error;
|
||||
|
||||
{
|
||||
//
|
||||
// Instead of invoking the vertex entry point directly from here, we delegate it to the bridge method in the Microsoft.Hpc.Linq assembly, specifically:
|
||||
// static int Microsoft.Hpc.Linq.Internal.VertexEnv.VertexBridge(string vertexBridgeArgs)
|
||||
// Instead of invoking the vertex entry point directly from here, we delegate it to the bridge method in the Microsoft.Research.DryadLinq assembly, specifically:
|
||||
// static int Microsoft.Research.DryadLinq.Internal.VertexEnv.VertexBridge(string vertexBridgeArgs)
|
||||
//
|
||||
// This indirect method of invoking the vertex entry point is used so that any type load / assembly load problems coming from user code
|
||||
// can be caught and reported with full details using the same mechanism that other vertex failures go through (exception dumped into vertexexception.txt etc.)
|
||||
|
|
@ -217,8 +223,10 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
|
|||
// The format of vertexBridgeArgs is simply a comma separated string packing vertex assembly, class, method name, and the *actual* vertex method args (==the native channel string)
|
||||
// L"<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
|
||||
//
|
||||
System::String ^bridgeAssemblyPartialName = gcnew System::String(L"Microsoft.Research.DryadLinq");
|
||||
System::String ^bridgeClassName = gcnew System::String(L"Microsoft.Research.DryadLinq.Internal.VertexEnv");
|
||||
System::String^ classFullName = gcnew System::String(GetArgument(2));
|
||||
System::String^ assemblyName = classFullName->Substring(0, classFullName->LastIndexOf('.'));
|
||||
System::String ^bridgeAssemblyName = gcnew System::String(assemblyName);
|
||||
System::String ^bridgeClassName = gcnew System::String(assemblyName + ".Internal.VertexEnv");
|
||||
System::String ^bridgeMethodName = gcnew System::String(L"VertexBridge");
|
||||
|
||||
//
|
||||
|
|
@ -235,21 +243,19 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
|
|||
vertexMethodArgs->Append(gcnew System::String(arg.GetString()));
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Get assembly path, class name, and method name, and construct the vertex bridge args with the following format:
|
||||
// "<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
|
||||
//
|
||||
System::Text::StringBuilder ^vertexBridgeArg = gcnew System::Text::StringBuilder();
|
||||
vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"c:\\HpcTemp\\user\\jobID\\Microsoft.Hpc.Linq0.dll";
|
||||
vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"\\HpcTemp\\user\\jobID\\Microsoft.Research.DryadLinq0.dll";
|
||||
vertexBridgeArg->Append(",");
|
||||
vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Hpc.Linq.HpcLinq__Vertex";
|
||||
vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Research.DryadLinq.DryadLinq__Vertex";
|
||||
vertexBridgeArg->Append(",");
|
||||
vertexBridgeArg->Append(gcnew System::String(GetArgument(3))); // vertex entry method name L"Select__1";
|
||||
vertexBridgeArg->Append(",");
|
||||
vertexBridgeArg->Append(vertexMethodArgs->ToString());
|
||||
|
||||
|
||||
DrLogI("ManagedWrapperVertex: Calling into Vertex Bridge to invoke Vertex Entry: %s", GetArgument(3));
|
||||
DrLogging::FlushLog();
|
||||
|
||||
|
|
@ -260,10 +266,20 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
|
|||
//
|
||||
try
|
||||
{
|
||||
System::Console::WriteLine("Assembly name " + bridgeAssemblyPartialName);
|
||||
System::Reflection::Assembly ^vertexBridgeAsm = System::Reflection::Assembly::LoadWithPartialName(bridgeAssemblyPartialName);
|
||||
System::Type ^vertexBridgeType = vertexBridgeAsm->GetType(gcnew System::String(bridgeClassName));
|
||||
System::Reflection::MethodInfo ^vertexBridgeMethod = vertexBridgeType->GetMethod(gcnew System::String(bridgeMethodName),
|
||||
System::Reflection::Assembly ^vertexBridgeAsm;
|
||||
try
|
||||
{
|
||||
vertexBridgeAsm = System::Reflection::Assembly::Load(bridgeAssemblyName);
|
||||
}
|
||||
catch (System::Exception ^ie)
|
||||
{
|
||||
DrLogI("ManagedWrapperVertex: Failed to load assembly %s: %s", bridgeAssemblyName, ie->ToString());
|
||||
System::String^ asmLoc = System::IO::Path::Combine("..", bridgeAssemblyName + ".dll");
|
||||
vertexBridgeAsm = System::Reflection::Assembly::LoadFrom(asmLoc);
|
||||
}
|
||||
System::Type ^vertexBridgeType = vertexBridgeAsm->GetType(bridgeClassName);
|
||||
System::Reflection::MethodInfo ^vertexBridgeMethod
|
||||
= vertexBridgeType->GetMethod(bridgeMethodName,
|
||||
static_cast<System::Reflection::BindingFlags>(System::Reflection::BindingFlags::NonPublic |
|
||||
System::Reflection::BindingFlags::Static));
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -21,13 +21,13 @@
|
|||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -18,19 +18,20 @@
|
|||
<Keyword>Win32Proj</Keyword>
|
||||
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir>
|
||||
<RestorePackages>true</RestorePackages>
|
||||
<ProjectName>VertexHostBody</ProjectName>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -56,6 +57,13 @@
|
|||
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<NuGetPackageImportStamp>26f18b05</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<TargetName>Microsoft.Research.Dryad.$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<TargetName>Microsoft.Research.Dryad.$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Midl>
|
||||
|
|
@ -112,6 +120,7 @@
|
|||
</PostBuildEvent>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="version.cpp" />
|
||||
<ClCompile Include="vertexHost.cpp" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
|
|
@ -124,13 +133,13 @@
|
|||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
</ImportGroup>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup>
|
||||
<ClCompile Include="vertexHost.cpp" />
|
||||
<ClCompile Include="version.cpp" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="app.config" />
|
||||
|
|
|
|||
|
|
@ -1,26 +1,42 @@
|
|||
<?xml version="1.0" encoding="utf-8" ?>
|
||||
<configuration>
|
||||
<runtime>
|
||||
<gcServer enabled="true"></gcServer>
|
||||
<gcConcurrent enabled="false"></gcConcurrent>
|
||||
<generatePublisherEvidence enabled="false"/>
|
||||
<gcAllowVeryLargeObjects enabled="true" />
|
||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
|
||||
</packages>
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
#ifdef _MANAGED
|
||||
#pragma managed
|
||||
|
||||
using namespace System::Reflection;
|
||||
using namespace System::Runtime::InteropServices;
|
||||
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.VertexHost")];
|
||||
[assembly: AssemblyDescription("")];
|
||||
[assembly: AssemblyConfiguration("")];
|
||||
[assembly: AssemblyCompany("Microsoft Corporation")];
|
||||
[assembly: AssemblyProduct("Microsoft.Research.Dryad.VertexHost")];
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
|
||||
[assembly: AssemblyTrademark("")];
|
||||
[assembly: AssemblyCulture("")];
|
||||
|
||||
[assembly: ComVisible(false)];
|
||||
|
||||
[assembly: AssemblyVersion("0.2.1.0")];
|
||||
[assembly: AssemblyFileVersion("0.2.1.0")];
|
||||
#endif
|
||||
|
|
@ -174,27 +174,27 @@ void SetLoggingLevel()
|
|||
GetLoggingFileName(logFileName);
|
||||
DrLogging::Initialize(logFileName);
|
||||
|
||||
WCHAR traceLevel [MAX_PATH];
|
||||
HRESULT hr = DrGetEnvironmentVariable(L"DRYAD_TRACE_LEVEL", traceLevel);
|
||||
WCHAR loggingLevel [MAX_PATH];
|
||||
HRESULT hr = DrGetEnvironmentVariable(L"DRYAD_LOGGING_LEVEL", loggingLevel);
|
||||
if(hr == DrError_OK)
|
||||
{
|
||||
if(wcscmp(traceLevel, L"OFF") == 0)
|
||||
if(wcscmp(loggingLevel, L"OFF") == 0)
|
||||
{
|
||||
DrLogging::SetLoggingLevel(LogLevel_Off);
|
||||
}
|
||||
else if(wcscmp(traceLevel, L"CRITICAL") == 0)
|
||||
else if(wcscmp(loggingLevel, L"CRITICAL") == 0)
|
||||
{
|
||||
DrLogging::SetLoggingLevel(LogLevel_Assert);
|
||||
}
|
||||
else if(wcscmp(traceLevel, L"ERROR") == 0)
|
||||
else if(wcscmp(loggingLevel, L"ERROR") == 0)
|
||||
{
|
||||
DrLogging::SetLoggingLevel(LogLevel_Error);
|
||||
}
|
||||
else if(wcscmp(traceLevel, L"WARN") == 0)
|
||||
else if(wcscmp(loggingLevel, L"WARN") == 0)
|
||||
{
|
||||
DrLogging::SetLoggingLevel(LogLevel_Warning);
|
||||
}
|
||||
else if(wcscmp(traceLevel, L"INFO") == 0)
|
||||
else if(wcscmp(loggingLevel, L"INFO") == 0)
|
||||
{
|
||||
DrLogging::SetLoggingLevel(LogLevel_Info);
|
||||
}
|
||||
|
|
@ -249,16 +249,24 @@ static void ExceptionHandler(System::Object^ sender, System::UnhandledExceptionE
|
|||
//
|
||||
// Start up vertex host
|
||||
//
|
||||
public ref class VertexHost
|
||||
{
|
||||
public:
|
||||
[System::Security::SecurityCriticalAttribute]
|
||||
[System::Runtime::ExceptionServices::HandleProcessCorruptedStateExceptionsAttribute]
|
||||
#if defined(_AMD64_)
|
||||
int wmain(int argc, wchar_t* wargv[])
|
||||
#else
|
||||
int __cdecl wmain(int argc, wchar_t* wargv[])
|
||||
#endif
|
||||
static int Main(array<System::String^>^ managedArgs)
|
||||
{
|
||||
try
|
||||
{
|
||||
int argc = managedArgs->Length;
|
||||
wchar_t** wargv = new wchar_t*[argc+1];
|
||||
for (int i=0; i<argc; ++i)
|
||||
{
|
||||
pin_ptr<const wchar_t> wch = PtrToStringChars(managedArgs[i]);
|
||||
wargv[i] = _wcsdup(wch);
|
||||
}
|
||||
wargv[argc] = NULL;
|
||||
|
||||
//
|
||||
// Enable logging based on environment variable
|
||||
//
|
||||
|
|
@ -276,7 +284,7 @@ int __cdecl wmain(int argc, wchar_t* wargv[])
|
|||
//
|
||||
// trace for startup
|
||||
//
|
||||
DrLogI("Vertex Host starting");
|
||||
DrLogE("Vertex Host starting");
|
||||
|
||||
//
|
||||
// Get environment variable to know whether to break into debugger
|
||||
|
|
@ -295,6 +303,12 @@ int __cdecl wmain(int argc, wchar_t* wargv[])
|
|||
char** argv;
|
||||
DrGetUtf8CommandArgs(argc, wargv, &argv);
|
||||
|
||||
for (int i=0; i<managedArgs->Length; ++i)
|
||||
{
|
||||
free(wargv[i]);
|
||||
}
|
||||
delete [] wargv;
|
||||
|
||||
//
|
||||
// Initialize the dryad communication layer with the command line arguments
|
||||
//
|
||||
|
|
@ -347,6 +361,7 @@ int __cdecl wmain(int argc, wchar_t* wargv[])
|
|||
return 1;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// Simple data class which contains the byte array and its length.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -25,13 +28,13 @@
|
|||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CLRSupport>Pure</CLRSupport>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>Pure</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -59,6 +62,7 @@
|
|||
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad</TargetName>
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad</TargetName>
|
||||
<NuGetPackageImportStamp>46822e65</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Midl>
|
||||
|
|
@ -223,13 +227,22 @@
|
|||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
|
||||
</ImportGroup>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
@ -5,10 +5,6 @@
|
|||
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
|
||||
<Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="Resource Files">
|
||||
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
|
||||
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="Source Files">
|
||||
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
|
||||
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ limitations under the License.
|
|||
|
||||
#pragma once
|
||||
|
||||
using namespace Microsoft::Research::Peloponnese::Storage;
|
||||
using namespace Microsoft::Research::Peloponnese::Azure;
|
||||
|
||||
DRCLASS(DrAzureInputStream) : public DrInputStream
|
||||
{
|
||||
|
|
|
|||
|
|
@ -129,37 +129,168 @@ DrHdfsInputStream::DrHdfsInputStream()
|
|||
m_hdfsInstance = DrNull;
|
||||
}
|
||||
|
||||
HRESULT DrHdfsInputStream::Open(DrUniversePtr universe, DrNativeString streamUri)
|
||||
HRESULT DrHdfsInputStream::Open(DrUniversePtr universe, DrNativeString streamUri, DrNativeString recordType)
|
||||
{
|
||||
DrString uri = DrString(streamUri);
|
||||
DrString record = DrString(recordType);
|
||||
|
||||
DrLogI("Opening instance for %s", uri.GetChars());
|
||||
DrLogI("Opening instance for %s record type %s", uri.GetChars(), record.GetChars());
|
||||
|
||||
return OpenInternal(universe, uri);
|
||||
return OpenInternal(universe, uri, record);
|
||||
}
|
||||
|
||||
|
||||
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri)
|
||||
#ifdef _MANAGED
|
||||
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType)
|
||||
{
|
||||
m_streamUri = streamUri;
|
||||
HRESULT err = S_OK;
|
||||
|
||||
#ifdef _MANAGED
|
||||
|
||||
try
|
||||
{
|
||||
#endif
|
||||
DrLogI("Opening instance for %s: %s", streamUri.GetChars(), recordType.GetChars());
|
||||
m_hdfsInstance = GetHdfsServiceInstance(streamUri);
|
||||
|
||||
HdfsFileInfo^ stream = m_hdfsInstance->GetFileInfo(streamUri.GetString(), true);
|
||||
m_fileNameArray = stream->fileNameArray;
|
||||
UInt32 totalPartitionCount;
|
||||
|
||||
if (recordType.Compare("Microsoft.Research.DryadLinq.LineRecord") == 0)
|
||||
{
|
||||
DrLogI("Getting block-level file info for %s", streamUri.GetChars());
|
||||
totalPartitionCount = static_cast<UInt32>(stream->blockArray->Length);
|
||||
}
|
||||
else
|
||||
{
|
||||
DrLogI("Getting file info for %s", streamUri.GetChars());
|
||||
totalPartitionCount = m_fileNameArray->Length;
|
||||
}
|
||||
|
||||
DrLogI("Partition count %d", totalPartitionCount);
|
||||
|
||||
/* Allocate these arrays even if they're size 0, to avoid
|
||||
NullReferenceException later */
|
||||
m_affinity = DrNew DrAffinityArray(totalPartitionCount);
|
||||
m_partOffsets = DrNew DrUINT64Array(totalPartitionCount);
|
||||
m_partFileIds = DrNew DrUINT32Array(totalPartitionCount);
|
||||
|
||||
if (recordType.Compare("Microsoft.Research.DryadLinq.LineRecord") == 0)
|
||||
{
|
||||
for (UINT32 i = 0; i < totalPartitionCount; ++i)
|
||||
{
|
||||
HdfsBlockInfo^ partition = stream->blockArray[i];
|
||||
m_affinity[i] = DrNew DrAffinity();
|
||||
m_affinity[i]->SetWeight(partition->Size);
|
||||
m_partOffsets[i] = partition->Offset;
|
||||
m_partFileIds[i] = partition->fileIndex;
|
||||
|
||||
for (int j = 0; j < partition->Hosts->Length; ++j)
|
||||
{
|
||||
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
|
||||
if (location != DrNull)
|
||||
{
|
||||
m_affinity[i]->AddLocality(location);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
int fileBlockIndex = 0;
|
||||
for (UINT32 i = 0; i < totalPartitionCount; ++i)
|
||||
{
|
||||
m_partOffsets[i] = 0;
|
||||
m_partFileIds[i] = i;
|
||||
|
||||
HdfsBlockInfo^ partition = stream->blockArray[fileBlockIndex];
|
||||
DrAssert(partition->fileIndex == i);
|
||||
|
||||
long long fileSize = partition->Size;
|
||||
|
||||
HashSet<DrResourceRef>^ locations = DrNew HashSet<DrResourceRef>();
|
||||
for (int j = 0; j < partition->Hosts->Length; ++j)
|
||||
{
|
||||
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
|
||||
if (location != DrNull)
|
||||
{
|
||||
locations->Add(location);
|
||||
}
|
||||
}
|
||||
|
||||
++fileBlockIndex;
|
||||
|
||||
while (fileBlockIndex < stream->blockArray->Length && stream->blockArray[fileBlockIndex]->fileIndex == i)
|
||||
{
|
||||
partition = stream->blockArray[fileBlockIndex];
|
||||
fileSize += partition->Size;
|
||||
|
||||
if (locations->Count > 0)
|
||||
{
|
||||
HashSet<DrResourceRef>^ newLocations = DrNew HashSet<DrResourceRef>();
|
||||
for (int j = 0; j < partition->Hosts->Length; ++j)
|
||||
{
|
||||
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
|
||||
if (location != DrNull)
|
||||
{
|
||||
newLocations->Add(location);
|
||||
}
|
||||
}
|
||||
|
||||
locations->IntersectWith(newLocations);
|
||||
}
|
||||
}
|
||||
|
||||
m_affinity[i] = DrNew DrAffinity();
|
||||
m_affinity[i]->SetWeight(fileSize);
|
||||
|
||||
System::Text::StringBuilder^ locationText;
|
||||
if (locations->Count > 0)
|
||||
{
|
||||
locationText = gcnew System::Text::StringBuilder("File " + m_fileNameArray[i] + " merged locations:");
|
||||
}
|
||||
else
|
||||
{
|
||||
locationText = gcnew System::Text::StringBuilder("File " + m_fileNameArray[i] + " no shared locations");
|
||||
}
|
||||
|
||||
HashSet<DrResourceRef>::Enumerator^ enumerator = locations->GetEnumerator();
|
||||
while (enumerator->MoveNext())
|
||||
{
|
||||
m_affinity[i]->AddLocality(enumerator->Current);
|
||||
locationText->Append(" ");
|
||||
locationText->Append(enumerator->Current->GetName().GetString());
|
||||
}
|
||||
|
||||
DrString locationLog(locationText->ToString());
|
||||
DrLogI("%s", locationLog.GetChars());
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (System::Exception ^e)
|
||||
{
|
||||
m_error = e->ToString();
|
||||
DrString msg(m_error);
|
||||
DrLogE("Got HDFS exception %s", msg.GetChars());
|
||||
err = System::Runtime::InteropServices::Marshal::GetHRForException(e);
|
||||
}
|
||||
finally
|
||||
{
|
||||
// TODO: How do we clean this up?
|
||||
//hdfsInstance->Discard();
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
#else
|
||||
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType)
|
||||
{
|
||||
m_streamUri = streamUri;
|
||||
HRESULT err = S_OK;
|
||||
|
||||
|
||||
DrLogI("Opening instance for %s", streamUri.GetChars());
|
||||
m_hdfsInstance = GetHdfsServiceInstance(streamUri);
|
||||
|
||||
#ifdef _MANAGED
|
||||
DrLogI("Getting file info for %s", streamUri.GetChars());
|
||||
HdfsFileInfo^ stream = m_hdfsInstance->GetFileInfo(streamUri.GetString(), true);
|
||||
m_fileNameArray = stream->fileNameArray;
|
||||
UInt32 totalPartitionCount = static_cast<UInt32>(stream->blockArray->Length);
|
||||
|
||||
#else
|
||||
bool ret = HdfsBridgeNative::Initialize();
|
||||
if (!ret)
|
||||
{
|
||||
|
|
@ -195,7 +326,6 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
|
|||
totalPartitionCount = fs.GetNumberOfBlocks();
|
||||
|
||||
m_fileNameArray = (const char **)fs.GetFileNameArray();
|
||||
#endif
|
||||
|
||||
DrLogI("Partition count %d", totalPartitionCount);
|
||||
|
||||
|
|
@ -207,21 +337,13 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
|
|||
|
||||
for (UINT32 i=0; i<totalPartitionCount; ++i)
|
||||
{
|
||||
#ifdef _MANAGED
|
||||
HdfsBlockInfo^ partition = stream->blockArray[i];
|
||||
#else
|
||||
HdfsBridgeNative::HdfsBlockLocInfo* partition = fs.GetBlockInfo(i);
|
||||
#endif
|
||||
m_affinity[i] = DrNew DrAffinity();
|
||||
m_affinity[i]->SetWeight(partition->Size);
|
||||
m_partOffsets[i] = partition->Offset;
|
||||
m_partFileIds[i] = partition->fileIndex;
|
||||
|
||||
#ifdef _MANAGED
|
||||
for (int j = 0; j < partition->Hosts->Length; ++j)
|
||||
#else
|
||||
for (int j = 0; j < partition->numberOfHosts; ++j)
|
||||
#endif
|
||||
{
|
||||
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
|
||||
if (location != DrNull)
|
||||
|
|
@ -229,28 +351,12 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
|
|||
m_affinity[i]->AddLocality(location);
|
||||
}
|
||||
}
|
||||
#ifndef _MANAGED
|
||||
delete partition;
|
||||
#endif
|
||||
}
|
||||
#ifdef _MANAGED
|
||||
}
|
||||
catch (System::Exception ^e)
|
||||
{
|
||||
m_error = e->ToString();
|
||||
DrString msg(m_error);
|
||||
DrLogE("Got HDFS exception %s", msg.GetChars());
|
||||
err = System::Runtime::InteropServices::Marshal::GetHRForException(e);
|
||||
}
|
||||
finally
|
||||
{
|
||||
// TODO: How do we clean this up?
|
||||
//hdfsInstance->Discard();
|
||||
}
|
||||
#endif
|
||||
|
||||
return err;
|
||||
}
|
||||
#endif
|
||||
|
||||
DrNativeString DrHdfsInputStream::GetError()
|
||||
{
|
||||
|
|
@ -468,6 +574,25 @@ HRESULT DrHdfsOutputStream::FinalizeSuccessfulParts(DrOutputPartitionArrayRef pa
|
|||
errorText.SetF("%s", DrString(m_error).GetChars());
|
||||
return E_FAIL;
|
||||
}
|
||||
|
||||
String^ userName = Environment::GetEnvironmentVariable("USER");
|
||||
if (userName == nullptr)
|
||||
{
|
||||
userName = Environment::UserName;
|
||||
}
|
||||
try
|
||||
{
|
||||
m_hdfsInstance->SetOwnerAndPermission(m_baseUri, userName, nullptr, Convert::ToInt16("0755", 8));
|
||||
}
|
||||
catch (Exception^ e)
|
||||
{
|
||||
DrString drDst(m_baseUri);
|
||||
DrString err(e->ToString());
|
||||
DrLogE("Can't set %s permissions finalizing HDFS output: %s", drDst.GetChars(), err.GetChars());
|
||||
m_error = "Can't set " + m_baseUri + " permissions finalizing HDFS output: " + e->ToString();
|
||||
errorText.SetF("%s", DrString(m_error).GetChars());
|
||||
return E_FAIL;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ DRCLASS(DrHdfsInputStream) : public DrInputStream
|
|||
{
|
||||
public:
|
||||
DrHdfsInputStream();
|
||||
HRESULT Open(DrUniversePtr universe, DrNativeString streamUri);
|
||||
HRESULT OpenInternal(DrUniversePtr universe, DrString streamUri);
|
||||
HRESULT Open(DrUniversePtr universe, DrNativeString streamUri, DrNativeString recordType);
|
||||
HRESULT OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType);
|
||||
DrNativeString GetError();
|
||||
|
||||
virtual DrString GetStreamName() DROVERRIDE;
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
|
|||
DrStringR remoteName,
|
||||
DrPartitionInputStream::OverridePtr over,
|
||||
bool mustOverride,
|
||||
bool pathIsRooted,
|
||||
DrString line,
|
||||
DrUniversePtr universe)
|
||||
{
|
||||
|
|
@ -112,9 +113,17 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
|
|||
sep = lineCopy.IndexOfChar(',');
|
||||
if (sep == DrStr_InvalidIndex)
|
||||
{
|
||||
DrLogW("Malformed line %s: no list of machines", line.GetChars());
|
||||
int n = sscanf_s(lineCopy.GetChars(), "%I64u", &parsedSize);
|
||||
if (n != 1)
|
||||
{
|
||||
DrLogW("Malformed line %s: can't parse size", line.GetChars());
|
||||
return false;
|
||||
}
|
||||
|
||||
affinity->SetWeight(parsedSize);
|
||||
|
||||
lineCopy = DrString("");
|
||||
}
|
||||
else
|
||||
{
|
||||
DrString partitionSizeString;
|
||||
|
|
@ -134,11 +143,17 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
|
|||
}
|
||||
|
||||
if (lineCopy.GetCharsLength() == 0)
|
||||
{
|
||||
if (!pathIsRooted || mustOverride)
|
||||
{
|
||||
DrLogW("Malformed line %s: no partition machines", line.GetChars());
|
||||
return false;
|
||||
}
|
||||
|
||||
remoteName.Set(" %Invalid% ");
|
||||
return true;
|
||||
}
|
||||
|
||||
int numberOfReplicas = 0;
|
||||
while (lineCopy.GetCharsLength() > 0)
|
||||
{
|
||||
|
|
@ -231,6 +246,12 @@ HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString st
|
|||
mustOverride = true;
|
||||
}
|
||||
|
||||
bool pathIsRooted = false;
|
||||
if (m_pathNameOnComputer.IndexOfChar(':') != DrStr_InvalidIndex)
|
||||
{
|
||||
pathIsRooted = true;
|
||||
}
|
||||
|
||||
int numberOfParts;
|
||||
int n = sscanf_s(partitionSizeLine.GetChars(), "%d", &numberOfParts);
|
||||
if (n != 1)
|
||||
|
|
@ -280,6 +301,7 @@ HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString st
|
|||
remoteName,
|
||||
m_override[i],
|
||||
mustOverride,
|
||||
pathIsRooted,
|
||||
partitionLine,
|
||||
universe) == false)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -19,5 +19,22 @@ limitations under the License.
|
|||
*/
|
||||
|
||||
#ifdef _MANAGED
|
||||
[assembly:System::Runtime::InteropServices::ComVisible(false)];
|
||||
#pragma managed
|
||||
|
||||
using namespace System::Reflection;
|
||||
using namespace System::Runtime::InteropServices;
|
||||
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad")];
|
||||
[assembly: AssemblyDescription("")];
|
||||
[assembly: AssemblyConfiguration("")];
|
||||
[assembly: AssemblyCompany("Microsoft Corporation")];
|
||||
[assembly: AssemblyProduct("Microsoft.Research.Dryad")];
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
|
||||
[assembly: AssemblyTrademark("")];
|
||||
[assembly: AssemblyCulture("")];
|
||||
|
||||
[assembly: ComVisible(false)];
|
||||
|
||||
[assembly: AssemblyVersion("1.0.0.0")];
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")];
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -570,7 +570,7 @@ DrString DrClusterInternal::TranslateFileToURI(DrString leafName, DrString direc
|
|||
}
|
||||
|
||||
void DrClusterInternal::ScheduleProcess(DrAffinityListRef affinities,
|
||||
DrString name, DrString commandLine,
|
||||
DrString name, DrString commandLineArgs,
|
||||
DrProcessTemplatePtr processTemplate,
|
||||
DrPSRListenerPtr listener)
|
||||
{
|
||||
|
|
@ -604,7 +604,7 @@ void DrClusterInternal::ScheduleProcess(DrAffinityListRef affinities,
|
|||
DrLogI("Starting schedule process for %s.%s",
|
||||
processTemplate->GetProcessClass().GetChars(), name.GetChars());
|
||||
|
||||
IProcess^ rawProcess = m_cluster->NewProcess(process, commandLine.GetString());
|
||||
IProcess^ rawProcess = m_cluster->NewProcess(process, processTemplate->GetCommandLineBase().GetString(), commandLineArgs.GetString());
|
||||
process->SetProcess(rawProcess);
|
||||
|
||||
DrLogI("Assigned GUID %s to process for %s.%s",
|
||||
|
|
|
|||
|
|
@ -45,9 +45,10 @@ public:
|
|||
DrResourcePtr srcResource, DrResourcePtr dstResource, int compressionMode) = 0;
|
||||
|
||||
virtual void ScheduleProcess(DrAffinityListRef affinities,
|
||||
DrString name, DrString commandLine,
|
||||
DrString name, DrString commandLineArgs,
|
||||
DrProcessTemplatePtr processTemplate,
|
||||
DrPSRListenerPtr listener) = 0;
|
||||
|
||||
virtual void CancelScheduleProcess(DrProcessHandlePtr process) = 0;
|
||||
|
||||
virtual void WaitForStateChange(DrProcessHandlePtr process, DrPSRListenerPtr listener) = 0;
|
||||
|
|
|
|||
|
|
@ -165,9 +165,10 @@ public:
|
|||
DrResourcePtr srcResource, DrResourcePtr dstResource, int compressionMode) DROVERRIDE;
|
||||
|
||||
virtual void ScheduleProcess(DrAffinityListRef affinities,
|
||||
DrString name, DrString commandLine,
|
||||
DrString name, DrString commandLineArgs,
|
||||
DrProcessTemplatePtr processTemplate,
|
||||
DrPSRListenerPtr listener) DROVERRIDE;
|
||||
|
||||
virtual void CancelScheduleProcess(DrProcessHandlePtr process) DROVERRIDE;
|
||||
|
||||
virtual void WaitForStateChange(DrProcessHandlePtr process, DrPSRListenerPtr listener) DROVERRIDE;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="Native" />
|
||||
</packages>
|
||||
|
|
@ -24,7 +24,8 @@ limitations under the License.
|
|||
#include <msclr\lock.h>
|
||||
|
||||
using namespace System::IO;
|
||||
using namespace Microsoft::Research::Peloponnese::Storage;
|
||||
using namespace Microsoft::Research::Peloponnese::Azure;
|
||||
using namespace Microsoft::Research::Peloponnese::Hdfs;
|
||||
using namespace msclr;
|
||||
|
||||
//
|
||||
|
|
@ -126,7 +127,7 @@ public:
|
|||
DrCalypsoReporter::DrCalypsoReporter(DrNativeString uriString)
|
||||
{
|
||||
System::Uri^ uri = DrNew System::Uri(uriString);
|
||||
if (uri->Scheme == AzureUtils::BlobScheme)
|
||||
if (uri->Scheme == Utils::BlobScheme)
|
||||
{
|
||||
m_logStream = DrNew AzureLogAppendStream(uri, 0x20, false, false, gcnew PeloponneseLogger());
|
||||
m_flushInterval = 1000;
|
||||
|
|
|
|||
|
|
@ -470,8 +470,7 @@ void DrCohort::StartProcess(DrGraphPtr graph, int version)
|
|||
processName.SetF("%s v.%d", m_description.GetChars(), version);
|
||||
|
||||
DrString commandLine;
|
||||
commandLine.SetF("%s --noredirect --startfrompn %d",
|
||||
m_processTemplate->GetCommandLineBase().GetChars(), m_list->Size());
|
||||
commandLine.SetF("--noredirect --startfrompn %d", m_list->Size());
|
||||
|
||||
for (i=0; i<m_list->Size(); ++i)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1743,7 +1743,7 @@ void DrTeeVertex::ReactToFailedVertex(DrVertexOutputGeneratorPtr failedGenerator
|
|||
DrAssert(m_inputEdges->GetNumberOfEdges() == 1);
|
||||
|
||||
DrEdge e = m_inputEdges->GetEdge(0);
|
||||
DrLogI("Tee vertex %d.%d: calling ReactToFailedVertex on remote edge", this->m_id, GetVersion());
|
||||
DrLogI("Tee vertex %d.%d: calling ReactToFailedVertex on remote edge", this->m_id, failedGenerator->GetVersion());
|
||||
e.m_remoteVertex->ReactToFailedVertex(failedGenerator, DrNull, DrNull, DrNull, originalReason);
|
||||
|
||||
/* fill in a new generator if it's already there, e.g. if the upstream vertex is a DrStorageVertex */
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
|
||||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("DryadAnalysis")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("MSIT")]
|
||||
[assembly: AssemblyProduct("DryadAnalysis")]
|
||||
[assembly: AssemblyCopyright("Copyright © MSIT 2008")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("286845bc-d060-49ff-9de8-4381471a0dc6")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("1.0.0.0")]
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -19,15 +19,24 @@ limitations under the License.
|
|||
|
||||
*/
|
||||
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Web;
|
||||
using System.Xml;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Research.Peloponnese.Hdfs;
|
||||
using Microsoft.Research.Peloponnese.Shared;
|
||||
using Microsoft.Research.Peloponnese.Yarn;
|
||||
using Microsoft.Research.Tools;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Research.Calypso.Tools;
|
||||
using JobStatus = Microsoft.Research.Peloponnese.ClusterUtils.JobStatus;
|
||||
|
||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
||||
namespace Microsoft.Research.JobObjectModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Dynamic information of all the jobs and machines in a cluster.
|
||||
|
|
@ -69,12 +78,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// See if a status is already cached.
|
||||
/// </summary>
|
||||
/// <param name="clusterName">Name of cluster.</param>
|
||||
/// <param name="config">Cluster configuration.</param>
|
||||
/// <returns>The cached status.</returns>
|
||||
public static ClusterStatus LookupStatus(string clusterName)
|
||||
public static ClusterStatus LookupStatus(ClusterConfiguration config)
|
||||
{
|
||||
if (ClusterStatuses.ContainsKey(clusterName))
|
||||
return ClusterStatuses[clusterName];
|
||||
if (ClusterStatuses.ContainsKey(config.Name))
|
||||
{
|
||||
var retval = ClusterStatuses[config.Name];
|
||||
if (retval.Config.Equals(config))
|
||||
return retval;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -119,22 +132,20 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// The cached of tasks on the cluster.
|
||||
/// </summary>
|
||||
/// <param name="virtualCluster">Virtual cluster selected; defined only for Scope clusters.</param>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="reportProgress">Used to report progress.</param>
|
||||
public IEnumerable<ClusterJobInformation> GetClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public IEnumerable<ClusterJobInformation> GetClusterJobList(string virtualCluster, CommManager manager)
|
||||
{
|
||||
this.RecomputeClusterJobList(virtualCluster, reporter, reportProgress);
|
||||
this.RecomputeClusterJobList(virtualCluster, manager);
|
||||
return this.clusterJobs.Values.ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Force the recomputation of the cluster job list.
|
||||
/// </summary>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
||||
/// <param name="reportProgress">Used to report progress.</param>
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
// ReSharper disable once UnusedParameter.Global
|
||||
protected abstract void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress);
|
||||
protected abstract void RecomputeClusterJobList(string virtualCluster, CommManager manager);
|
||||
|
||||
/// <summary>
|
||||
/// Discover the (unique) dryadlinq job corresponding to a cluster job.
|
||||
|
|
@ -160,12 +171,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// </summary>
|
||||
/// <param name="job">Job to discover.</param>
|
||||
/// <returns>The cluster job, or null if not found.</returns>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="reportProgress">Used to report progress.</param>
|
||||
public virtual ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, StatusReporter reporter, Action<int> reportProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public virtual ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, CommManager manager)
|
||||
{
|
||||
if (this.clusterJobs == null)
|
||||
this.RecomputeClusterJobList(job.VirtualCluster, reporter, reportProgress);
|
||||
this.RecomputeClusterJobList(job.VirtualCluster, manager);
|
||||
return this.clusterJobs[job.ClusterJobId];
|
||||
}
|
||||
|
||||
|
|
@ -173,13 +183,12 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// Refresh the job summary status.
|
||||
/// </summary>
|
||||
/// <param name="summary">Summary to refresh.</param>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="reportProgress">Used to report progress.</param>
|
||||
public virtual void RefreshStatus(DryadLinqJobSummary summary, StatusReporter reporter, Action<int> reportProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public virtual void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
|
||||
{
|
||||
// refresh the whole list
|
||||
this.RecomputeClusterJobList(summary.VirtualCluster, reporter, reportProgress);
|
||||
ClusterJobInformation info = this.DiscoverClusterJob(summary, reporter, reportProgress);
|
||||
// refresh the whole list: too expensive
|
||||
// this.RecomputeClusterJobList(summary.VirtualCluster, manager);
|
||||
ClusterJobInformation info = this.DiscoverClusterJob(summary, manager);
|
||||
if (info == null)
|
||||
{
|
||||
summary.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
||||
|
|
@ -198,6 +207,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// A fake cluster keeps some information about past jobs on a local filesystem, to allow post-mortem debugging.
|
||||
/// </summary>
|
||||
|
|
@ -226,10 +236,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// Recompute the list of jobs on the cluster and add them to the clusterJobs field.
|
||||
/// </summary>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="virtualCluster">Unused.</param>
|
||||
/// <param name="reportProgress">Used to report progress.</param>
|
||||
protected override void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
|
||||
{
|
||||
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
||||
if (string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
|
||||
|
|
@ -242,32 +251,32 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
string[] files = Directory.GetFiles(joblist, "*.xml");
|
||||
foreach (var file in files)
|
||||
{
|
||||
manager.Token.ThrowIfCancellationRequested();
|
||||
DryadLinqJobSummary job = Utilities.LoadXml<DryadLinqJobSummary>(file);
|
||||
string cjid = job.Cluster + "-" + job.ClusterJobId; // there may be two jobs with same id from different clusters
|
||||
ClusterJobInformation ci = new ClusterJobInformation(this.Config.Name, job.Cluster, cjid, job.Name, job.User, job.Date, job.EndTime - job.Date, job.Status);
|
||||
ci.SetAssociatedSummary(job);
|
||||
if (this.clusterJobs.ContainsKey(cjid))
|
||||
{
|
||||
reporter("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error);
|
||||
manager.Status("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error);
|
||||
continue;
|
||||
}
|
||||
this.clusterJobs.Add(cjid, ci);
|
||||
}
|
||||
reportProgress(100);
|
||||
manager.Progress(100);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Refresh the job summary status.
|
||||
/// </summary>
|
||||
/// <param name="job">Summary to refresh.</param>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="reportProgress">Used to report progres.</param>
|
||||
public override void RefreshStatus(DryadLinqJobSummary job, StatusReporter reporter, Action<int> reportProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public override void RefreshStatus(DryadLinqJobSummary job, CommManager manager)
|
||||
{
|
||||
ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job);
|
||||
ClusterStatus actualStatus = actual.CreateClusterStatus();
|
||||
actualStatus.RefreshStatus(job, reporter, reportProgress);
|
||||
ClusterJobInformation info = actualStatus.DiscoverClusterJob(job, reporter, reportProgress);
|
||||
actualStatus.RefreshStatus(job, manager);
|
||||
ClusterJobInformation info = actualStatus.DiscoverClusterJob(job, manager);
|
||||
if (info == null)
|
||||
{
|
||||
job.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
||||
|
|
@ -303,13 +312,12 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// </summary>
|
||||
/// <param name="job">Cluster job.</param>
|
||||
/// <returns>Throws an exception.</returns>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="reportProgress">Used to report progress.</param>
|
||||
public override ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, StatusReporter reporter, Action<int> reportProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public override ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, CommManager manager)
|
||||
{
|
||||
ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job);
|
||||
ClusterStatus actualStatus = actual.CreateClusterStatus();
|
||||
return actualStatus.DiscoverClusterJob(job, reporter, reportProgress);
|
||||
return actualStatus.DiscoverClusterJob(job, manager);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -358,10 +366,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// Force the recomputation of the cluster job list.
|
||||
/// </summary>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
||||
/// <param name="reportProgress">Used to report progress.</param>
|
||||
protected override void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
|
||||
{
|
||||
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
||||
if (!Directory.Exists(this.config.JobsFolder))
|
||||
|
|
@ -371,6 +378,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
int done = 0;
|
||||
foreach (var job in subfolders)
|
||||
{
|
||||
manager.Token.ThrowIfCancellationRequested();
|
||||
string jobId = Path.GetFileName(job);
|
||||
ClusterJobInformation info = this.GetJobInfo(job, jobId);
|
||||
if (info != null)
|
||||
|
|
@ -378,9 +386,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
// ReSharper disable once AssignNullToNotNullAttribute
|
||||
this.clusterJobs.Add(jobId, info);
|
||||
}
|
||||
reportProgress(done++ *100/subfolders.Length);
|
||||
manager.Progress(done++ *100/subfolders.Length);
|
||||
}
|
||||
reportProgress(100);
|
||||
manager.Progress(100);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -449,20 +457,17 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// Status of an Azure DFS cluster.
|
||||
/// </summary>
|
||||
public class AzureDfsClusterStatus : ClusterStatus
|
||||
public abstract class DfsClusterStatus : ClusterStatus
|
||||
{
|
||||
private AzureDfsClusterConfiguration config;
|
||||
|
||||
/// <summary>
|
||||
/// Create a cluster containing just the local machine.
|
||||
/// </summary>
|
||||
/// <param name="config">Configuration for the local machine.</param>
|
||||
public AzureDfsClusterStatus(ClusterConfiguration config)
|
||||
protected DfsClusterStatus(ClusterConfiguration config)
|
||||
: base(config)
|
||||
{
|
||||
if (!(config is AzureDfsClusterConfiguration))
|
||||
throw new ArgumentException("Expected a AzureYarnClusterConfiguration, got a " + config.GetType());
|
||||
this.config = config as AzureDfsClusterConfiguration;
|
||||
if (!(config is DfsClusterConfiguration))
|
||||
throw new ArgumentException("Expected a DfsClusterConfiguration, got a " + config.GetType());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -475,31 +480,6 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Force the recomputation of the cluster job list.
|
||||
/// </summary>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
||||
/// <param name="reportProgress">Used to report progress.</param>
|
||||
protected override void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
|
||||
{
|
||||
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
||||
var jobs = this.config.AzureClient.EnumerateDirectory("").ToList();
|
||||
|
||||
int done = 0;
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
ClusterJobInformation info = this.GetJobInfo(job);
|
||||
if (info != null)
|
||||
{
|
||||
// ReSharper disable once AssignNullToNotNullAttribute
|
||||
this.clusterJobs.Add(job, info);
|
||||
}
|
||||
reportProgress(100*done++/jobs.Count);
|
||||
}
|
||||
reportProgress(100);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Discover the (unique) dryadlinq job corresponding to a cluster job.
|
||||
/// </summary>
|
||||
|
|
@ -510,7 +490,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
DryadLinqJobSummary result = new DryadLinqJobSummary(
|
||||
clusterJob.Cluster,
|
||||
this.config.TypeOfCluster,
|
||||
this.Config.TypeOfCluster,
|
||||
"", // virtual cluster
|
||||
"", // machine
|
||||
clusterJob.ClusterJobID, // jobId
|
||||
|
|
@ -536,6 +516,78 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
throw new InvalidOperationException();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cancel the specified job.
|
||||
/// </summary>
|
||||
/// <param name="job">Job whose execution is cancelled.</param>
|
||||
/// <returns>True if the cancellation succeeded.</returns>
|
||||
public override bool CancelJob(DryadLinqJobSummary job)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Status of an Azure DFS cluster.
|
||||
/// </summary>
|
||||
public class AzureDfsClusterStatus : DfsClusterStatus
|
||||
{
|
||||
private AzureDfsClusterConfiguration config;
|
||||
|
||||
/// <summary>
|
||||
/// Create a cluster containing just the local machine.
|
||||
/// </summary>
|
||||
/// <param name="config">Configuration for the local machine.</param>
|
||||
public AzureDfsClusterStatus(ClusterConfiguration config)
|
||||
: base(config)
|
||||
{
|
||||
if (!(config is AzureDfsClusterConfiguration))
|
||||
throw new ArgumentException("Expected a AzureDfsClusterConfiguration, got a " + config.GetType());
|
||||
this.config = config as AzureDfsClusterConfiguration;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Force the recomputation of the cluster job list.
|
||||
/// </summary>
|
||||
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
|
||||
{
|
||||
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
||||
var jobs = this.config.AzureClient.ExpandFileOrDirectory(AzureDfsFile.UriFromPath(this.config, "")).ToList();
|
||||
|
||||
int done = 0;
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
manager.Token.ThrowIfCancellationRequested();
|
||||
string jobRootFolder = AzureDfsFile.PathFromUri(this.config, job);
|
||||
ClusterJobInformation info = this.GetJobInfo(jobRootFolder);
|
||||
if (info != null)
|
||||
{
|
||||
// ReSharper disable once AssignNullToNotNullAttribute
|
||||
this.clusterJobs.Add(job.AbsolutePath, info);
|
||||
}
|
||||
manager.Progress(100*done++/jobs.Count);
|
||||
}
|
||||
manager.Progress(100);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extract blob name from a path.
|
||||
/// </summary>
|
||||
/// <param name="container">Container name.</param>
|
||||
/// <param name="path">Path.</param>
|
||||
/// <returns>The blob part of path.</returns>
|
||||
public static string GetBlobName(string container, string path)
|
||||
{
|
||||
if (path.StartsWith("/" + container + "/"))
|
||||
path = path.Substring(container.Length + 2);
|
||||
int q = path.IndexOf('?');
|
||||
if (q > 0)
|
||||
path = path.Substring(0, q);
|
||||
return path;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extract the job information from a folder with logs on the local machine.
|
||||
/// </summary>
|
||||
|
|
@ -548,24 +600,18 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
||||
bool found = false;
|
||||
|
||||
var jobsFolders = this.config.AzureClient.EnumerateDirectory(jobRootFolder).ToList();
|
||||
Uri uri = AzureDfsFile.UriFromPath(this.config, jobRootFolder);
|
||||
var jobsFolders = this.config.AzureClient.ExpandFileOrDirectory(uri).ToList();
|
||||
|
||||
jobRootFolder = GetBlobName(this.config.Container, jobRootFolder);
|
||||
string jobName = jobRootFolder;
|
||||
|
||||
foreach (var file in jobsFolders)
|
||||
{
|
||||
if (file.Contains("DryadLinqProgram__"))
|
||||
if (file.AbsolutePath.EndsWith("heartbeat"))
|
||||
{
|
||||
var blob = this.config.AzureClient.Container.GetBlockBlobReference(file);
|
||||
blob.FetchAttributes();
|
||||
var props = blob.Properties;
|
||||
if (props.LastModified.HasValue)
|
||||
{
|
||||
date = props.LastModified.Value.DateTime;
|
||||
date = date.ToLocalTime();
|
||||
}
|
||||
found = true;
|
||||
}
|
||||
else if (file.EndsWith("heartbeat"))
|
||||
{
|
||||
var blob = this.config.AzureClient.Container.GetPageBlobReference(file);
|
||||
string blobName = GetBlobName(this.config.Container, file.AbsolutePath);
|
||||
var blob = this.config.AzureClient.Container.GetPageBlobReference(blobName);
|
||||
blob.FetchAttributes();
|
||||
var props = blob.Metadata;
|
||||
if (props.ContainsKey("status"))
|
||||
|
|
@ -582,6 +628,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
case "running":
|
||||
status = ClusterJobInformation.ClusterJobStatus.Running;
|
||||
break;
|
||||
case "killed":
|
||||
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
|
||||
break;
|
||||
default:
|
||||
Console.WriteLine("Unknown status " + st);
|
||||
break;
|
||||
|
|
@ -599,6 +648,31 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
status = ClusterJobInformation.ClusterJobStatus.Failed;
|
||||
}
|
||||
}
|
||||
if (props.ContainsKey("jobname"))
|
||||
{
|
||||
jobName = props["jobname"];
|
||||
}
|
||||
if (props.ContainsKey("starttime"))
|
||||
{
|
||||
var t = props["starttime"];
|
||||
if (DateTime.TryParse(t, out date))
|
||||
date = date.ToLocalTime();
|
||||
}
|
||||
|
||||
found = true;
|
||||
}
|
||||
else if (file.AbsolutePath.Contains("DryadLinqProgram__") &&
|
||||
// newer heartbeats contain the date
|
||||
date != DateTime.MinValue)
|
||||
{
|
||||
var blob = this.config.AzureClient.Container.GetBlockBlobReference(AzureDfsFile.PathFromUri(this.config, file));
|
||||
blob.FetchAttributes();
|
||||
var props = blob.Properties;
|
||||
if (props.LastModified.HasValue)
|
||||
{
|
||||
date = props.LastModified.Value.DateTime;
|
||||
date = date.ToLocalTime();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -607,7 +681,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
TimeSpan running = TimeSpan.Zero;
|
||||
if (date != DateTime.MinValue && lastHeartBeat != DateTime.MinValue)
|
||||
running = lastHeartBeat - date;
|
||||
var info = new ClusterJobInformation(this.config.Name, "", jobRootFolder, jobRootFolder, Environment.UserName, date, running, status);
|
||||
var info = new ClusterJobInformation(this.config.Name, "", jobRootFolder, jobName, Environment.UserName, date, running, status);
|
||||
return info;
|
||||
}
|
||||
|
||||
|
|
@ -615,11 +689,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// Refresh the job summary status.
|
||||
/// </summary>
|
||||
/// <param name="summary">Summary to refresh.</param>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="reportProgress">Used to report progress.</param>
|
||||
public override void RefreshStatus(DryadLinqJobSummary summary, StatusReporter reporter, Action<int> reportProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public override void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
|
||||
{
|
||||
// refresh the whole list
|
||||
ClusterJobInformation info = this.GetJobInfo(summary.JobID);
|
||||
if (info == null)
|
||||
{
|
||||
|
|
@ -636,7 +708,224 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <returns>True if the cancellation succeeded.</returns>
|
||||
public override bool CancelJob(DryadLinqJobSummary job)
|
||||
{
|
||||
throw new InvalidOperationException();
|
||||
Microsoft.Research.Peloponnese.Azure.Utils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cluster status of a WebHdfs cluster.
|
||||
/// </summary>
|
||||
public class WebHdfsClusterStatus : DfsClusterStatus
|
||||
{
|
||||
private WebHdfsClusterConfiguration config;
|
||||
/// <summary>
|
||||
/// Yarn client to access job status.
|
||||
/// </summary>
|
||||
private NativeYarnClient yarnClient;
|
||||
|
||||
/// <summary>
|
||||
/// Create a cluster containing just the local machine.
|
||||
/// </summary>
|
||||
/// <param name="conf">Configuration for the local machine.</param>
|
||||
public WebHdfsClusterStatus(ClusterConfiguration conf)
|
||||
: base(conf)
|
||||
{
|
||||
if (!(conf is WebHdfsClusterConfiguration))
|
||||
throw new ArgumentException("Expected a WebHdfsClusterConfiguration, got a " + conf.GetType());
|
||||
this.config = conf as WebHdfsClusterConfiguration;
|
||||
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extract the job information from a folder with logs on the local machine.
|
||||
/// </summary>
|
||||
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
|
||||
/// <returns>The job information, or null if not found.</returns>
|
||||
private ClusterJobInformation GetJobInfo(string jobRootFolder)
|
||||
{
|
||||
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
|
||||
long time;
|
||||
long size;
|
||||
this.config.DfsClient.GetFileStatus(uri, out time, out size);
|
||||
|
||||
DateTime date = DfsFile.TimeFromLong(time);
|
||||
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
||||
string jobName = Path.GetFileName(jobRootFolder);
|
||||
|
||||
string errorMsg = "";
|
||||
|
||||
try
|
||||
{
|
||||
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
|
||||
var jobstatus = jobinfo.GetStatus();
|
||||
errorMsg = jobinfo.ErrorMsg;
|
||||
switch (jobstatus)
|
||||
{
|
||||
case JobStatus.NotSubmitted:
|
||||
case JobStatus.Waiting:
|
||||
status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
||||
break;
|
||||
case JobStatus.Running:
|
||||
status = ClusterJobInformation.ClusterJobStatus.Running;
|
||||
break;
|
||||
case JobStatus.Success:
|
||||
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
|
||||
break;
|
||||
case JobStatus.Cancelled:
|
||||
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
|
||||
break;
|
||||
case JobStatus.Failure:
|
||||
status = ClusterJobInformation.ClusterJobStatus.Failed;
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException();
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
}
|
||||
|
||||
TimeSpan running = TimeSpan.Zero;
|
||||
var info = new ClusterJobInformation(this.config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
|
||||
return info;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Force the recomputation of the cluster job list.
|
||||
/// </summary>
|
||||
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
// ReSharper disable once UnusedParameter.Global
|
||||
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
|
||||
{
|
||||
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
||||
var uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
|
||||
var jobsEnum = this.config.DfsClient.EnumerateSubdirectories(uri);
|
||||
List<Uri> jobs = jobsEnum != null ? jobsEnum.ToList() : new List<Uri>();
|
||||
|
||||
int done = 0;
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
manager.Token.ThrowIfCancellationRequested();
|
||||
ClusterJobInformation info = this.GetJobInfo(DfsFile.PathFromUri(this.config.JobsFolderUri, job));
|
||||
if (info != null)
|
||||
{
|
||||
// ReSharper disable once AssignNullToNotNullAttribute
|
||||
this.clusterJobs.Add(info.ClusterJobID, info);
|
||||
}
|
||||
manager.Progress(100 * done++ / jobs.Count);
|
||||
}
|
||||
manager.Progress(100);
|
||||
}
|
||||
}
|
||||
/// <summary>
|
||||
/// Cluster status of a WebHdfs cluster.
|
||||
/// </summary>
|
||||
public class HdfsClusterStatus : DfsClusterStatus
|
||||
{
|
||||
private HdfsClusterConfiguration config;
|
||||
/// <summary>
|
||||
/// Yarn client to access job status.
|
||||
/// </summary>
|
||||
private NativeYarnClient yarnClient;
|
||||
|
||||
/// <summary>
|
||||
/// Create a cluster containing just the local machine.
|
||||
/// </summary>
|
||||
/// <param name="conf">Configuration for the local machine.</param>
|
||||
public HdfsClusterStatus(ClusterConfiguration conf)
|
||||
: base(conf)
|
||||
{
|
||||
if (!(conf is HdfsClusterConfiguration))
|
||||
throw new ArgumentException("Expected an HdfsClusterConfiguration, got a " + conf.GetType());
|
||||
this.config = conf as HdfsClusterConfiguration;
|
||||
// make a fake call to initialize the cluster on the foreground thread
|
||||
// HDFS does not work if initialized on the background thread.
|
||||
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
|
||||
this.config.DfsClient.IsFileExists(uri); // ignore result
|
||||
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extract the job information from a folder with logs on the local machine.
|
||||
/// </summary>
|
||||
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
|
||||
/// <returns>The job information, or null if not found.</returns>
|
||||
private ClusterJobInformation GetJobInfo(string jobRootFolder)
|
||||
{
|
||||
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
|
||||
long time;
|
||||
long size;
|
||||
this.config.DfsClient.GetFileStatus(uri, out time, out size);
|
||||
|
||||
DateTime date = DfsFile.TimeFromLong(time);
|
||||
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
||||
string jobName = Path.GetFileName(jobRootFolder);
|
||||
|
||||
string errorMsg = "";
|
||||
|
||||
try
|
||||
{
|
||||
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
|
||||
var jobstatus = jobinfo.GetStatus();
|
||||
errorMsg = jobinfo.ErrorMsg;
|
||||
switch (jobstatus)
|
||||
{
|
||||
case JobStatus.NotSubmitted:
|
||||
case JobStatus.Waiting:
|
||||
status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
||||
break;
|
||||
case JobStatus.Running:
|
||||
status = ClusterJobInformation.ClusterJobStatus.Running;
|
||||
break;
|
||||
case JobStatus.Success:
|
||||
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
|
||||
break;
|
||||
case JobStatus.Cancelled:
|
||||
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
|
||||
break;
|
||||
case JobStatus.Failure:
|
||||
status = ClusterJobInformation.ClusterJobStatus.Failed;
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException();
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
}
|
||||
|
||||
TimeSpan running = TimeSpan.Zero;
|
||||
var info = new ClusterJobInformation(config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
|
||||
return info;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Force the recomputation of the cluster job list.
|
||||
/// </summary>
|
||||
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
// ReSharper disable once UnusedParameter.Global
|
||||
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
|
||||
{
|
||||
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
||||
var uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
|
||||
var jobs = this.config.DfsClient.EnumerateSubdirectories(uri).ToList();
|
||||
|
||||
int done = 0;
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
manager.Token.ThrowIfCancellationRequested();
|
||||
ClusterJobInformation info = this.GetJobInfo(DfsFile.PathFromUri(this.config.JobsFolderUri, job));
|
||||
if (info != null)
|
||||
{
|
||||
// ReSharper disable once AssignNullToNotNullAttribute
|
||||
this.clusterJobs.Add(info.ClusterJobID, info);
|
||||
}
|
||||
manager.Progress(100 * done++ / jobs.Count);
|
||||
}
|
||||
manager.Progress(100);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.6-alpha042\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.6-alpha042\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
|
|
@ -9,61 +15,17 @@
|
|||
<ProjectGuid>{77739535-7FAC-4487-887F-FEBA197E7572}</ProjectGuid>
|
||||
<OutputType>Library</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>DistributedDataCollection</RootNamespace>
|
||||
<AssemblyName>Microsoft.Research.DryadLinq.JobBrowser.DistributedDataCollection</AssemblyName>
|
||||
<RootNamespace>JobObjectModel</RootNamespace>
|
||||
<AssemblyName>JobObjectModel</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<StartupObject>
|
||||
</StartupObject>
|
||||
<FileUpgradeFlags>
|
||||
</FileUpgradeFlags>
|
||||
<UpgradeBackupLocation>
|
||||
</UpgradeBackupLocation>
|
||||
<OldToolsVersion>3.5</OldToolsVersion>
|
||||
<PublishUrl>publish\</PublishUrl>
|
||||
<Install>true</Install>
|
||||
<InstallFrom>Disk</InstallFrom>
|
||||
<UpdateEnabled>false</UpdateEnabled>
|
||||
<UpdateMode>Foreground</UpdateMode>
|
||||
<UpdateInterval>7</UpdateInterval>
|
||||
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
|
||||
<UpdatePeriodically>false</UpdatePeriodically>
|
||||
<UpdateRequired>false</UpdateRequired>
|
||||
<MapFileExtensions>true</MapFileExtensions>
|
||||
<ApplicationRevision>0</ApplicationRevision>
|
||||
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
|
||||
<IsWebBootstrapper>false</IsWebBootstrapper>
|
||||
<UseApplicationTrust>false</UseApplicationTrust>
|
||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||
<TargetFrameworkProfile />
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<DocumentationFile>bin\Debug\Microsoft.Research.DryadLinq.JobBrowser.DistributedDataCollection.xml</DocumentationFile>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<Prefer32Bit>false</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<Prefer32Bit>false</Prefer32Bit>
|
||||
<NuGetPackageImportStamp>b735dff6</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<DocumentationFile>bin\Debug\DistributedDataCollection.XML</DocumentationFile>
|
||||
<DocumentationFile>bin\Debug\JobObjectModel.XML</DocumentationFile>
|
||||
<DebugType>full</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
|
|
@ -78,83 +40,13 @@
|
|||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hpc.Scheduler, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\Dependencies\Microsoft.Hpc.Scheduler.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hpc.Scheduler.Properties, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\Dependencies\Microsoft.Hpc.Scheduler.Properties.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core">
|
||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data.Services.Client" />
|
||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq">
|
||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data.DataSetExtensions">
|
||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="ClusterAccess.cs" />
|
||||
<Compile Include="ClusterConfiguration.cs" />
|
||||
<Compile Include="ClusterStatus.cs" />
|
||||
<Compile Include="Cosmos.cs" />
|
||||
<Compile Include="dryadlog.cs" />
|
||||
<Compile Include="jobinfo.cs" />
|
||||
<Compile Include="JobSummary.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
<Compile Include="Storage.cs" />
|
||||
</ItemGroup>
|
||||
|
|
@ -165,36 +57,121 @@
|
|||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
|
||||
<Install>false</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.5 SP1</ProductName>
|
||||
<Install>true</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>Windows Installer 3.1</ProductName>
|
||||
<Install>true</Install>
|
||||
</BootstrapperPackage>
|
||||
<None Include="app.config" />
|
||||
<None Include="packages.config">
|
||||
<SubType>Designer</SubType>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="app.config" />
|
||||
<None Include="packages.config" />
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.HDInsight.Net.Http.Formatting, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.HDInsight.Net.Http.Formatting.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks">
|
||||
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks.Extensions">
|
||||
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
|
||||
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Common">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.3.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System">
|
||||
<HintPath>..\..\..\..\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5\System.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.Extensions">
|
||||
<HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Net.Http.Primitives">
|
||||
<HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.XML" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.0-beta023\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.0-beta023\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,488 @@
|
|||
|
||||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using Microsoft.Research.Tools;
|
||||
|
||||
namespace Microsoft.Research.JobObjectModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Error during conversation with cluster.
|
||||
/// </summary>
|
||||
public sealed class ClusterException : Exception
|
||||
{
|
||||
/// <summary>
|
||||
/// Create an exception about handling a cluster.
|
||||
/// </summary>
|
||||
/// <param name="message">Exception message.</param>
|
||||
public ClusterException(string message) : base(message) { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Identifier for a Dryad process; for now we are using globally-unique process GUID.
|
||||
/// A Dryad Job identifier is always the same as the job manager process guid.
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public class DryadProcessIdentifier : IEquatable<DryadProcessIdentifier>
|
||||
{
|
||||
/// <summary>
|
||||
/// Process identifier.
|
||||
/// </summary>
|
||||
private string processIdentifier;
|
||||
|
||||
/// <summary>
|
||||
/// Used just for XML serialization.
|
||||
/// </summary>
|
||||
public DryadProcessIdentifier()
|
||||
{ }
|
||||
|
||||
/// <summary>
|
||||
/// create an indentifier struct.
|
||||
/// Be sure to initialize all fields before use;
|
||||
/// </summary>
|
||||
/// <param name="pid">The id of the process (platform-dependent).</param>
|
||||
public DryadProcessIdentifier(string pid)
|
||||
{
|
||||
this.processIdentifier = pid;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Process identifier; public for serialization only.
|
||||
/// </summary>
|
||||
public string ProcessIdentifier
|
||||
{
|
||||
get { return this.processIdentifier; }
|
||||
set { this.processIdentifier = value; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// If true the process identifier is not known.
|
||||
/// </summary>
|
||||
public bool IsUnknown { get { return this.ProcessIdentifier == null; } }
|
||||
|
||||
/// <summary>
|
||||
/// Human-readable description of the process identifier.
|
||||
/// </summary>
|
||||
/// <returns>An empty string if the pid is not set.</returns>
|
||||
public override string ToString()
|
||||
{
|
||||
if (this.ProcessIdentifier != null) return this.ProcessIdentifier;
|
||||
return "";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Equality test.
|
||||
/// </summary>
|
||||
/// <param name="obj">Object to compare to.</param>
|
||||
/// <returns>True if both objects represent the same process id.</returns>
|
||||
public override bool Equals(object obj)
|
||||
{
|
||||
if (!(obj is DryadProcessIdentifier))
|
||||
return false;
|
||||
return this.Equals((DryadProcessIdentifier)obj);
|
||||
}
|
||||
|
||||
#region IEquatable<DryadProcessIdentifier> Members
|
||||
/// <summary>
|
||||
/// Equality test.
|
||||
/// </summary>
|
||||
/// <param name="other">Process id to compare to.</param>
|
||||
/// <returns>True if the id's represent the same process.</returns>
|
||||
public bool Equals(DryadProcessIdentifier other)
|
||||
{
|
||||
if (this.IsUnknown)
|
||||
return other.IsUnknown;
|
||||
if (other.IsUnknown)
|
||||
return false;
|
||||
return this.ProcessIdentifier.Equals(other.ProcessIdentifier);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Overriden implementation of getHashCode.
|
||||
/// </summary>
|
||||
/// <returns>The hashcode of the process id.</returns>
|
||||
public override int GetHashCode()
|
||||
{
|
||||
// ReSharper disable once BaseObjectGetHashCodeCallInGetHashCode
|
||||
return base.GetHashCode();
|
||||
}
|
||||
#endregion
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Brief summary of an executed DryadLINQ job.
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public sealed class DryadLinqJobSummary : IEquatable<DryadLinqJobSummary>
|
||||
{
|
||||
/// <summary>
|
||||
/// Empty constructor for XML serialization.
|
||||
/// </summary>
|
||||
public DryadLinqJobSummary()
|
||||
{ }
|
||||
|
||||
/// <summary>
|
||||
/// Initialize a job summary.
|
||||
/// </summary>
|
||||
/// <param name="cluster">Cluster where the job ran.</param>
|
||||
/// <param name="clusterType">A string corresponding to the type of ClusterConfiguration.</param>
|
||||
/// <param name="machine">Machine where job manager ran.</param>
|
||||
/// <param name="jobId">Id of job.</param>
|
||||
/// <param name="jmProcessGuid">Guid of job manager process.</param>
|
||||
/// <param name="clusterJobId">Id of job on the cluster.</param>
|
||||
/// <param name="friendlyname">Friendly name used.</param>
|
||||
/// <param name="username">Who ran the job.</param>
|
||||
/// <param name="date">Start date (not completion date).</param>
|
||||
/// <param name="status">Job status.</param>
|
||||
/// <param name="endTime">Estimated end running time.</param>
|
||||
/// <param name="virtualcluster">Virtual cluster where job ran.</param>
|
||||
public DryadLinqJobSummary(
|
||||
string cluster,
|
||||
ClusterConfiguration.ClusterType clusterType,
|
||||
string virtualcluster,
|
||||
string machine,
|
||||
string jobId,
|
||||
string clusterJobId,
|
||||
DryadProcessIdentifier jmProcessGuid,
|
||||
string friendlyname,
|
||||
string username,
|
||||
DateTime date,
|
||||
DateTime endTime,
|
||||
ClusterJobInformation.ClusterJobStatus status)
|
||||
{
|
||||
this.VirtualCluster = virtualcluster;
|
||||
this.Cluster = cluster;
|
||||
this.ClusterType = clusterType;
|
||||
this.Machine = machine;
|
||||
this.Name = friendlyname;
|
||||
this.User = username;
|
||||
this.Date = date;
|
||||
this.EndTime = endTime;
|
||||
this.Status = status;
|
||||
this.ManagerProcessGuid = jmProcessGuid;
|
||||
this.JobID = jobId;
|
||||
this.ClusterJobId = clusterJobId;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cluster where the job ran.
|
||||
/// </summary>
|
||||
public string Cluster { get; /*private*/ set; }
|
||||
/// <summary>
|
||||
/// Id of cluster job that originated this DryadLinq job (can be used to find the cluster job from the dryadlinq job).
|
||||
/// </summary>
|
||||
public string ClusterJobId { get; /*private*/ set; }
|
||||
/// <summary>
|
||||
/// Cluster where the job ran.
|
||||
/// </summary>
|
||||
public DateTime EndTime { get; /*private*/ set; }
|
||||
/// <summary>
|
||||
/// String describing cluster type.
|
||||
/// </summary>
|
||||
public ClusterConfiguration.ClusterType ClusterType { get; /*private*/ set; }
|
||||
/// <summary>
|
||||
/// Virtual cluster where job ran.
|
||||
/// </summary>
|
||||
public string VirtualCluster { get; /*private*/ set; }
|
||||
/// <summary>
|
||||
/// (Friendly) name of the job.
|
||||
/// </summary>
|
||||
public string Name { get; /*private*/ set; }
|
||||
/// <summary>
|
||||
/// User who submitted job.
|
||||
/// </summary>
|
||||
public string User { get; /*private*/ set; }
|
||||
/// <summary>
|
||||
/// ID of job on the cluster.
|
||||
/// </summary>
|
||||
public string JobID { get; /*private*/ set; }
|
||||
/// <summary>
|
||||
/// The Guid of the job manager process.
|
||||
/// </summary>
|
||||
public DryadProcessIdentifier ManagerProcessGuid { set; get; }
|
||||
|
||||
/// <summary>
|
||||
/// User who submitted job.
|
||||
/// </summary>
|
||||
public string GetAlias()
|
||||
{
|
||||
int pos = User.IndexOf(@"\");
|
||||
return User.Substring(pos + 1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Date when job was submitted.
|
||||
/// </summary>
|
||||
public DateTime Date { get; /*private*/ set; }
|
||||
|
||||
/// <summary>
|
||||
/// Did the job fail?
|
||||
/// </summary>
|
||||
public ClusterJobInformation.ClusterJobStatus Status { get; /*internal*/ set; }
|
||||
|
||||
/// <summary>
|
||||
/// Machine where the job manager ran.
|
||||
/// </summary>
|
||||
public string Machine { get; /*private*/ set; }
|
||||
|
||||
/// <summary>
|
||||
/// Get a short name for this job summary.
|
||||
/// </summary>
|
||||
/// <returns>Short name of job summary.</returns>
|
||||
public string ShortName()
|
||||
{
|
||||
// we use the starting time to uniquify the job name
|
||||
return this.Date.ToString("s") + "-" + this.Name;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True if these two summaries are the same. The status and end time do not matter, since the job may still be running.
|
||||
/// </summary>
|
||||
/// <param name="other">Summary to compare against.</param>
|
||||
/// <returns>True if they are equal.</returns>
|
||||
public bool Equals(DryadLinqJobSummary other)
|
||||
{
|
||||
return this.Cluster == other.Cluster &&
|
||||
this.ClusterJobId == other.ClusterJobId &&
|
||||
this.Date == other.Date &&
|
||||
this.Machine == other.Machine &&
|
||||
this.Name == other.Name &&
|
||||
this.User == other.User;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Hashcode proper for the equality test.
|
||||
/// </summary>
|
||||
/// <returns>The object hashcode.</returns>
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return this.ClusterJobId.GetHashCode() ^ this.ClusterJobId.GetHashCode() ^ this.Date.GetHashCode() ^ this.Machine.GetHashCode() ^ this.Name.GetHashCode() ^ this.User.GetHashCode();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A string describing the unique identifying part of the summary.
|
||||
/// Two different summaries may represent the same job at different times.
|
||||
/// </summary>
|
||||
/// <returns>The part common to all jobs.</returns>
|
||||
public string AsIdentifyingString()
|
||||
{
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.AppendFormat("Cluster={0} ClusterJobID={1} Date={2} Machine={3} Name={4} User={5}",
|
||||
this.Cluster, this.ClusterJobId, this.Date, this.Machine, this.Name, this.User);
|
||||
return builder.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// This class is an abstraction of a cluster-level job, as opposed to a DryadLINQ job.
|
||||
/// In Cosmos that's called a task, in HPC that's called a Job.
|
||||
/// (In cosmos a task is a recurring job. In DryadLINQ running on top of cosmos, a task always contains exactly one job.)
|
||||
/// </summary>
|
||||
public class ClusterJobInformation : IEquatable<ClusterJobInformation>
|
||||
{
|
||||
/// <summary>
|
||||
/// Status of a cluster job.
|
||||
/// </summary>
|
||||
public enum ClusterJobStatus
|
||||
{
|
||||
/// <summary>
|
||||
/// Job is still running.
|
||||
/// </summary>
|
||||
Running,
|
||||
/// <summary>
|
||||
/// Job has finished successfully.
|
||||
/// </summary>
|
||||
Succeeded,
|
||||
/// <summary>
|
||||
/// Job has finished and has failed.
|
||||
/// </summary>
|
||||
Failed,
|
||||
/// <summary>
|
||||
/// Job has been cancelled. Not precise on cosmos clusters.
|
||||
/// </summary>
|
||||
Cancelled,
|
||||
/// <summary>
|
||||
/// Could not determine job status.
|
||||
/// </summary>
|
||||
Unknown,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// True if job is finished, false if not, or unknown.
|
||||
/// </summary>
|
||||
/// <param name="status">Job status.</param>
|
||||
/// <returns>True if the job is no longer running.</returns>
|
||||
public static bool JobIsFinished(ClusterJobStatus status)
|
||||
{
|
||||
switch (status)
|
||||
{
|
||||
case ClusterJobInformation.ClusterJobStatus.Failed:
|
||||
case ClusterJobInformation.ClusterJobStatus.Succeeded:
|
||||
case ClusterJobInformation.ClusterJobStatus.Cancelled:
|
||||
return true;
|
||||
case ClusterJobInformation.ClusterJobStatus.Running:
|
||||
case ClusterJobInformation.ClusterJobStatus.Unknown:
|
||||
return false;
|
||||
default:
|
||||
throw new InvalidDataException("Invalid job status " + status);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a cluster job structure from a bunch of information.
|
||||
/// </summary>
|
||||
/// <param name="cluster">Cluster where the job is running.</param>
|
||||
/// <param name="clusterJobGuid">Cluster job guid.</param>
|
||||
/// <param name="jobName">Name of the cluster job.</param>
|
||||
/// <param name="username">User who submitted cluster job.</param>
|
||||
/// <param name="date">Last execution of cluster job.</param>
|
||||
/// <param name="status">Execution status.</param>
|
||||
/// <param name="runningTime">Time the job ran.</param>
|
||||
/// <param name="virtualCluster">Cluster where the job has run.</param>
|
||||
public ClusterJobInformation(
|
||||
string cluster,
|
||||
string virtualCluster,
|
||||
string clusterJobGuid,
|
||||
string jobName,
|
||||
string username,
|
||||
DateTime date,
|
||||
TimeSpan runningTime,
|
||||
ClusterJobStatus status)
|
||||
{
|
||||
this.VirtualCluster = virtualCluster;
|
||||
this.Cluster = cluster;
|
||||
this.ClusterJobID = clusterJobGuid;
|
||||
this.Name = jobName;
|
||||
this.User = username;
|
||||
this.Date = date;
|
||||
this.EstimatedRunningTime = runningTime;
|
||||
this.Status = status;
|
||||
this.JobSummary = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Name of cluster job.
|
||||
/// </summary>
|
||||
public string Name { get; set; }
|
||||
/// <summary>
|
||||
/// Uset who submitted cluster job.
|
||||
/// </summary>
|
||||
public string User { get; set; }
|
||||
/// <summary>
|
||||
/// Date when job was submitted.
|
||||
/// </summary>
|
||||
public DateTime Date { get; set; }
|
||||
/// <summary>
|
||||
/// ID of Job on cluster.
|
||||
/// </summary>
|
||||
public string ClusterJobID { get; set; }
|
||||
/// <summary>
|
||||
/// Status of the execution.
|
||||
/// </summary>
|
||||
public ClusterJobStatus Status { get; set; }
|
||||
/// <summary>
|
||||
/// Cluster where the job ran.
|
||||
/// </summary>
|
||||
public string Cluster { get; set; }
|
||||
/// <summary>
|
||||
/// In some installations a cluster is composed of multiple virtual clusters.
|
||||
/// </summary>
|
||||
public string VirtualCluster { get; set; }
|
||||
/// <summary>
|
||||
/// Is the cluster job information still available on the cluster?
|
||||
/// </summary>
|
||||
public bool IsUnavailable { get; set; }
|
||||
/// <summary>
|
||||
/// Cache here the associated job, if available. Null if not cached.
|
||||
/// </summary>
|
||||
private DryadLinqJobSummary JobSummary { get; set; }
|
||||
/// <summary>
|
||||
/// Estimated time the job ran.
|
||||
/// </summary>
|
||||
public TimeSpan EstimatedRunningTime { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// If known, set the associated job summary.
|
||||
/// </summary>
|
||||
/// <param name="summary">Job summary for this cluster job.</param>
|
||||
public void SetAssociatedSummary(DryadLinqJobSummary summary)
|
||||
{
|
||||
this.JobSummary = summary;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Discover the dryadlinq job associated with a cluster job.
|
||||
/// </summary>
|
||||
/// <param name="status">Cluster configuration.</param>
|
||||
/// <returns>The job, if any</returns>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
public DryadLinqJobSummary DiscoverDryadLinqJob(ClusterStatus status, StatusReporter reporter)
|
||||
{
|
||||
if (this.IsUnavailable)
|
||||
return null;
|
||||
if (this.JobSummary != null)
|
||||
return this.JobSummary;
|
||||
|
||||
DryadLinqJobSummary j = status.DiscoverDryadLinqJobFromClusterJob(this, reporter);
|
||||
if (j == null)
|
||||
{
|
||||
this.IsUnavailable = true;
|
||||
}
|
||||
return this.JobSummary = j;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copy the content of a cluster job.
|
||||
/// </summary>
|
||||
/// <param name="refresh">The value to copy from.</param>
|
||||
internal void Copy(ClusterJobInformation refresh)
|
||||
{
|
||||
this.Name = refresh.Name;
|
||||
this.Status = refresh.Status;
|
||||
this.User = refresh.User;
|
||||
this.JobSummary = refresh.JobSummary;
|
||||
this.ClusterJobID = refresh.ClusterJobID;
|
||||
this.Date = refresh.Date;
|
||||
this.IsUnavailable = refresh.IsUnavailable;
|
||||
this.VirtualCluster = refresh.VirtualCluster;
|
||||
this.EstimatedRunningTime = refresh.EstimatedRunningTime;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True if these two records represent the same job.
|
||||
/// </summary>
|
||||
/// <param name="other"></param>
|
||||
/// <returns></returns>
|
||||
public bool Equals(ClusterJobInformation other)
|
||||
{
|
||||
return
|
||||
this.Cluster == other.Cluster &&
|
||||
this.VirtualCluster == other.VirtualCluster &&
|
||||
this.Name == other.Name &&
|
||||
this.User == other.User &&
|
||||
this.ClusterJobID == other.ClusterJobID &&
|
||||
this.Date == other.Date;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -28,9 +28,9 @@ using System.Runtime.InteropServices;
|
|||
[assembly: AssemblyTitle("DistributedDataCollection")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("Microsoft Corporation")]
|
||||
[assembly: AssemblyCompany("")]
|
||||
[assembly: AssemblyProduct("DistributedDataCollection")]
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
|
||||
[assembly: AssemblyCopyright("Copyright © 2009")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
|
|
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
|
|||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("0.1.0.0")]
|
||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
||||
[assembly: AssemblyVersion("1.0.0.0")]
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||
|
|
|
|||
|
|
@ -2,26 +2,42 @@
|
|||
<configuration>
|
||||
<runtime>
|
||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
@ -19,16 +19,22 @@ limitations under the License.
|
|||
|
||||
*/
|
||||
|
||||
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Research.Calypso.Tools;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using Microsoft.Research.Peloponnese.Azure;
|
||||
using Microsoft.Research.Tools;
|
||||
using Microsoft.WindowsAzure.Storage.Blob;
|
||||
using Microsoft.Research.Peloponnese.Shared;
|
||||
using Microsoft.Research.Peloponnese.WebHdfs;
|
||||
using Microsoft.Research.Peloponnese.Hdfs;
|
||||
|
||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
||||
namespace Microsoft.Research.JobObjectModel
|
||||
{
|
||||
/// <summary>
|
||||
/// A cluster-resident object is a file or a folder.
|
||||
|
|
@ -43,7 +49,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
|
||||
/// </summary>
|
||||
/// <returns>A stream that can be used to access the object contents.</returns>
|
||||
ISharedStreamReader GetStream();
|
||||
/// <param name="keepNewline">If true the streamReader will not strip the newlines.</param>
|
||||
ISharedStreamReader GetStream(bool keepNewline);
|
||||
/// <summary>
|
||||
/// If the current object is a folder, it returns the contained objects.
|
||||
/// </summary>
|
||||
|
|
@ -158,7 +165,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
if (file.Job == null)
|
||||
return;
|
||||
if (string.IsNullOrEmpty(file.LocalCachePath))
|
||||
throw new CalypsoClusterException("Missing expected LocalCachePath");
|
||||
throw new ClusterException("Missing expected LocalCachePath");
|
||||
|
||||
CachedClusterResidentObject.RecordCachedFile(file.Job, file.LocalCachePath);
|
||||
}
|
||||
|
|
@ -203,12 +210,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// A stream to the local cache, or null if the file is not cached.
|
||||
/// </summary>
|
||||
/// <returns>A stream to access the file.</returns>
|
||||
public virtual ISharedStreamReader GetStream()
|
||||
/// <param name="keepNewline">If true keep newlines.</param>
|
||||
public virtual ISharedStreamReader GetStream(bool keepNewline)
|
||||
{
|
||||
if (this.LocalCachePath != null && File.Exists(this.LocalCachePath))
|
||||
{
|
||||
CachedClusterResidentObject.Record(this);
|
||||
return new FileSharedStreamReader(this.LocalCachePath);
|
||||
return new FileSharedStreamReader(this.LocalCachePath, keepNewline);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
|
@ -355,14 +363,15 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// The stream with the file contents.
|
||||
/// </summary>
|
||||
/// <returns>A stream reder.</returns>
|
||||
public override ISharedStreamReader GetStream()
|
||||
/// <param name="keepNewlines">If true keep the newlines.</param>
|
||||
public override ISharedStreamReader GetStream(bool keepNewlines)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!this.RepresentsAFolder)
|
||||
{
|
||||
//this.LocalCachePath = this.CachePath(this.Pathname);
|
||||
ISharedStreamReader baseStream = base.GetStream();
|
||||
ISharedStreamReader baseStream = base.GetStream(keepNewlines);
|
||||
if (baseStream != null)
|
||||
{
|
||||
// file is cached
|
||||
|
|
@ -375,15 +384,15 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
// cache it
|
||||
if (this.RepresentsAFolder)
|
||||
throw new CalypsoClusterException("Cannot cache folders");
|
||||
throw new ClusterException("Cannot cache folders");
|
||||
|
||||
StreamWriter writer = this.CreateTempStream();
|
||||
return new FileSharedStreamReader(this.Pathname.ToString(), writer, this.OnClose);
|
||||
return new FileSharedStreamReader(this.Pathname.ToString(), writer, keepNewlines, this.OnClose);
|
||||
}
|
||||
else
|
||||
{
|
||||
// dont cache it
|
||||
return new FileSharedStreamReader(this.Pathname.ToString());
|
||||
return new FileSharedStreamReader(this.Pathname.ToString(), keepNewlines);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
|
@ -504,7 +513,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
get
|
||||
{
|
||||
if (this.RepresentsAFolder)
|
||||
throw new CalypsoClusterException("Cannot get size of a folder");
|
||||
throw new ClusterException("Cannot get size of a folder");
|
||||
if (File.Exists(this.LocalCachePath))
|
||||
{
|
||||
FileInfo info = new FileInfo(this.LocalCachePath);
|
||||
|
|
@ -650,9 +659,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// The contents of the folder.
|
||||
/// </summary>
|
||||
/// <returns>The contents of the folder.</returns>
|
||||
public override ISharedStreamReader GetStream()
|
||||
/// <param name="keepNewline">If true keep newlines.</param>
|
||||
public override ISharedStreamReader GetStream(bool keepNewline)
|
||||
{
|
||||
return this.OriginalFolder.GetStream();
|
||||
return this.OriginalFolder.GetStream(keepNewline);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -720,9 +730,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// A stream returning the contents.
|
||||
/// </summary>
|
||||
/// <returns>The contents of this object.</returns>
|
||||
public ISharedStreamReader GetStream()
|
||||
/// <param name="keepNewlines">If true keep newlines.</param>
|
||||
public ISharedStreamReader GetStream(bool keepNewlines)
|
||||
{
|
||||
return new StringIteratorStreamReader(this.contents);
|
||||
return new StringIteratorStreamReader(this.contents, keepNewlines);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -732,7 +743,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <returns>Throws an exception.</returns>
|
||||
public IEnumerable<IClusterResidentObject> GetFilesAndFolders(string match)
|
||||
{
|
||||
throw new CalypsoClusterException("Object is not a folder");
|
||||
throw new ClusterException("Object is not a folder");
|
||||
}
|
||||
|
||||
private long size;
|
||||
|
|
@ -787,7 +798,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <returns>Throws an exception.</returns>
|
||||
public IClusterResidentObject GetFile(string filename)
|
||||
{
|
||||
throw new CalypsoClusterException("Object is not a folder");
|
||||
throw new ClusterException("Object is not a folder");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -797,7 +808,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <returns>Throws an exception.</returns>
|
||||
public IClusterResidentObject GetFolder(string foldername)
|
||||
{
|
||||
throw new CalypsoClusterException("Object is not a folder");
|
||||
throw new ClusterException("Object is not a folder");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -835,9 +846,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
|
||||
/// </summary>
|
||||
/// <returns>A stream that can be used to access the object contents.</returns>
|
||||
public ISharedStreamReader GetStream()
|
||||
/// <param name="keepNewlines">If true keep the newlines.</param>
|
||||
public ISharedStreamReader GetStream(bool keepNewlines)
|
||||
{
|
||||
return new FileSharedStreamReader(this.path);
|
||||
return new FileSharedStreamReader(this.path, keepNewlines);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -969,8 +981,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
this.RepresentsAFolder = isFolder;
|
||||
this.size = -1;
|
||||
|
||||
if (!string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
|
||||
this.LocalCachePath = Path.Combine(CachedClusterResidentObject.CacheDirectory, this.path);
|
||||
if (!String.IsNullOrEmpty(CacheDirectory))
|
||||
this.LocalCachePath = Path.Combine(CacheDirectory, this.path);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -982,14 +994,46 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
protected set;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a File URI from a file path.
|
||||
/// </summary>
|
||||
/// <param name="path">Path to file.</param>
|
||||
/// <returns>The File uri.</returns>
|
||||
/// <param name="config">Azure cluster configuration.</param>
|
||||
public static Uri UriFromPath(AzureDfsClusterConfiguration config, string path)
|
||||
{
|
||||
if (path.StartsWith(config.Container))
|
||||
throw new InvalidOperationException("Path contains container name");
|
||||
var retval = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(config.AccountName, config.Container, path, null, config.AccountKey);
|
||||
//Console.WriteLine("Uri {0}", retval);
|
||||
return retval;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a path from a URI.
|
||||
/// </summary>
|
||||
/// <param name="uri">URI of an HDFS file.</param>
|
||||
/// <returns>The path to the file.</returns>
|
||||
/// <param name="config">Azure cluster configuration.</param>
|
||||
public static string PathFromUri(AzureDfsClusterConfiguration config, Uri uri)
|
||||
{
|
||||
string path = uri.LocalPath;
|
||||
path = path.Trim('/');
|
||||
if (path.StartsWith(config.Container))
|
||||
path = path.Substring(config.Container.Length);
|
||||
path = path.Trim('/');
|
||||
return path;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
|
||||
/// </summary>
|
||||
/// <returns>A stream that can be used to access the object contents.</returns>
|
||||
public override ISharedStreamReader GetStream()
|
||||
/// <param name="keepNewlines">If true keep the newlines.</param>
|
||||
public override ISharedStreamReader GetStream(bool keepNewlines)
|
||||
{
|
||||
|
||||
ISharedStreamReader baseStream = base.GetStream();
|
||||
ISharedStreamReader baseStream = base.GetStream(keepNewlines);
|
||||
if (baseStream != null)
|
||||
{
|
||||
// file is cached
|
||||
|
|
@ -1000,31 +1044,43 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
Stream stream;
|
||||
if (this.IsDfsStream)
|
||||
{
|
||||
var dfsFileStream = this.client.GetDfsFileStream(this.path);
|
||||
stream = dfsFileStream.Stream;
|
||||
Uri uri = UriFromPath(this.Config as AzureDfsClusterConfiguration, this.path);
|
||||
stream = this.client.GetDfsStreamReader(uri);
|
||||
}
|
||||
else
|
||||
{
|
||||
string p = AzureDfsClusterStatus.GetBlobName(this.client.ContainerName,this.path);
|
||||
stream = new AzureLogReaderStream(
|
||||
this.client.AccountName,
|
||||
this.client.AccountKey,
|
||||
this.client.ContainerName,
|
||||
this.path);
|
||||
p);
|
||||
}
|
||||
StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, false, 1024 * 1024);
|
||||
|
||||
long sz = this.Size;
|
||||
int bufferSize = 1024*1024;
|
||||
if (sz >= 0)
|
||||
{
|
||||
bufferSize = (int)(sz/10);
|
||||
if (bufferSize < 1024*1024)
|
||||
bufferSize = 1024*1024;
|
||||
if (bufferSize > 20*1024*1024)
|
||||
bufferSize = 20*1024*1024;
|
||||
}
|
||||
SimpleStreamReader reader = new SimpleStreamReader(stream, true, Encoding.UTF8, false, bufferSize);
|
||||
|
||||
if (this.ShouldCacheLocally && this.LocalCachePath != null)
|
||||
{
|
||||
// cache it
|
||||
if (this.RepresentsAFolder)
|
||||
throw new CalypsoClusterException("Cannot cache folders");
|
||||
throw new ClusterException("Cannot cache folders");
|
||||
StreamWriter writer = this.CreateTempStream();
|
||||
return new SharedStreamReader(reader, writer, this.OnClose);
|
||||
return new SharedStreamReader(reader, writer, keepNewlines, this.OnClose);
|
||||
}
|
||||
else
|
||||
{
|
||||
// dont cache it
|
||||
return new SharedStreamReader(reader);
|
||||
return new SharedStreamReader(reader, keepNewlines);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1049,7 +1105,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
if (item is CloudBlockBlob)
|
||||
{
|
||||
CloudBlockBlob blob = (CloudBlockBlob)item;
|
||||
blocks.Add(blob.Name, blob.Properties.Length);
|
||||
this.blocks.Add(blob.Name, blob.Properties.Length);
|
||||
}
|
||||
else if (item is CloudPageBlob)
|
||||
{
|
||||
|
|
@ -1061,8 +1117,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
if (metadata.ContainsKey("writePosition"))
|
||||
{
|
||||
long sz;
|
||||
if (long.TryParse(metadata["writePosition"], out sz))
|
||||
pages.Add(pageBlob.Name, sz);
|
||||
if (Int64.TryParse(metadata["writePosition"], out sz))
|
||||
this.pages.Add(pageBlob.Name, sz);
|
||||
}
|
||||
}
|
||||
else if (item is CloudBlobDirectory)
|
||||
|
|
@ -1083,29 +1139,28 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
this.PopulateCache();
|
||||
long length = -1;
|
||||
|
||||
foreach (var child in this.client.EnumerateDirectory(this.path))
|
||||
Uri uri = UriFromPath(this.Config as AzureDfsClusterConfiguration, this.path);
|
||||
Console.WriteLine("AzureDfsFile.GetFileAndFolders({0}) -> {1}", this.path, uri);
|
||||
foreach (var child in this.client.ExpandFileOrDirectory(uri))
|
||||
{
|
||||
Regex re = Utilities.RegexFromSearchPattern(match);
|
||||
if (!re.IsMatch(child)) continue;
|
||||
if (!re.IsMatch(child.AbsolutePath)) continue;
|
||||
|
||||
bool isFolder = false;
|
||||
bool isDfsStream = false;
|
||||
|
||||
if (blocks.ContainsKey(child))
|
||||
if (this.blocks.ContainsKey(child.AbsolutePath))
|
||||
{
|
||||
isDfsStream = true;
|
||||
length = blocks[child];
|
||||
length = this.blocks[child.AbsolutePath];
|
||||
}
|
||||
else if (pages.ContainsKey(child))
|
||||
else if (this.pages.ContainsKey(child.AbsolutePath))
|
||||
{
|
||||
isDfsStream = false;
|
||||
length = pages[child];
|
||||
length = this.pages[child.AbsolutePath];
|
||||
}
|
||||
else if (this.client != null)
|
||||
// otherwise this information may be incorrect
|
||||
isFolder = true;
|
||||
|
||||
var file = new AzureDfsFile(this.Config, this.Job, this.client, child, this.ShouldCacheLocally, isFolder);
|
||||
var file = new AzureDfsFile(this.Config, this.Job, this.client, PathFromUri(this.Config as AzureDfsClusterConfiguration, child), this.ShouldCacheLocally, isFolder);
|
||||
file.IsDfsStream = isDfsStream;
|
||||
file.size = length;
|
||||
|
||||
|
|
@ -1147,30 +1202,24 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
this.PopulateCache();
|
||||
|
||||
string filepath;
|
||||
if (this.client != null)
|
||||
filepath = this.client.Combine(this.path, filename);
|
||||
else
|
||||
filepath = Path.Combine(this.path, filename);
|
||||
string combined = Path.Combine(this.path, filename);
|
||||
Uri filepath = UriFromPath(this.Config as AzureDfsClusterConfiguration, combined);
|
||||
bool isFolder = false;
|
||||
bool isDfsStream = false;
|
||||
long sz = -1;
|
||||
|
||||
if (blocks.ContainsKey(filepath))
|
||||
if (this.blocks.ContainsKey(combined))
|
||||
{
|
||||
isDfsStream = true;
|
||||
sz = blocks[filepath];
|
||||
sz = this.blocks[filepath.AbsolutePath];
|
||||
}
|
||||
else if (pages.ContainsKey(filepath))
|
||||
else if (this.pages.ContainsKey(combined))
|
||||
{
|
||||
isDfsStream = false;
|
||||
sz = pages[filepath];
|
||||
sz = this.pages[filepath.AbsolutePath];
|
||||
}
|
||||
else if (this.client != null)
|
||||
// if the client is null the information may be incorrect
|
||||
isFolder = true;
|
||||
|
||||
var file = new AzureDfsFile(this.Config, this.Job, this.client, filepath, this.ShouldCacheLocally, isFolder);
|
||||
var file = new AzureDfsFile(this.Config, this.Job, this.client, combined, this.ShouldCacheLocally, isFolder);
|
||||
file.IsDfsStream = isDfsStream;
|
||||
file.size = sz;
|
||||
return file;
|
||||
|
|
@ -1202,4 +1251,232 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
return this.path;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A file residing on HDFS (accessed either using HDFS or WebHdfs).
|
||||
/// </summary>
|
||||
public class DfsFile : CachedClusterResidentObject
|
||||
{
|
||||
private Uri baseUri;
|
||||
private Uri uri;
|
||||
private HdfsClientBase client;
|
||||
|
||||
/// <summary>
|
||||
/// A file with the specified path.
|
||||
/// </summary>
|
||||
/// <param name="path">Path to the file.</param>
|
||||
/// <param name="client">Azure client.</param>
|
||||
/// <param name="config">Cluster configuration.</param>
|
||||
/// <param name="job">Job accessing this file.</param>
|
||||
/// <param name="jobFolderUri">Uri to base folder.</param>
|
||||
/// <param name="isFolder">If true this must be a folder.</param>
|
||||
/// <param name="canCache">True if the file can be cached (it is immutable for sure).</param>
|
||||
public DfsFile(ClusterConfiguration config, Uri jobFolderUri, DryadLinqJobSummary job, HdfsClientBase client, string path, bool canCache, bool isFolder)
|
||||
: base(config, job)
|
||||
{
|
||||
this.client = client;
|
||||
this.Exception = null;
|
||||
this.baseUri = jobFolderUri;
|
||||
this.uri = UriFromPath(jobFolderUri, path);
|
||||
this.ShouldCacheLocally = canCache;
|
||||
this.RepresentsAFolder = isFolder;
|
||||
this.size = -1;
|
||||
|
||||
Console.WriteLine("DfsFile Uri={0}", this.uri);
|
||||
if (!string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
|
||||
this.LocalCachePath = Path.Combine(CachedClusterResidentObject.CacheDirectory, PathFromUri(this.baseUri, this.uri));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a File URI from a file path.
|
||||
/// </summary>
|
||||
/// <param name="baseUri">Base URI point to jobs folder.</param>
|
||||
/// <param name="path">Path to file.</param>
|
||||
/// <returns>The File uri.</returns>
|
||||
public static Uri UriFromPath(Uri baseUri, string path)
|
||||
{
|
||||
UriBuilder builder = new UriBuilder(baseUri);
|
||||
builder.Path = builder.Path.TrimEnd('/') + "/" + path.TrimStart('/');
|
||||
return builder.Uri;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a path from a URI.
|
||||
/// </summary>
|
||||
/// <param name="uri">URI of an HDFS file.</param>
|
||||
/// <param name="baseUri">Base URI (should be a prefix of the other uri).</param>
|
||||
/// <returns>The path to the file.</returns>
|
||||
public static string PathFromUri(Uri baseUri, Uri uri)
|
||||
{
|
||||
string path = uri.PathAndQuery;
|
||||
if (path.StartsWith(baseUri.PathAndQuery))
|
||||
path = path.Substring(baseUri.PathAndQuery.Length);
|
||||
path = path.Trim('/');
|
||||
return path;
|
||||
}
|
||||
|
||||
private DfsFile(ClusterConfiguration config, DryadLinqJobSummary job, Exception ex)
|
||||
: base(config, job)
|
||||
{
|
||||
this.Exception = ex;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True if the object is a folder.
|
||||
/// </summary>
|
||||
public override bool RepresentsAFolder
|
||||
{
|
||||
get;
|
||||
protected set;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
|
||||
/// </summary>
|
||||
/// <returns>A stream that can be used to access the object contents.</returns>
|
||||
/// <param name="keepNewlines">If true keep the newlines.</param>
|
||||
public override ISharedStreamReader GetStream(bool keepNewlines)
|
||||
{
|
||||
|
||||
ISharedStreamReader baseStream = base.GetStream(keepNewlines);
|
||||
if (baseStream != null)
|
||||
{
|
||||
// file is cached
|
||||
Trace.TraceInformation("Reading from local cache {0}", baseStream);
|
||||
return baseStream;
|
||||
}
|
||||
|
||||
Stream stream = this.client.GetDfsStreamReader(this.uri);
|
||||
long sz = this.Size;
|
||||
int bufferSize = 1024 * 1024;
|
||||
if (sz >= 0)
|
||||
{
|
||||
bufferSize = (int)(sz / 10);
|
||||
if (bufferSize < 1024 * 1024)
|
||||
bufferSize = 1024 * 1024;
|
||||
if (bufferSize > 20 * 1024 * 1024)
|
||||
bufferSize = 20 * 1024 * 1024;
|
||||
}
|
||||
SimpleStreamReader reader = new SimpleStreamReader(stream, true, Encoding.UTF8, false, bufferSize);
|
||||
|
||||
if (this.ShouldCacheLocally && this.LocalCachePath != null)
|
||||
{
|
||||
// cache it
|
||||
if (this.RepresentsAFolder)
|
||||
throw new ClusterException("Cannot cache folders");
|
||||
StreamWriter writer = this.CreateTempStream();
|
||||
return new SharedStreamReader(reader, writer, keepNewlines, this.OnClose);
|
||||
}
|
||||
else
|
||||
{
|
||||
// dont cache it
|
||||
return new SharedStreamReader(reader, keepNewlines);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// If the current object is a folder, it returns the contained objects.
|
||||
/// </summary>
|
||||
/// <returns>An iterator over all contained objects that match the specified string.</returns>
|
||||
/// <param name="match">A shell expression (similar to the argument of Directory.GetFiles()).</param>
|
||||
public override IEnumerable<IClusterResidentObject> GetFilesAndFolders(string match)
|
||||
{
|
||||
HashSet<Uri> folders = new HashSet<Uri>();
|
||||
foreach (var child in this.client.EnumerateSubdirectories(this.uri))
|
||||
{
|
||||
folders.Add(child);
|
||||
}
|
||||
|
||||
Regex re = Utilities.RegexFromSearchPattern(match);
|
||||
foreach (var child in this.client.ExpandFileOrDirectory(this.uri))
|
||||
{
|
||||
if (!re.IsMatch(PathFromUri(this.baseUri, child))) continue;
|
||||
|
||||
bool isFolder = folders.Contains(child);
|
||||
var file = new DfsFile(this.Config, this.baseUri, this.Job, this.client, PathFromUri(this.baseUri, child), this.ShouldCacheLocally, isFolder);
|
||||
|
||||
long length;
|
||||
long time;
|
||||
this.client.GetFileStatus(child, out time, out length);
|
||||
file.size = length;
|
||||
file.CreationTime = TimeFromLong(time);
|
||||
|
||||
yield return file;
|
||||
}
|
||||
}
|
||||
|
||||
private long size;
|
||||
/// <summary>
|
||||
/// Size of the object in bytes (if not a folder). The size can be -1 when it is unknown.
|
||||
/// </summary>
|
||||
public override long Size
|
||||
{
|
||||
get { return this.size; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Short name of the object.
|
||||
/// </summary>
|
||||
public override string Name
|
||||
{
|
||||
get { return Path.GetFileName(PathFromUri(this.baseUri, this.uri)); }
|
||||
}
|
||||
|
||||
private static DateTime origin = new DateTime(1970, 1, 1);
|
||||
/// <summary>
|
||||
/// Convert file status time into a DateTime.
|
||||
/// </summary>
|
||||
/// <param name="fileTime">File time obtained from client.</param>
|
||||
/// <returns>A DateTime object.</returns>
|
||||
public static DateTime TimeFromLong(long fileTime)
|
||||
{
|
||||
return origin + TimeSpan.FromMilliseconds(fileTime);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Date when object was created.
|
||||
/// </summary>
|
||||
public override DateTime CreationTime
|
||||
{
|
||||
get { return DateTime.Now; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// For a folder object, returns the contained file with the specified name.
|
||||
/// </summary>
|
||||
/// <param name="filename">File name within the folder.</param>
|
||||
/// <returns>The file within the folder.</returns>
|
||||
public override IClusterResidentObject GetFile(string filename)
|
||||
{
|
||||
var matchingFiles = this.GetFilesAndFolders(filename).ToList();
|
||||
if (matchingFiles.Count == 1)
|
||||
return matchingFiles[0];
|
||||
|
||||
return new DfsFile(this.Config, this.Job, new Exception("Ambiguous name " + filename + " in " + this));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// For a folder object, returns the contained folder with the specified name.
|
||||
/// </summary>
|
||||
/// <param name="foldername">Folder name within the folder.</param>
|
||||
/// <returns>The subfolder within the folder.</returns>
|
||||
public override IClusterResidentObject GetFolder(string foldername)
|
||||
{
|
||||
var file = this.GetFile(foldername);
|
||||
if (!file.RepresentsAFolder) throw new InvalidOperationException(foldername + " is not a folder");
|
||||
return file;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a string that represents the current object.
|
||||
/// </summary>
|
||||
/// <returns>
|
||||
/// A string that represents the current object.
|
||||
/// </returns>
|
||||
/// <filterpriority>2</filterpriority>
|
||||
public override string ToString()
|
||||
{
|
||||
return this.uri.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,17 +19,17 @@ limitations under the License.
|
|||
|
||||
*/
|
||||
|
||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
||||
namespace Microsoft.Research.JobObjectModel
|
||||
{
|
||||
using System.Text.RegularExpressions;
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
|
||||
/// <summary>
|
||||
/// Information about a standard Cosmos log entry.
|
||||
/// Information about a standard Dryad log entry.
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public class CosmosLogEntry : IParse
|
||||
public class DryadLogEntry : IParse
|
||||
{
|
||||
/// <summary>
|
||||
/// Message severity.
|
||||
|
|
@ -98,7 +98,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// Allocate an empty log entry
|
||||
/// </summary>
|
||||
public CosmosLogEntry()
|
||||
public DryadLogEntry()
|
||||
{
|
||||
this.Malformed = true;
|
||||
}
|
||||
|
|
@ -107,7 +107,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// Allocate a log entry from a given string.
|
||||
/// </summary>
|
||||
/// <param name="line">String to initialize the log entry.</param>
|
||||
public CosmosLogEntry(string line)
|
||||
public DryadLogEntry(string line)
|
||||
{
|
||||
this.Malformed = true;
|
||||
// ReSharper disable once DoNotCallOverridableMethodsInConstructor
|
||||
|
|
@ -167,7 +167,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// An extended log entry is like a cosmos log entry, but it has a prefix: GUID,Machine
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public class ExtendedLogEntry : CosmosLogEntry
|
||||
public class ExtendedLogEntry : DryadLogEntry
|
||||
{
|
||||
static Regex loglineregex =
|
||||
new Regex(@"([-0-9A-F]+), # 1 guid, inserted by reader
|
||||
|
|
@ -19,32 +19,29 @@ limitations under the License.
|
|||
|
||||
*/
|
||||
|
||||
#undef USE_LINQ_TO_DRYAD
|
||||
#undef USE_HPC
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Research.Calypso.Tools;
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Research.Tools;
|
||||
|
||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
||||
namespace Microsoft.Research.JobObjectModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Exception throw by Calypso when it cannot understand the structure of a Dryad/DryadLINQ job.
|
||||
/// Exception thrown when we cannot understand the structure of a Dryad/DryadLINQ job.
|
||||
/// </summary>
|
||||
public class CalypsoDryadException : Exception
|
||||
public class DryadException : Exception
|
||||
{
|
||||
/// <summary>
|
||||
/// Create a new CalypsoDryadException.
|
||||
/// Create a new DryadException.
|
||||
/// </summary>
|
||||
/// <param name="message">Message conveyed by the exception.</param>
|
||||
public CalypsoDryadException(string message) : base(message) { }
|
||||
public DryadException(string message) : base(message) { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -195,7 +192,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
this.SuccessfulVertices++;
|
||||
break;
|
||||
default:
|
||||
throw new CalypsoDryadException("Unexpected vertex state " + vertex.State);
|
||||
throw new DryadException("Unexpected vertex state " + vertex.State);
|
||||
}
|
||||
}
|
||||
this.TotalInitiatedVertices -= this.AbandonedVertices;
|
||||
|
|
@ -423,22 +420,21 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <param name="cf">Cluster configuration.</param>
|
||||
/// <param name="summary">Summary description of the job.</param>
|
||||
/// <returns>The Dryad job description, or null.</returns>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="fill">If true, fill all the information, otherwise the user will have to call FillInformation on the result later.</param>
|
||||
/// <param name="updateProgress">Delegate used to report progress.</param>
|
||||
public static DryadLinqJobInfo CreateDryadLinqJobInfo(ClusterConfiguration cf, DryadLinqJobSummary summary, bool fill, StatusReporter reporter, Action<int> updateProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public static DryadLinqJobInfo CreateDryadLinqJobInfo(ClusterConfiguration cf, DryadLinqJobSummary summary, bool fill, CommManager manager)
|
||||
{
|
||||
try
|
||||
{
|
||||
DryadLinqJobInfo job = new DryadLinqJobInfo(cf, summary);
|
||||
if (fill)
|
||||
job.CollectEssentialInformation(reporter, updateProgress);
|
||||
job.CollectEssentialInformation(manager);
|
||||
return job;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Trace.TraceInformation(e.ToString());
|
||||
reporter("Could not collect job information for " + summary.Name + ": " + e.Message, StatusKind.Error);
|
||||
manager.Status("Could not collect job information for " + summary.Name + ": " + e.Message, StatusKind.Error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -506,7 +502,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
|
||||
if (this.stdoutpath == null)
|
||||
{
|
||||
throw new CalypsoClusterException("Could not locate JM standard output file in folder " + jmdir);
|
||||
throw new ClusterException("Could not locate JM standard output file in folder " + jmdir);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -515,9 +511,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// Refresh the job status.
|
||||
/// </summary>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="updateProgress">Used to report progress.</param>
|
||||
public void RefreshJobStatus(StatusReporter reporter, Action<int> updateProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public void RefreshJobStatus(CommManager manager)
|
||||
{
|
||||
// skip if job is finished
|
||||
if (this.Summary.Status == ClusterJobInformation.ClusterJobStatus.Failed ||
|
||||
|
|
@ -526,18 +521,17 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
return;
|
||||
|
||||
ClusterStatus status = this.ClusterConfiguration.CreateClusterStatus();
|
||||
status.RefreshStatus(this.Summary, reporter, updateProgress);
|
||||
status.RefreshStatus(this.Summary, manager);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fill the job info by parsing the stdout.txt.
|
||||
/// <param name="statusReporter">Delegate used to report errors.</param>
|
||||
/// <returns>True if it succeeds, false otherwise.</returns>
|
||||
/// <param name="updateProgress">Delegate used to report progress.</param>
|
||||
/// <returns>The updated job.</returns>
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
/// </summary>
|
||||
public bool CollectEssentialInformation(StatusReporter statusReporter, Action<int> updateProgress)
|
||||
public bool CollectEssentialInformation(CommManager manager)
|
||||
{
|
||||
this.RefreshJobStatus(statusReporter, updateProgress);
|
||||
this.RefreshJobStatus(manager);
|
||||
if (this.ManagerVertex == null)
|
||||
{
|
||||
this.ManagerVertex = new ExecutedVertexInstance(this, -1, 0, "JobManager", "", this.Summary.Date);
|
||||
|
|
@ -562,13 +556,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
|
||||
if (this.stdoutpath == null)
|
||||
return false;
|
||||
bool success = this.ParseStdout(this.stdoutpath, statusReporter, updateProgress);
|
||||
updateProgress(100);
|
||||
bool success = this.ParseStdout(this.stdoutpath, manager);
|
||||
manager.Progress(100);
|
||||
if (!success)
|
||||
return false;
|
||||
|
||||
this.JobInfoCannotBeCollected = false;
|
||||
statusReporter("Stdout parsed", StatusKind.OK);
|
||||
manager.Status("Stdout parsed", StatusKind.OK);
|
||||
|
||||
this.LastUpdatetime = DateTime.Now;
|
||||
if (this.Summary.Status == ClusterJobInformation.ClusterJobStatus.Running)
|
||||
|
|
@ -600,13 +594,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
Match m = numberre.Match(vertexlist);
|
||||
if (!m.Success)
|
||||
throw new CalypsoDryadException("Could not find vertex number in " + vertexlist);
|
||||
throw new DryadException("Could not find vertex number in " + vertexlist);
|
||||
string number = m.Groups[1].Value;
|
||||
|
||||
// now scan a balanced number of parantheses
|
||||
string rest = m.Groups[2].Value;
|
||||
if (rest[0] != '(')
|
||||
throw new CalypsoDryadException("Expecting open parens after vertex number");
|
||||
throw new DryadException("Expecting open parens after vertex number");
|
||||
int opened = 0;
|
||||
int i;
|
||||
for (i = 0; i < rest.Length; i++)
|
||||
|
|
@ -624,7 +618,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
}
|
||||
}
|
||||
if (opened != 0 || i <= 2)
|
||||
throw new CalypsoDryadException("did not find matched parantheses in vertex name in " + vertexlist + ", can't parse");
|
||||
throw new DryadException("did not find matched parantheses in vertex name in " + vertexlist + ", can't parse");
|
||||
string name = rest.Substring(1, i - 2); // skip first and last paranthesis
|
||||
yield return new Tuple<string, int>(name, int.Parse(number));
|
||||
vertexlist = rest.Substring(i);
|
||||
|
|
@ -664,6 +658,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
return 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sometimes the StreamReader can read a partial line only.
|
||||
/// We cache here the previously read line and if it looks like the current line is just a fragment, then we concatentate it
|
||||
/// to the previous line. The line is a fragment if it does not start with 'logtimelocal'.
|
||||
/// </summary>
|
||||
private string previousLine = "";
|
||||
|
||||
/// <summary>
|
||||
/// New JM stdout parsing code, for YARN-based DryadLINQ.
|
||||
/// </summary>
|
||||
|
|
@ -672,6 +673,17 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
private bool ParseStdoutLineNew(string line)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line)) return true;
|
||||
// The line should start with logtimelocal, otherwise it's probably a fragment of the previous line
|
||||
this.previousLine += line;
|
||||
|
||||
if (!(line.EndsWith("\r") || line.EndsWith("\n")))
|
||||
{
|
||||
// line is incomplete, return now, parse later
|
||||
return true;
|
||||
}
|
||||
|
||||
line = this.previousLine.Trim();
|
||||
this.previousLine = "";
|
||||
|
||||
Dictionary<string, string> kvp = Utilities.ParseCSVKVP(line);
|
||||
if (kvp == null) return false;
|
||||
|
|
@ -761,7 +773,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
case "running":
|
||||
{
|
||||
string process;
|
||||
kvp.TryGetValue("id", out process); // "process" is also good
|
||||
kvp.TryGetValue("id", out process);
|
||||
if (process == null)
|
||||
kvp.TryGetValue("process", out process);
|
||||
string machine = kvp["computer"];
|
||||
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
||||
this.jobVertices.Remap(vi, process);
|
||||
|
|
@ -776,6 +790,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
vi.SetState(ExecutedVertexInstance.VertexState.Successful);
|
||||
vi.End = timeStamp;
|
||||
vi.ExitCode = "";
|
||||
this.UsefulCPUTime += vi.RunningTime;
|
||||
break;
|
||||
}
|
||||
case "failed":
|
||||
|
|
@ -784,7 +799,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
if (vi.State != ExecutedVertexInstance.VertexState.Started)
|
||||
vi.SetState(ExecutedVertexInstance.VertexState.Cancelled);
|
||||
else
|
||||
{
|
||||
vi.SetState(ExecutedVertexInstance.VertexState.Failed);
|
||||
if (vi.RunningTime > TimeSpan.Zero)
|
||||
this.WastedCPUTime += vi.RunningTime;
|
||||
}
|
||||
if (kvp.ContainsKey("errorstring"))
|
||||
vi.AddErrorString(kvp["errorstring"]);
|
||||
string exitcode;
|
||||
|
|
@ -797,33 +816,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
}
|
||||
else if (kvp.ContainsKey("outputChannel"))
|
||||
{
|
||||
string chan = kvp["outputChannel"];
|
||||
int channelNo = int.Parse(chan);
|
||||
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
||||
|
||||
if (!kvp.ContainsKey("errorstatus"))
|
||||
{
|
||||
}
|
||||
else
|
||||
{
|
||||
if (kvp.ContainsKey("errorstring"))
|
||||
vi.AddErrorString(kvp["errorstring"]);
|
||||
}
|
||||
}
|
||||
else if (kvp.ContainsKey("inputChannel"))
|
||||
{
|
||||
string chan = kvp["inputChannel"];
|
||||
int channelNo = int.Parse(chan);
|
||||
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
||||
|
||||
if (!kvp.ContainsKey("errorstatus"))
|
||||
{
|
||||
}
|
||||
else
|
||||
{
|
||||
if (kvp.ContainsKey("errorstring"))
|
||||
vi.AddErrorString(kvp["errorstring"]);
|
||||
}
|
||||
}
|
||||
else if (kvp.ContainsKey("io"))
|
||||
{
|
||||
if (kvp["io"] == "starting")
|
||||
|
|
@ -834,6 +836,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
|
||||
if (vi.InputChannels == null)
|
||||
vi.InputChannels = new Dictionary<int, ChannelEndpointDescription>();
|
||||
|
||||
for (int i = 0; i < numberOfInputs; i++)
|
||||
{
|
||||
string uri;
|
||||
|
|
@ -861,7 +864,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
||||
|
||||
long totalRead = TryGetNumeric(kvp, "totalRead");
|
||||
long tempRead = TryGetNumeric(kvp, "tempRead");
|
||||
//long tempRead = TryGetNumeric(kvp, "tempRead");
|
||||
long tempReadInRack = TryGetNumeric(kvp, "tempReadInRack");
|
||||
long tempReadCrossRack = TryGetNumeric(kvp, "tempReadCrossRack");
|
||||
long localRead = TryGetNumeric(kvp, "localRead");
|
||||
|
|
@ -870,6 +873,24 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
vi.DataRead = totalRead;
|
||||
vi.DataWritten = totalWritten;
|
||||
|
||||
if (vi.InputChannels != null)
|
||||
{
|
||||
foreach (int ch in vi.InputChannels.Keys)
|
||||
{
|
||||
long bytes = TryGetNumeric(kvp, "rb." + ch);
|
||||
vi.InputChannels[ch].Size = bytes;
|
||||
}
|
||||
}
|
||||
|
||||
if (vi.OutputChannels != null)
|
||||
{
|
||||
foreach (int ch in vi.OutputChannels.Keys)
|
||||
{
|
||||
long bytes = TryGetNumeric(kvp, "wb." + ch);
|
||||
vi.OutputChannels[ch].Size = bytes;
|
||||
}
|
||||
}
|
||||
|
||||
this.TotalDataRead += totalRead;
|
||||
this.LocalReadData += localRead;
|
||||
this.CrossPodDataRead += tempReadCrossRack;
|
||||
|
|
@ -879,17 +900,26 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
||||
|
||||
if (vi.InputChannels != null)
|
||||
{
|
||||
foreach (int ch in vi.InputChannels.Keys)
|
||||
{
|
||||
long bytes = TryGetNumeric(kvp, "rb." + ch);
|
||||
vi.InputChannels[ch].Size = bytes;
|
||||
|
||||
bytes = TryGetNumeric(kvp, "tb." + ch);
|
||||
vi.InputChannels[ch].TotalSize = bytes;
|
||||
}
|
||||
}
|
||||
|
||||
if (vi.InputChannels != null)
|
||||
{
|
||||
foreach (int ch in vi.OutputChannels.Keys)
|
||||
{
|
||||
long bytes = TryGetNumeric(kvp, "wb." + ch);
|
||||
vi.OutputChannels[ch].Size = bytes;
|
||||
}
|
||||
}
|
||||
|
||||
long totalRead = TryGetNumeric(kvp, "totalRead");
|
||||
long totalWritten = TryGetNumeric(kvp, "totalWritten");
|
||||
|
|
@ -1062,7 +1092,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
else if (m.Groups[1].Value == "ices")
|
||||
onevertex = false;
|
||||
else
|
||||
throw new CalypsoDryadException("Can't figure out if one or many vertices");
|
||||
throw new DryadException("Can't figure out if one or many vertices");
|
||||
|
||||
IEnumerable<Tuple<string, int>> vertexList = DryadLinqJobInfo.ParseVertices(vertices);
|
||||
|
||||
|
|
@ -1084,7 +1114,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
}
|
||||
|
||||
if (vertexcount > 1 && onevertex)
|
||||
throw new CalypsoDryadException("Expected one vertex, found " + vertexcount);
|
||||
throw new DryadException("Expected one vertex, found " + vertexcount);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -1225,7 +1255,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
if (vi.State == ExecutedVertexInstance.VertexState.Started)
|
||||
{
|
||||
Console.WriteLine("Timing information while vertex is still running " + vi);
|
||||
//throw new CalypsoClusterException("Timing information for vertex still running: " + vi);
|
||||
//throw new ClusterException("Timing information for vertex still running: " + vi);
|
||||
}
|
||||
DateTime last = vi.SetTiming(createtime, m.Groups[5].Value, m.Groups[6].Value, m.Groups[7].Value, m.Groups[8].Value, m.Groups[9].Value);
|
||||
if (last > this.lastTimestampSeen)
|
||||
|
|
@ -1245,7 +1275,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
}
|
||||
}
|
||||
else
|
||||
throw new CalypsoDryadException("Unmatched timing information line " + line);
|
||||
throw new DryadException("Unmatched timing information line " + line);
|
||||
}
|
||||
else if (line.Contains("Process has failed"))
|
||||
{
|
||||
|
|
@ -1410,7 +1440,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
return retval;
|
||||
}
|
||||
|
||||
private ISharedStreamReader cachedStdoutReader = null;
|
||||
private ISharedStreamReader cachedStdoutReader;
|
||||
|
||||
/// <summary>
|
||||
/// Remember how many lines were parsed, and skip them on a second invocation.
|
||||
|
|
@ -1420,10 +1450,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// Parse the stdout.txt file from the job manager.
|
||||
/// </summary>
|
||||
/// <param name="file">File to parse.</param>
|
||||
/// <param name="statusReporter">Delegate used to report errors.</param>
|
||||
/// <param name="updateProgress">Delegate used to report progress.</param>
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
/// <returns>True if the parsing succeeds.</returns>
|
||||
private bool ParseStdout(IClusterResidentObject file, StatusReporter statusReporter, Action<int> updateProgress)
|
||||
private bool ParseStdout(IClusterResidentObject file, CommManager manager)
|
||||
{
|
||||
int currentLine = 0;
|
||||
if (this.stdoutLinesParsed == 0)
|
||||
|
|
@ -1440,15 +1469,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
string message = "Scanning JM stdout " + file;
|
||||
if (filesize >= 0)
|
||||
message += string.Format("({0:N0} bytes)", filesize);
|
||||
statusReporter(message, StatusKind.LongOp);
|
||||
manager.Status(message, StatusKind.LongOp);
|
||||
|
||||
if (this.cachedStdoutReader == null)
|
||||
this.cachedStdoutReader = file.GetStream();
|
||||
this.cachedStdoutReader = file.GetStream(true);
|
||||
if (this.cachedStdoutReader.Exception != null)
|
||||
{
|
||||
statusReporter("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
|
||||
manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
|
||||
return false;
|
||||
}
|
||||
|
||||
while (!this.cachedStdoutReader.EndOfStream)
|
||||
{
|
||||
string line = this.cachedStdoutReader.ReadLine();
|
||||
|
|
@ -1457,23 +1487,27 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
while (true)
|
||||
{
|
||||
int startLine = currentLine;
|
||||
manager.Token.ThrowIfCancellationRequested();
|
||||
bool completeLine = true;
|
||||
try
|
||||
{
|
||||
if (this.ClusterConfiguration is DfsClusterConfiguration)
|
||||
completeLine = this.ParseStdoutLineNew(line);
|
||||
else
|
||||
{
|
||||
this.ParseStdoutLine(line);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
statusReporter(string.Format("Line {0}: Exception {1}", currentLine, ex.Message), StatusKind.Error);
|
||||
manager.Status(string.Format("Line {0}: Exception {1}", currentLine, ex.Message), StatusKind.Error);
|
||||
Console.WriteLine("Line {0}: Exception {1}", currentLine, ex);
|
||||
}
|
||||
if (!completeLine)
|
||||
{
|
||||
if (this.cachedStdoutReader.EndOfStream)
|
||||
{
|
||||
throw new Exception("File ended while scanning for closing quote started on line " + startLine);
|
||||
}
|
||||
// no exception, the log may be truncated
|
||||
break;
|
||||
|
||||
string extraline = this.cachedStdoutReader.ReadLine();
|
||||
line += "\n" + extraline;
|
||||
|
|
@ -1485,7 +1519,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
currentLine++;
|
||||
if (currentLine % 100 == 0 && filesize > 0)
|
||||
{
|
||||
updateProgress(Math.Min(100, (int)(100 * readbytes / filesize)));
|
||||
manager.Progress(Math.Min(100, (int)(100 * readbytes / filesize)));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1500,13 +1534,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
// we are done with this stream
|
||||
if (this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Failed ||
|
||||
this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Successful)
|
||||
{
|
||||
this.cachedStdoutReader.Close();
|
||||
this.cachedStdoutReader = null; // will force reopening if refreshed
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
statusReporter("Exception while reading stdout " + e.Message, StatusKind.Error);
|
||||
manager.Status("Exception while reading stdout " + e.Message, StatusKind.Error);
|
||||
Trace.TraceInformation(e.ToString());
|
||||
return false;
|
||||
}
|
||||
|
|
@ -1581,7 +1618,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
bool success = true;
|
||||
|
||||
ISharedStreamReader sr = logfile.GetStream();
|
||||
ISharedStreamReader sr = logfile.GetStream(false);
|
||||
if (sr.Exception != null)
|
||||
{
|
||||
statusReporter("Exception while opening file " + logfile + ":" + sr.Exception.Message, StatusKind.Error);
|
||||
|
|
@ -1592,7 +1629,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
string line = sr.ReadLine();
|
||||
if (!line.Contains("DryadProfiler")) continue;
|
||||
|
||||
CosmosLogEntry le = new CosmosLogEntry(line);
|
||||
DryadLogEntry le = new DryadLogEntry(line);
|
||||
if (le.Subsystem != "DryadProfiler") continue;
|
||||
if (!le.Message.EndsWith("channel status")) continue;
|
||||
|
||||
|
|
@ -1905,7 +1942,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// Parse the query plan: cluster-specific.
|
||||
/// </summary>
|
||||
protected abstract void ParseQueryPlan();
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
protected abstract void ParseQueryPlan(CommManager manager);
|
||||
|
||||
int fictitiousStages;
|
||||
|
||||
|
|
@ -2009,11 +2047,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// Factory: create the plan for a given job.
|
||||
/// </summary>
|
||||
/// <param name="dryadLinqJobInfo">Job to create plan for.</param>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <returns>The plan or null.</returns>
|
||||
public static DryadJobStaticPlan CreatePlan(DryadLinqJobInfo dryadLinqJobInfo, StatusReporter reporter)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public static DryadJobStaticPlan CreatePlan(DryadLinqJobInfo dryadLinqJobInfo, CommManager manager)
|
||||
{
|
||||
reporter("Trying to build static plan", StatusKind.LongOp);
|
||||
manager.Status("Trying to build static plan", StatusKind.LongOp);
|
||||
ClusterConfiguration config = dryadLinqJobInfo.ClusterConfiguration;
|
||||
IClusterResidentObject file = config.JobQueryPlan(dryadLinqJobInfo.Summary);
|
||||
if (config is CacheClusterConfiguration)
|
||||
|
|
@ -2023,14 +2061,14 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
DryadJobStaticPlan retval;
|
||||
{
|
||||
retval = new DryadLinqJobStaticPlan(config, file.GetStream());
|
||||
retval = new DryadLinqJobStaticPlan(config, file.GetStream(false));
|
||||
}
|
||||
retval.ParseQueryPlan();
|
||||
retval.ParseQueryPlan(manager);
|
||||
return retval;
|
||||
}
|
||||
else
|
||||
{
|
||||
reporter("Exception while looking for plan " + file.Exception.Message, StatusKind.Error);
|
||||
manager.Status("Exception while looking for plan " + file.Exception.Message, StatusKind.Error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -2056,9 +2094,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// Parse an XML query plan and represent that information.
|
||||
/// </summary>
|
||||
protected override void ParseQueryPlan()
|
||||
/// <param name="manager">Communicaton manager.</param>
|
||||
protected override void ParseQueryPlan(CommManager manager)
|
||||
{
|
||||
string planString = this.planStream.ReadToEnd();
|
||||
string planString = this.planStream.ReadToEnd(manager.Token);
|
||||
|
||||
XDocument plan = XDocument.Parse(planString);
|
||||
// ReSharper disable PossibleNullReferenceException
|
||||
|
|
@ -2104,7 +2143,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
info.Arity = Connection.ConnectionType.AllToAll;
|
||||
break;
|
||||
default:
|
||||
throw new CalypsoDryadException("Don't know about connection of type " + connection);
|
||||
throw new DryadException("Don't know about connection of type " + connection);
|
||||
}
|
||||
switch (cht)
|
||||
{
|
||||
|
|
@ -2118,7 +2157,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
info.ChannelKind = Connection.ChannelType.Fifo;
|
||||
break;
|
||||
default:
|
||||
throw new CalypsoDryadException("Don't know about channel of type " + cht);
|
||||
throw new DryadException("Don't know about channel of type " + cht);
|
||||
}
|
||||
this.perNodeConnectionInfo.Add(stage.Id, info);
|
||||
}
|
||||
|
|
@ -2247,14 +2286,15 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// Parse the Algebra file.
|
||||
/// </summary>
|
||||
private void ParseAlgebra()
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
private void ParseAlgebra(CommManager manager)
|
||||
{
|
||||
// TODO: this parser is not really complete, as I don't understand the semantics of all xml elements.
|
||||
Dictionary<string, string> outToStage = new Dictionary<string, string>(); // map an output to a stage name. Assume that ios have unique names.
|
||||
Dictionary<string, List<string>> inputs = new Dictionary<string, List<string>>();
|
||||
|
||||
// <CsJobAlgebra> <graph> <process> ...
|
||||
string planString = this.planStream.ReadToEnd();
|
||||
string planString = this.planStream.ReadToEnd(manager.Token);
|
||||
XDocument plan = XDocument.Parse(planString);
|
||||
// ReSharper disable PossibleNullReferenceException
|
||||
XElement graph = plan.Root.Element("graph"); // graph node, children are stages
|
||||
|
|
@ -2416,13 +2456,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// Parse the vertex definition file.
|
||||
/// </summary>
|
||||
private void ParseVertexDef()
|
||||
private void ParseVertexDef(CommManager manager)
|
||||
{
|
||||
if (this.vertexDef.Exception != null)
|
||||
return;
|
||||
|
||||
// <ScopeVertices> <ScopeVertex> <operator> <input> </input> <output> </output>
|
||||
string planString = this.vertexDef.ReadToEnd();
|
||||
string planString = this.vertexDef.ReadToEnd(manager.Token);
|
||||
XDocument vxDef = XDocument.Parse(planString);
|
||||
|
||||
XElement vertices = vxDef.Root;
|
||||
|
|
@ -2469,10 +2509,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <summary>
|
||||
/// Parse the query plan for a Scope job.
|
||||
/// </summary>
|
||||
protected override void ParseQueryPlan()
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
protected override void ParseQueryPlan(CommManager manager)
|
||||
{
|
||||
this.ParseAlgebra();
|
||||
this.ParseVertexDef();
|
||||
this.ParseAlgebra(manager);
|
||||
this.ParseVertexDef(manager);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2653,9 +2694,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// </summary>
|
||||
public string LocalPath { get; protected set; }
|
||||
/// <summary>
|
||||
/// How big is the channel (0 if it cannot be determined, e.g. FIFO, -1 if the channel data cannot be retrieved).
|
||||
/// The actual data read/written so far (0 if it cannot be determined, e.g. FIFO, -1 if the channel data cannot be retrieved).
|
||||
/// </summary>
|
||||
public long Size { get; set; }
|
||||
/// <summary>
|
||||
/// How much of the channel was
|
||||
/// </summary>
|
||||
public long TotalSize { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// String representation of the endpoint.
|
||||
|
|
@ -2664,7 +2709,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
string uritype = this.UriType;
|
||||
string localpath = this.LocalPath;
|
||||
if (this.TotalSize == 0)
|
||||
return string.Format("{0,4} {1,20:N0} {2}://{3}", this.Number, this.Size, uritype, localpath);
|
||||
else
|
||||
return string.Format("{0,4} {1,20:N0}/{2,20:N0} {3}://{4}", this.Number, this.Size, this.TotalSize, uritype, localpath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -2684,7 +2732,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
|
||||
int sepindex = uri.IndexOf("://");
|
||||
if (sepindex < 0)
|
||||
throw new CalypsoDryadException("Channel URI " + uri + " does not contain separator ://");
|
||||
throw new DryadException("Channel URI " + uri + " does not contain separator ://");
|
||||
|
||||
this.UriType = uri.Substring(0, sepindex);
|
||||
// some HPC URIs use the compression scheme as an "option" (not really defined for file:// uris, but...)
|
||||
|
|
@ -2694,7 +2742,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
uri = uri.Substring(0, option);
|
||||
}
|
||||
this.LocalPath = uri.Substring(sepindex + 3);
|
||||
this.LocalPath = uri.Substring(sepindex + 3).Trim();
|
||||
|
||||
if (uripathprefix != null) {
|
||||
// Unfortunately the uri is absolute, although it should be relative sometimes. We fix this here.
|
||||
|
|
@ -2737,10 +2785,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
this.Number = number;
|
||||
int sepindex = uri.IndexOf("://");
|
||||
if (sepindex < 0)
|
||||
throw new CalypsoClusterException("Channel URI " + uri + " does not contain separator ://");
|
||||
throw new ClusterException("Channel URI " + uri + " does not contain separator ://");
|
||||
this.UriType = uri.Substring(0, sepindex);
|
||||
this.LocalPath = uri.Substring(sepindex + 3);
|
||||
this.Size = size;
|
||||
this.TotalSize = size;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2883,7 +2931,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
this.StdoutFile = job.ClusterConfiguration.ProcessStdoutFile(this.ProcessIdentifier, false, machine, job.Summary);
|
||||
this.SetState(VertexState.Started);
|
||||
if (approxStartTime == DateTime.MinValue)
|
||||
throw new CalypsoDryadException("Unexpected small start time for vertex");
|
||||
throw new DryadException("Unexpected small start time for vertex");
|
||||
this.LogDirectory = job.ClusterConfiguration.ProcessLogDirectory(this.ProcessIdentifier, false, machine, job.Summary);
|
||||
this.LogFilesPattern = job.ClusterConfiguration.VertexLogFilesPattern(false, job.Summary);
|
||||
this.UniqueID = uniqueId;
|
||||
|
|
@ -3049,7 +3097,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
Trace.TraceInformation("Vertex {0} which is not started is still running?", this.Name);
|
||||
return;
|
||||
//throw new CalypsoClusterException("Vertex which is not started is still running?");
|
||||
//throw new ClusterException("Vertex which is not started is still running?");
|
||||
}
|
||||
if (this.Start > when)
|
||||
// This can happen if the cluster clocks are not synchronized with the local machine clocks.
|
||||
|
|
@ -3088,7 +3136,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
TimeSpan total = TimeSpan.FromSeconds(totSeconds);
|
||||
DateTime totalTime = creation + total;
|
||||
if (totSeconds < 0)
|
||||
throw new CalypsoDryadException("Negative total time for vertex " + this.Name);
|
||||
throw new DryadException("Negative total time for vertex " + this.Name);
|
||||
|
||||
// if the vertex has no machine just ignore the times
|
||||
if (string.IsNullOrEmpty(this.Machine))
|
||||
|
|
@ -3209,9 +3257,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <param name="uriprefix">If the channel is an output, prefix the path with this; this is null for inputs.</param>
|
||||
/// <param name="skip">If true, do not return anything (still useful to advance the stream reader).</param>
|
||||
/// <param name="fast">If true the channel sizes are not discovered; this is much faster, since no remote machines are queried for files.</param>
|
||||
/// <param name="updateProgress">Delegate used to report progress.</param>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
private Dictionary<int, ChannelEndpointDescription> DiscoverOriginalInfoChannels(ISharedStreamReader sr, string uriprefix, bool skip, bool fast, StatusReporter reporter, Action<int> updateProgress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
private Dictionary<int, ChannelEndpointDescription> DiscoverOriginalInfoChannels(ISharedStreamReader sr, string uriprefix, bool skip, bool fast, CommManager manager)
|
||||
{
|
||||
bool isInput = uriprefix == null;
|
||||
|
||||
|
|
@ -3231,21 +3278,18 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
string channel = sr.ReadLine();
|
||||
if (channel == null)
|
||||
{
|
||||
if (updateProgress != null)
|
||||
updateProgress(100);
|
||||
manager.Progress(100);
|
||||
return null;
|
||||
}
|
||||
if (!skip)
|
||||
{
|
||||
ChannelEndpointDescription desc = new ChannelEndpointDescription(isInput, i, channel, uriprefix, fast, reporter);
|
||||
ChannelEndpointDescription desc = new ChannelEndpointDescription(isInput, i, channel, uriprefix, fast, manager.Status);
|
||||
channels.Add(i, desc);
|
||||
if (updateProgress != null)
|
||||
updateProgress(i * 100 / channelCount);
|
||||
manager.Progress(i * 100 / channelCount);
|
||||
}
|
||||
}
|
||||
|
||||
if (updateProgress != null)
|
||||
updateProgress(100);
|
||||
manager.Progress(100);
|
||||
if (skip)
|
||||
return null;
|
||||
return channels;
|
||||
|
|
@ -3258,9 +3302,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <param name="inputs">If true discover the inputs.</param>
|
||||
/// <param name="outputs">If true discover the outputs.</param>
|
||||
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
|
||||
/// <param name="progress">Delegate used to report progress.</param>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
public bool DiscoverOriginalInfoChannels(bool inputs, bool outputs, bool fast, StatusReporter reporter, Action<int> progress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public bool DiscoverOriginalInfoChannels(bool inputs, bool outputs, bool fast, CommManager manager)
|
||||
{
|
||||
string filename = string.Format("vertex-{0}-{1}-rerun-originalInfo.txt", this.Number, this.Version);
|
||||
bool success = true;
|
||||
|
|
@ -3269,8 +3312,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
if (this.InputChannels != null)
|
||||
// skip discovery
|
||||
inputs = false;
|
||||
ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream();
|
||||
var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, reporter, progress);
|
||||
ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream(false);
|
||||
var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, manager);
|
||||
if (channels == null)
|
||||
{
|
||||
if (inputs)
|
||||
|
|
@ -3281,7 +3324,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
if (this.OutputChannels != null)
|
||||
// skip discovery
|
||||
outputs = false;
|
||||
channels = this.DiscoverOriginalInfoChannels(sr, this.WorkDirectory.ToString(), !outputs, fast, reporter, progress);
|
||||
channels = this.DiscoverOriginalInfoChannels(sr, this.WorkDirectory.ToString(), !outputs, fast, manager);
|
||||
if (channels == null)
|
||||
{
|
||||
if (outputs)
|
||||
|
|
@ -3300,28 +3343,27 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <param name="inputs">If true discover the inputs.</param>
|
||||
/// <param name="outputs">If true discover the outputs.</param>
|
||||
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
|
||||
/// <param name="progress">Delegate used to report progress.</param>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
// ReSharper disable UnusedParameter.Global
|
||||
public bool DiscoverScopeChannels(bool inputs, bool outputs, bool fast, StatusReporter reporter, Action<int> progress)
|
||||
public bool DiscoverScopeChannels(bool inputs, bool outputs, bool fast, CommManager manager)
|
||||
// ReSharper restore UnusedParameter.Global
|
||||
{
|
||||
// find the xml file
|
||||
var files = this.WorkDirectory.GetFilesAndFolders("vcmdStart*.xml").ToList();
|
||||
if (files.Count != 1)
|
||||
{
|
||||
reporter("Cannot locate vcmdStart*.xml file", StatusKind.Error);
|
||||
manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error);
|
||||
return false;
|
||||
}
|
||||
ISharedStreamReader sr = files.First().GetStream();
|
||||
ISharedStreamReader sr = files.First().GetStream(false);
|
||||
if (sr.Exception != null)
|
||||
{
|
||||
reporter("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);
|
||||
manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);
|
||||
return false;
|
||||
}
|
||||
|
||||
// ReSharper disable PossibleNullReferenceException
|
||||
XDocument plan = XDocument.Parse(sr.ReadToEnd());
|
||||
XDocument plan = XDocument.Parse(sr.ReadToEnd(manager.Token));
|
||||
if (inputs && this.InputChannels == null)
|
||||
{
|
||||
var channels = new Dictionary<int, ChannelEndpointDescription>();
|
||||
|
|
@ -3365,9 +3407,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
/// <param name="inputs">If true discover the inputs.</param>
|
||||
/// <param name="outputs">If true discover the outputs.</param>
|
||||
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
|
||||
/// <param name="progress">Delegate used to report progress.</param>
|
||||
/// <param name="reporter">Delegate used to report errors.</param>
|
||||
public bool DiscoverChannels(bool inputs, bool outputs, bool fast, StatusReporter reporter, Action<int> progress)
|
||||
/// <param name="manager">Communication manager.</param>
|
||||
public bool DiscoverChannels(bool inputs, bool outputs, bool fast, CommManager manager)
|
||||
{
|
||||
// check if the result is already cached
|
||||
if ((this.InputChannels != null || !inputs) &&
|
||||
|
|
@ -3395,7 +3436,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
|
||||
if (wd is UNCFile)
|
||||
{
|
||||
result = this.DiscoverOriginalInfoChannels(inputs, outputs, fast, reporter, progress);
|
||||
result = this.DiscoverOriginalInfoChannels(inputs, outputs, fast, manager);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -3448,9 +3489,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
internal void Update(string name, string guid)
|
||||
{
|
||||
if (this.State != VertexState.Cancelled && this.State != VertexState.Abandoned)
|
||||
throw new CalypsoDryadException("Updating a non-cancelled/abandoned vertex");
|
||||
throw new DryadException("Updating a non-cancelled/abandoned vertex");
|
||||
if (this.Name != name)
|
||||
throw new CalypsoDryadException("Vertex changed name");
|
||||
throw new DryadException("Vertex changed name");
|
||||
this.UniqueID = guid;
|
||||
this.SetState(VertexState.Created);
|
||||
// the stdoutfile is expected to change, so I don't invalidate the cache
|
||||
|
|
|
|||
|
|
@ -1,13 +1,25 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl.Build" version="1.0.21" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.3.2.0" targetFramework="net45" />
|
||||
<package id="Microsoft.Net.Http" version="2.2.28" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Client" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.WebHdfs" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Common" version="1.3.0" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.1.0" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.3.2.0" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.Storage" version="3.0.0" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.2" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -19,16 +19,14 @@ limitations under the License.
|
|||
|
||||
*/
|
||||
|
||||
#undef USE_DSC
|
||||
#undef USE_TIDYFS
|
||||
|
||||
using Microsoft.Research.Calypso.Tools;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Microsoft.Research.Tools;
|
||||
|
||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
||||
namespace Microsoft.Research.JobObjectModel
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -197,7 +195,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
|||
{
|
||||
var sourceInputs = plan.GetStageConnections(source, true).ToList();
|
||||
if (sourceInputs.Count() != 1)
|
||||
throw new CalypsoDryadException("Unexpected number of inputs for stage " + source.Name);
|
||||
throw new DryadException("Unexpected number of inputs for stage " + source.Name);
|
||||
source = sourceInputs.First().From;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 2013
|
||||
VisualStudioVersion = 12.0.30110.0
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
# Visual Studio 2012
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JobBrowser", "JobBrowser\JobBrowser.csproj", "{EDDD2E0B-A52B-4E25-9436-B874017673FF}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools", "Tools\Tools.csproj", "{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}"
|
||||
|
|
@ -13,72 +11,26 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UsefulForms", "UsefulForms\
|
|||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Debug|Mixed Platforms = Debug|Mixed Platforms
|
||||
Debug|x64 = Debug|x64
|
||||
Debug|x86 = Debug|x86
|
||||
Release|Any CPU = Release|Any CPU
|
||||
Release|Mixed Platforms = Release|Mixed Platforms
|
||||
Release|x64 = Release|x64
|
||||
Release|x86 = Release|x86
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x64.Build.0 = Debug|x64
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x64.ActiveCfg = Release|x64
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x64.Build.0 = Release|x64
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x64.Build.0 = Debug|x64
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x64.ActiveCfg = Release|x64
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x64.Build.0 = Release|x64
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x64.Build.0 = Debug|x64
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x64.ActiveCfg = Release|x64
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x64.Build.0 = Release|x64
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x64.Build.0 = Debug|x64
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x64.ActiveCfg = Release|x64
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x64.Build.0 = Release|x64
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue