berkus
/
Dryad
mirror of https://github.com/MicrosoftResearch/Dryad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: berkus:v0.1.5
Branches Tags
berkus:master
berkus:gh-pages
berkus:Release_0.2
berkus:v0.2.1-fixed
berkus:v0.2.1
berkus:v0.2.0
berkus:v0.1.5
berkus:v0.1.3
berkus:v0.1.2
berkus:v0.1.1
berkus:v0.1.0
...
compare: berkus:master
Branches Tags
berkus:gh-pages
berkus:master
berkus:Release_0.2
berkus:v0.2.1-fixed
berkus:v0.2.1
berkus:v0.2.0
berkus:v0.1.5
berkus:v0.1.3
berkus:v0.1.2
berkus:v0.1.1
berkus:v0.1.0

10 Commits
v0.1.5 ... master

Author SHA1 Message Date
misard ad28579dd8 Merge branch 'Release_0.2' 2014-10-23 10:32:28 -07:00
misard 6980be9895 Added missing documentation file 2014-10-23 10:29:37 -07:00
misard e72fef34c6 Merge branch 'Release_0.2' 2014-10-23 10:08:51 -07:00
Michael Isard 04ecff4353 Updated README 2014-10-23 09:50:48 -07:00
misard 5e9dde8ab6 Merge branch 'Release_0.2' 2014-10-06 19:26:04 -07:00
Michael Isard 47d0131083 updated versions 2014-10-06 19:13:07 -07:00
misard b31fadd55e updated with fix for azure subscription lookup 2014-10-06 19:13:06 -07:00
misard d3643fc5fb Merge branch 'Release_0.2' 2014-09-30 16:16:12 -07:00
misard efc79b749b Release 0.2 edits 2014-09-30 16:15:56 -07:00
Michael Isard b9cda36f69 release 0.2.0 2014-09-30 15:03:43 -07:00
149 changed files with 5906 additions and 3217 deletions
Show Stats Expand all files Collapse all files

118
ClusterInterface/ClusterInterface.csproj
View File

@ -1,6 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -9,10 +10,11 @@
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Dryad.ClusterInterface</RootNamespace>
<AssemblyName>DryadHttpClusterInterface</AssemblyName>
<AssemblyName>Microsoft.Research.Dryad.HttpClusterInterface</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
<NuGetPackageImportStamp>0025dfe5</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
@ -33,99 +35,12 @@
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
@ -134,29 +49,34 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="Interfaces.cs" />
<Compile Include="HttpScheduler.cs" />
<Compile Include="Process.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
<None Include="packages.config" />
<None Include="app.config">
<SubType>Designer</SubType>
</None>
<None Include="packages.config">
<SubType>Designer</SubType>
</None>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

27
ClusterInterface/Properties/AssemblyInfo.cs
View File

@ -24,32 +24,7 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("DryadHttpClusterInterface")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyProduct("Dryad")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
[assembly: AssemblyTitle("Microsoft.Research.Dryad.HttpClusterInterface")]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("9d262d5f-f6c5-4f9c-a1cc-7568cb399bf9")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

24
ClusterInterface/app.config
View File

@ -4,24 +4,40 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

20
ClusterInterface/packages.config
View File

@ -1,21 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
</packages>

26
Dryad.sln
View File

@ -1,6 +1,8 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2012
# Visual Studio 2013
VisualStudioVersion = 12.0.30723.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DryadVertex", "DryadVertex", "{99F5E7FE-ADD4-4E9D-BA90-5D3D3409BEB3}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "VertexHost", "VertexHost", "{37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D}"
@ -27,7 +29,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dprocess", "DryadVertex\Ver
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ManagedWrapperVertex", "DryadVertex\VertexHost\vertex\managedwrappervertex\ManagedWrapperVertex.vcxproj", "{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}"
EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHost", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}"
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHostBody", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}"
ProjectSection(ProjectDependencies) = postProject
{AA529122-F51C-48D7-A8C1-C0B24F570885} = {AA529122-F51C-48D7-A8C1-C0B24F570885}
{482E0741-E244-4974-97D4-3A7167581E91} = {482E0741-E244-4974-97D4-3A7167581E91}
@ -78,6 +80,19 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalScheduler", "LocalSche
{A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6} = {A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6}
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VertexHostMain", "DryadVertex\VertexHost\vertex\VertexHostMain\VertexHostMain.csproj", "{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "NuSpec", "NuSpec", "{35121ABA-5FD2-4E1D-9C05-033D78D824F3}"
ProjectSection(SolutionItems) = preProject
Microsoft.Research.Dryad.nuspec = Microsoft.Research.Dryad.nuspec
Microsoft.Research.Dryad.targets = Microsoft.Research.Dryad.targets
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Misc", "Misc", "{D361F025-2F7F-4C1A-8FBC-4AD826CC656D}"
ProjectSection(SolutionItems) = preProject
SharedAssemblyInfo.cs = SharedAssemblyInfo.cs
EndProjectSection
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64
@ -144,6 +159,10 @@ Global
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Debug|x64.Build.0 = Debug|x64
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.ActiveCfg = Release|x64
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.Build.0 = Release|x64
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Debug|x64.ActiveCfg = Debug|x64
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Debug|x64.Build.0 = Debug|x64
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Release|x64.ActiveCfg = Release|x64
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -156,11 +175,12 @@ Global
{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{57663B94-E11B-431E-BE4B-E2C61112DEC5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{AA529122-F51C-48D7-A8C1-C0B24F570885} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{A0033286-9C5F-4113-BAA5-58A1274F95C5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{BDEDD3BB-C7E2-498F-A212-F99786C8E23C} = {DC277807-506B-4F7C-BECD-345079B91044}
{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37} = {DC277807-506B-4F7C-BECD-345079B91044}
{AB9EA66C-5811-49A7-B002-24203AEB9083} = {DC277807-506B-4F7C-BECD-345079B91044}
{3EE0920C-0607-4569-9EC3-5C12BB6EF244} = {DC277807-506B-4F7C-BECD-345079B91044}
{89C5654B-02E4-478D-A7E6-50D79F638B4F} = {DC277807-506B-4F7C-BECD-345079B91044}
{A0033286-9C5F-4113-BAA5-58A1274F95C5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352} = {DC277807-506B-4F7C-BECD-345079B91044}
EndGlobalSection
EndGlobal

113
DryadLinqGraphManager/DryadLinqGraphManager.csproj
View File

@ -1,6 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -10,7 +13,7 @@
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Dryad.GraphManager</RootNamespace>
<AssemblyName>DryadLinqGraphManager</AssemblyName>
<AssemblyName>Microsoft.Research.Dryad.GraphManager</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<FileUpgradeFlags>
@ -33,6 +36,7 @@
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<NuGetPackageImportStamp>5c14de70</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
@ -53,80 +57,28 @@
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
<HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
<HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core">
@ -138,18 +90,10 @@
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
<HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq">
@ -162,6 +106,9 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="DryadLinqApplication.cs" />
<Compile Include="GraphBuilder.cs" />
<Compile Include="LinqToDryadException.cs" />
@ -203,19 +150,23 @@
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

16
DryadLinqGraphManager/GraphBuilder.cs
View File

@ -92,7 +92,7 @@ namespace Microsoft.Research.Dryad.GraphManager
err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read DSC input file {0}", info.sources[0]);
string msg = String.Format("Could not read Partitioned file input {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
@ -127,7 +127,9 @@ namespace Microsoft.Research.Dryad.GraphManager
DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");
err = input.Open(app.GetUniverse(), info.sources[0]);
Uri srcUri = new Uri(info.sources[0]);
err = input.Open(app.GetUniverse(), srcUri.GetLeftPart(UriPartial.Path), info.recordType);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError());
@ -182,10 +184,16 @@ namespace Microsoft.Research.Dryad.GraphManager
if ( info.ioType == VertexInfo.IOType.PARTITIONEDFILE )
{
DrPartitionOutputStream output = new DrPartitionOutputStream();
int err = output.Open(new Uri(info.sources[0]).AbsolutePath, info.partitionUncPath);
Uri sourceUri = new Uri(info.sources[0]);
string sourcePath = sourceUri.AbsolutePath;
if (!String.IsNullOrEmpty(sourceUri.Host))
{
sourcePath = @"\\" + sourceUri.Host + sourcePath;
}
int err = output.Open(sourcePath, info.partitionUncPath);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]);
string msg = String.Format("Could not open output fileset {0}", sourcePath);
throw new LinqToDryadException(msg, err);
}

60
DryadLinqGraphManager/LinqToDryadJM.cs
View File

@ -73,12 +73,12 @@ namespace Microsoft.Research.Dryad.GraphManager
public static Uri AddDfsJobDirectoryFromArgs(string arg)
{
string jobId = Microsoft.Research.Peloponnese.Storage.AzureUtils.ApplicationIdFromEnvironment();
string jobDirectoryBase = Microsoft.Research.Peloponnese.Storage.AzureUtils.CmdLineDecode(arg.Substring(arg.IndexOf('=') + 1));
string jobId = Microsoft.Research.Peloponnese.Yarn.Utils.ApplicationIdFromEnvironment();
string jobDirectoryBase = Microsoft.Research.Peloponnese.Utils.CmdLineDecode(arg.Substring(arg.IndexOf('=') + 1));
string jobDirectorySpecific = jobDirectoryBase.Replace("_JOBID_", jobId);
UriBuilder logLocation = new UriBuilder(jobDirectorySpecific);
logLocation.Path = logLocation.Path + "calypso.log";
logLocation.Path = logLocation.Path.TrimEnd('/') + "/calypso.log";
DebugHelper.AddReporter(new DrCalypsoReporter(logLocation.Uri.AbsoluteUri));
@ -94,7 +94,7 @@ namespace Microsoft.Research.Dryad.GraphManager
{
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
{
string dfsPath = dfsDirectory.AbsolutePath + Path.GetFileName(localPath);
string dfsPath = dfsDirectory.AbsolutePath.TrimEnd('/') + "/" + Path.GetFileName(localPath);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
hdfs.UploadAll(localPath, dfsPath);
}
@ -102,11 +102,12 @@ namespace Microsoft.Research.Dryad.GraphManager
else if (dfsDirectory.Scheme == "azureblob")
{
string account, key, container, blob;
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container);
string dfsPath = blob + Path.GetFileName(localPath);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
azure.PutDfsFile(localPath, dfsPath);
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
string dfsPath = blob.TrimEnd('/') + "/" + Path.GetFileName(localPath);
Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsUri.AbsoluteUri);
azure.PutDfsFile(dfsUri, localPath);
}
else if (dfsDirectory.Scheme == "file")
{
@ -138,10 +139,11 @@ namespace Microsoft.Research.Dryad.GraphManager
else if (dfsDirectory.Scheme == "azureblob")
{
string account, key, container, blob;
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container);
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
string dfsPath = blob + dfsName;
azure.PutDfsFile(payloadBytes, dfsPath);
Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
azure.PutDfsFile(dfsUri, payloadBytes);
}
else if (dfsDirectory.Scheme == "file")
{
@ -207,7 +209,39 @@ namespace Microsoft.Research.Dryad.GraphManager
logDir = logDir.Split(',').First().Trim();
DrLogging.Initialize(Path.Combine(logDir, "graphmanager"), false);
}
DrLogging.SetLoggingLevel(DrLogTypeManaged.Info);
string logLevel = Environment.GetEnvironmentVariable("DRYAD_LOGGING_LEVEL");
if (logLevel == null)
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Debug);
}
else
{
if (logLevel == "OFF")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Off);
}
else if (logLevel == "CRITICAL")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Assert);
}
else if (logLevel == "ERROR")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Error);
}
else if (logLevel == "WARN")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Warning);
}
else if (logLevel == "INFO")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Info);
}
else
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Debug);
}
}
// Report start time to Artemis - must come after
// DrLogging is initialized so stdout is redirected

7
DryadLinqGraphManager/Program.cs
View File

@ -28,6 +28,13 @@ using Microsoft.Research.Dryad;
namespace Microsoft.Research.Dryad.GraphManager
{
internal class ConfigDependency : Microsoft.Research.Peloponnese.Shared.AssemblyDependencyAttribute
{
public ConfigDependency() : base("Microsoft.Research.Dryad.GraphManager.exe.config", false)
{
}
}
class Program
{
static Uri dfsDirectory;

28
DryadLinqGraphManager/Properties/AssemblyInfo.cs
View File

@ -22,34 +22,12 @@ using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
[assembly: Microsoft.Research.Dryad.GraphManager.ConfigDependency()]
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("DryadLinqGraphManager")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyProduct("Dryad")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
[assembly: AssemblyTitle("Microsoft.Research.Dryad.GraphManager")]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("55c7f4b4-02ab-4309-b276-3c9ab5a927e0")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

6
DryadLinqGraphManager/QueryParser.cs
View File

@ -237,6 +237,12 @@ namespace Microsoft.Research.Dryad.GraphManager
vertex.info = new VertexInfo();
vertex.info.ioType = GetIoType(ioType);
XmlNode recordType = storageSet.SelectSingleNode("RecordType");
if (recordType != null)
{
vertex.info.recordType = recordType.InnerXml;
}
XmlNodeList storageUris = storageSet.SelectNodes("SourceURI");
vertex.info.sources = new string[storageUris.Count];
for (int indexStorageUri=0; indexStorageUri<storageUris.Count; indexStorageUri++)

20
DryadLinqGraphManager/app.config
View File

@ -7,19 +7,19 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
<bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
@ -27,11 +27,19 @@
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.0.7" newVersion="1.1.0.7" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.0.7" newVersion="1.1.0.7" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
</assemblyBinding>
</runtime>

27
DryadLinqGraphManager/packages.config
View File

@ -1,21 +1,14 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
</packages>

21
DryadLinqTests/ApplyAndForkTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq;
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;

21
DryadLinqTests/BasicAPITests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq;
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;

44
DryadLinqTests/DryadLinqTests.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -12,9 +12,26 @@
<AssemblyName>DryadLinqTests</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
<RestorePackages>true</RestorePackages>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget>
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
@ -187,6 +204,29 @@
<None Include="App.config" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\LinqToDryad\LinqToDryad.csproj">
<Project>{d33c34cc-6db2-417c-88b7-299830711774}</Project>
<Name>LinqToDryad</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<BootstrapperPackage Include=".NETFramework,Version=v4.5">
<Visible>False</Visible>
<ProductName>Microsoft .NET Framework 4.5 %28x86 and x64%29</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>false</Install>
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">

21
DryadLinqTests/GroupByReduceTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq;
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;

21
DryadLinqTests/MiscBugFixTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq;
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;

19
DryadLinqTests/Properties/AssemblyInfo.cs
View File

@ -1,3 +1,22 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

21
DryadLinqTests/RangePartitionAPICoverageTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq;
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.DryadLinq.Internal;
using Microsoft.Research.Peloponnese.Storage;
using System;

21
DryadLinqTests/SerializationTestTypes.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq;
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using System;
using System.Collections.Generic;
using System.Linq;

21
DryadLinqTests/SerializationTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq;
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;

21
DryadLinqTests/SimpleTests.cs
View File

@ -1,4 +1,23 @@
using System;
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

21
DryadLinqTests/TestLog.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq;
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;

21
DryadLinqTests/TypesInQueryTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq;
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;

19
DryadLinqTests/Utils.cs
View File

@ -1,3 +1,22 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.DryadLinq.Internal;
using Microsoft.Research.Peloponnese.Storage;

19
DryadLinqTests/WordCount.cs
View File

@ -1,3 +1,22 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
// comment the following line to run on Azure
#define local

37
DryadVertex/VertexHost/system/channel/channel.vcxproj
View File

@ -1,6 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -24,13 +25,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -54,6 +55,13 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<NuGetPackageImportStamp>b0b5ea03</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl>
@ -61,7 +69,7 @@
</Midl>
<ClCompile>
<Optimization>Disabled</Optimization>
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\..\peloponnese\HadoopBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>WIN32;_DEBUG;_LIB;WIN32_LEAN_AND_MEAN;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<MinimalRebuild>false</MinimalRebuild>
<BasicRuntimeChecks>Default</BasicRuntimeChecks>
@ -72,10 +80,6 @@
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile>
<Lib>
<AdditionalDependencies>Microsoft.Research.Peloponnese.HadoopBridge.lib</AdditionalDependencies>
<AdditionalLibraryDirectories>..\..\..\..\..\peloponnese\bin\$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Midl>
@ -88,13 +92,9 @@
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\..\peloponnese\HadoopBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile>
<Lib>
<AdditionalDependencies>Microsoft.Research.Peloponnese.HadoopBridge.lib</AdditionalDependencies>
<AdditionalLibraryDirectories>..\..\..\..\..\peloponnese\bin\$(Configuration)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="include\channelbuffer.h" />
@ -149,13 +149,16 @@
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
</ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
</Target>
</Project>

3
DryadVertex/VertexHost/system/channel/packages.config
View File

@ -1,4 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
</packages>

48
DryadVertex/VertexHost/system/channel/src/channelbufferhdfs.cpp
View File

@ -929,6 +929,37 @@ RChannelBufferHdfsWriter::RChannelBufferHdfsWriter(const char* uri)
Hdfs::InstanceAccessor ia(bridge);
ia.Discard();
}
char envUserName[128];
DWORD ret = ::GetEnvironmentVariableA("USER", envUserName, sizeof(envUserName));
if (ret > 0 && ret < sizeof(envUserName)-1)
{
m_user.Set(envUserName);
DrLogI("Hdfs writer set user to %s from environment", m_user.GetString());
}
else
{
if (ret == 0 && GetLastError() != ERROR_ENVVAR_NOT_FOUND)
{
DrLogW("Can't get USER environment variable: %s", DRERRORSTRING(DrGetLastError()));
}
if (ret >= sizeof(envUserName)-1)
{
DrLogW("USER environment is too long: %d", ret);
}
DWORD bufSize = sizeof(envUserName);
BOOL bRet = ::GetUserNameA(envUserName, &bufSize);
if (bRet == 0)
{
DrLogW("Can't get username: %s", DRERRORSTRING(DrGetLastError()));
}
else
{
m_user.Set(envUserName);
DrLogI("Hdfs writer set user to %s from username", m_user.GetString());
}
}
}
}
}
@ -1021,7 +1052,7 @@ bool RChannelBufferHdfsWriter::Open(Hdfs::Instance** pInstance,
Hdfs::InstanceAccessor ia(instance);
Hdfs::Writer* writer;
bool openedWriter = ia.OpenWriter(filePath.GetString(), false, &writer);
bool openedWriter = ia.OpenCreate(filePath.GetString(), 1024 * 1024, -1, &writer);
if (!openedWriter)
{
char* errorMsg = ia.GetExceptionMessage();
@ -1036,6 +1067,21 @@ bool RChannelBufferHdfsWriter::Open(Hdfs::Instance** pInstance,
ia.Discard();
return false;
}
bool setPermissions = ia.SetOwnerAndPermission(filePath.GetString(), m_user.GetString(), NULL, 0644);
if (!setPermissions)
{
char* errorMsg = ia.GetExceptionMessage();
DrStr64 description;
description.SetF("Can't set HDFS file permissions '%s': %s",
m_uri.GetString(), errorMsg);
HadoopNative::DisposeString(errorMsg);
m_completionItem.Attach(MakeErrorItem(DryadError_ChannelOpenError,
description.GetString()));
ia.Discard();
return false;
}
DrLogI("Opened Hdfs writer for %s", m_uri.GetString());

1
DryadVertex/VertexHost/system/channel/src/channelbufferhdfs.h
View File

@ -149,6 +149,7 @@ private:
bool Open(Hdfs::Instance** pInstance, Hdfs::Writer** pWriter);
bool AddToQueue(WriteEntry* entry);
DrStr64 m_user;
DrStr64 m_uri;
DryadBList<WriteEntry> m_queue;
UInt32 m_queueLength;

2
DryadVertex/VertexHost/system/channel/src/channelbuffernativewriter.cpp
View File

@ -2205,7 +2205,7 @@ void RChannelBufferWriterNativeFile::EagerCloseFile()
}
else
{
DrLogI( "Set final file pointer");
DrLogI( "Set final file pointer. Length %I64u", finalLength.QuadPart);
}
ok = SetEndOfFile(m_bufferedFileHandle);

10
DryadVertex/VertexHost/system/classlib/classlib.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -21,13 +21,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>false</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>false</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -150,4 +150,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
</Project>

10
DryadVertex/VertexHost/system/common/common.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -19,14 +19,14 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>MultiByte</CharacterSet>
<CLRSupport>false</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -138,4 +138,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
</Project>

37
DryadVertex/VertexHost/system/dprocess/dprocess.vcxproj
View File

@ -1,6 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -22,7 +24,7 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
<CLRSupport>true</CLRSupport>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
@ -30,7 +32,7 @@
<ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -54,6 +56,13 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<NuGetPackageImportStamp>7f5f48b5</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl>
@ -72,10 +81,6 @@
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile>
<Lib>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Midl>
@ -91,10 +96,6 @@
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;..\channel\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile>
<Lib>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup>
<ItemGroup>
<ClInclude Include="include\dryadvertex.h" />
@ -123,13 +124,19 @@
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
</ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
</Target>
</Project>

4
DryadVertex/VertexHost/system/dprocess/packages.config
View File

@ -1,4 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="Native" />
</packages>

16
DryadVertex/VertexHost/system/managedchannel/AzureReader.cs
View File

@ -27,7 +27,7 @@ using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Research.Peloponnese.Storage;
using Microsoft.Research.Peloponnese.Azure;
namespace Microsoft.Research.Dryad.Channel
{
@ -74,7 +74,7 @@ namespace Microsoft.Research.Dryad.Channel
{
Log.LogInformation("Opening read for " + source.AbsoluteUri);
string account, key, container, blobName;
AzureUtils.FromAzureUri(source, out account, out key, out container, out blobName);
Utils.FromAzureUri(source, out account, out key, out container, out blobName);
client = new AzureDfsClient(account, key, container, false, new PeloponneseLogger(Log.Logger));
client.SetParallelThreadCount(4);
@ -94,7 +94,7 @@ namespace Microsoft.Research.Dryad.Channel
private async Task OpenBlob()
{
string account, key, container, blobName;
AzureUtils.FromAzureUri(source, out account, out key, out container, out blobName);
Utils.FromAzureUri(source, out account, out key, out container, out blobName);
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(source.Query);
@ -105,9 +105,11 @@ namespace Microsoft.Research.Dryad.Channel
++blobIndex;
}
Log.LogInformation("Opening read for blob " + blobName);
Uri blobUri = Utils.ToAzureUri(account, container, blobName, null, key);
Log.LogInformation("Opening read for blob " + blobUri.AbsoluteUri);
readStream = (await client.GetDfsFileStreamAsync(blobName, ExecutionTimeout, new PeloponneseLogger(Log.Logger))).Stream;
readStream = (await client.GetDfsStreamReaderAsync(blobUri, ExecutionTimeout, new PeloponneseLogger(Log.Logger)));
long offset = -1;
if (query["offset"] != null)
@ -215,7 +217,7 @@ namespace Microsoft.Research.Dryad.Channel
int toRead = (int)Math.Min((long)managedBuffer.Length, bytesToRead);
Log.LogInformation("About to read buffer length " + toRead);
readData.nRead = await AzureUtils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
readData.nRead = await Utils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
{
return await readStream.ReadAsync(managedBuffer, 0, toRead);
});
@ -301,7 +303,7 @@ namespace Microsoft.Research.Dryad.Channel
{
int toRead = (int)Math.Min(buffer.LongLength, endOffset - currentOffset);
int nRead = await AzureUtils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
int nRead = await Utils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
{
return await readStream.ReadAsync(buffer, 0, toRead);
});

2
DryadVertex/VertexHost/system/managedchannel/AzureWriter.cs
View File

@ -25,7 +25,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Research.Peloponnese.Storage;
using Microsoft.Research.Peloponnese.Azure;
namespace Microsoft.Research.Dryad.Channel
{

26
DryadVertex/VertexHost/system/managedchannel/Properties/AssemblyInfo.cs
View File

@ -24,32 +24,8 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("managedchannel")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("dryad")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
[assembly: AssemblyTitle("Microsoft.Research.Dryad.ManagedChannel")]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("f806a093-c5ae-4752-82d3-26bcc2341b29")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

28
DryadVertex/VertexHost/system/managedchannel/app.config
View File

@ -4,24 +4,44 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
<bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.HDInsight.Threading" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.0" newVersion="1.2.1.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

109
DryadVertex/VertexHost/system/managedchannel/managedchannel.csproj
View File

@ -1,6 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -9,11 +11,12 @@
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Dryad.Channel</RootNamespace>
<AssemblyName>DryadManagedChannel</AssemblyName>
<AssemblyName>Microsoft.Research.Dryad.ManagedChannel</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir>
<RestorePackages>true</RestorePackages>
<NuGetPackageImportStamp>72e7670e</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
@ -35,98 +38,38 @@
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
<HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
<HintPath>..\..\..\..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" />
@ -136,6 +79,9 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\..\..\..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="AzureReader.cs" />
<Compile Include="AzureWriter.cs" />
<Compile Include="FileWriter.cs" />
@ -151,19 +97,20 @@
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
</Target>
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

26
DryadVertex/VertexHost/system/managedchannel/packages.config
View File

@ -1,21 +1,13 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
</packages>

43
DryadVertex/VertexHost/vertex/VertexHostMain/Program.cs Normal file
View File

@ -0,0 +1,43 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace VertexHostMain
{
internal class ConfigDependency : Microsoft.Research.Peloponnese.Shared.AssemblyDependencyAttribute
{
public
ConfigDependency()
: base("Microsoft.Research.Dryad.VertexHost.exe.config", false)
{
}
}
class Program
{
static int Main(string[] args)
{
return VertexHost.Main(args);
}
}
}

32
DryadVertex/VertexHost/vertex/VertexHostMain/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,32 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
[assembly: VertexHostMain.ConfigDependency()]
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Microsoft.Research.Dryad.VertexHost")]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("5534b858-8472-4dc3-8f24-5b59f8805156")]

79
DryadVertex/VertexHost/vertex/VertexHostMain/VertexHostMain.csproj Normal file
View File

@ -0,0 +1,79 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>VertexHostMain</RootNamespace>
<AssemblyName>Microsoft.Research.Dryad.VertexHost</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
<NuGetPackageImportStamp>fb5afcd2</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>..\..\..\..\bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
<OutputPath>..\..\..\..\bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\..\..\..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\vertexHost\VertexHost.vcxproj">
<Project>{0cf3d1d5-9bbe-4175-979b-ec6138ef4f37}</Project>
<Name>VertexHostBody</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
</Target>
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

48
DryadVertex/VertexHost/vertex/VertexHostMain/app.config Normal file
View File

@ -0,0 +1,48 @@
<?xml version="1.0"?>
<configuration>
<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/></startup>
<runtime>
<gcServer enabled="true"></gcServer>
<gcConcurrent enabled="false"></gcConcurrent>
<generatePublisherEvidence enabled="false"/>
<gcAllowVeryLargeObjects enabled="true" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

4
DryadVertex/VertexHost/vertex/VertexHostMain/packages.config Normal file
View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
</packages>

10
DryadVertex/VertexHost/vertex/WrapperNativeInfo/WrapperNativeInfo.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -21,13 +21,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -100,4 +100,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
</Project>

2
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/DryadLINQNativeChannels.def
View File

@ -1,4 +1,4 @@
LIBRARY "DryadLINQNativeChannels.dll"
LIBRARY "Microsoft.Research.Dryad.DryadLinq.NativeWrapper.dll"
EXPORTS
GetTotalLength

15
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/WrapperNativeInfoDll.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -19,14 +19,14 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
<CLRSupport>true</CLRSupport>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -52,8 +52,8 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">DryadLINQNativeChannels</TargetName>
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">DryadLINQNativeChannels</TargetName>
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad.DryadLinq.NativeWrapper</TargetName>
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad.DryadLinq.NativeWrapper</TargetName>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl>
@ -111,6 +111,7 @@
<ClInclude Include="stdafx.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="version.cpp" />
<ClCompile Include="wrappernativeinfostubs.cpp" />
</ItemGroup>
<ItemGroup>
@ -119,4 +120,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
</Project>

3
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/WrapperNativeInfoDll.vcxproj.filters
View File

@ -23,6 +23,9 @@
<ClCompile Include="wrappernativeinfostubs.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="version.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="DryadLINQNativeChannels.def">

40
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/version.cpp Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
#ifdef _MANAGED
#pragma managed
using namespace System::Reflection;
using namespace System::Runtime::InteropServices;
[assembly: AssemblyTitle("Microsoft.Research.Dryad.DryadLinq.NativeWrapper")];
[assembly: AssemblyDescription("")];
[assembly: AssemblyConfiguration("")];
[assembly: AssemblyCompany("Microsoft Corporation")];
[assembly: AssemblyProduct("Microsoft.Research.Dryad.DryadLinq.NativeWrapper")];
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
[assembly: AssemblyTrademark("")];
[assembly: AssemblyCulture("")];
[assembly: ComVisible(false)];
[assembly: AssemblyVersion("0.2.1.0")];
[assembly: AssemblyFileVersion("0.2.1.0")];
#endif

39
DryadVertex/VertexHost/vertex/managedwrappervertex/ManagedWrapperVertex.cpp
View File

@ -197,6 +197,14 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
logName = System::IO::Path::Combine(logDirectory, logName);
}
int threadsPerWorker = 1;
System::String^ threadsPerWorkerStr = System::Environment::GetEnvironmentVariable("DRYAD_THREADS_PER_WORKER");
if (threadsPerWorkerStr != nullptr)
{
threadsPerWorker = Int32::Parse(threadsPerWorkerStr);
}
DrLogI("ManagedWrapperVertex: threadsPerWorker %u", threadsPerWorker);
DrLogI("ManagedWrapperVertex: %p %u %u", nativeInfo, numberOfInputChannels, numberOfOutputChannels);
DrLogI("ManagedWrapperVertex: Calling %s.%s", GetArgument(2), GetArgument(3));
DrLogging::FlushLog();
@ -215,9 +223,9 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
// The format of vertexBridgeArgs is simply a comma separated string packing vertex assembly, class, method name, and the *actual* vertex method args (==the native channel string)
// L"<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
//
System::String^ classFullName = gcnew System::String(GetArgument(2));
System::String^ assemblyName = classFullName->Substring(0, classFullName->LastIndexOf('.'));
System::String ^bridgeAssemblyPartialName = gcnew System::String(assemblyName);
System::String^ classFullName = gcnew System::String(GetArgument(2));
System::String^ assemblyName = classFullName->Substring(0, classFullName->LastIndexOf('.'));
System::String ^bridgeAssemblyName = gcnew System::String(assemblyName);
System::String ^bridgeClassName = gcnew System::String(assemblyName + ".Internal.VertexEnv");
System::String ^bridgeMethodName = gcnew System::String(L"VertexBridge");
@ -240,7 +248,7 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
// "<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
//
System::Text::StringBuilder ^vertexBridgeArg = gcnew System::Text::StringBuilder();
vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"c:\\HpcTemp\\user\\jobID\\Microsoft.Hpc.Linq0.dll";
vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"\\HpcTemp\\user\\jobID\\Microsoft.Research.DryadLinq0.dll";
vertexBridgeArg->Append(",");
vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Research.DryadLinq.DryadLinq__Vertex";
vertexBridgeArg->Append(",");
@ -248,7 +256,6 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
vertexBridgeArg->Append(",");
vertexBridgeArg->Append(vertexMethodArgs->ToString());
DrLogI("ManagedWrapperVertex: Calling into Vertex Bridge to invoke Vertex Entry: %s", GetArgument(3));
DrLogging::FlushLog();
@ -259,12 +266,22 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
//
try
{
System::Console::WriteLine("Assembly name " + bridgeAssemblyPartialName);
System::Reflection::Assembly ^vertexBridgeAsm = System::Reflection::Assembly::LoadWithPartialName(bridgeAssemblyPartialName);
System::Type ^vertexBridgeType = vertexBridgeAsm->GetType(gcnew System::String(bridgeClassName));
System::Reflection::MethodInfo ^vertexBridgeMethod = vertexBridgeType->GetMethod(gcnew System::String(bridgeMethodName),
static_cast<System::Reflection::BindingFlags>(System::Reflection::BindingFlags::NonPublic |
System::Reflection::BindingFlags::Static));
System::Reflection::Assembly ^vertexBridgeAsm;
try
{
vertexBridgeAsm = System::Reflection::Assembly::Load(bridgeAssemblyName);
}
catch (System::Exception ^ie)
{
DrLogI("ManagedWrapperVertex: Failed to load assembly %s: %s", bridgeAssemblyName, ie->ToString());
System::String^ asmLoc = System::IO::Path::Combine("..", bridgeAssemblyName + ".dll");
vertexBridgeAsm = System::Reflection::Assembly::LoadFrom(asmLoc);
}
System::Type ^vertexBridgeType = vertexBridgeAsm->GetType(bridgeClassName);
System::Reflection::MethodInfo ^vertexBridgeMethod
= vertexBridgeType->GetMethod(bridgeMethodName,
static_cast<System::Reflection::BindingFlags>(System::Reflection::BindingFlags::NonPublic |
System::Reflection::BindingFlags::Static));
cli::array<System::Object^> ^invokeArgs = gcnew array<System::Object^>(2);
invokeArgs[0] = logName;

10
DryadVertex/VertexHost/vertex/managedwrappervertex/ManagedWrapperVertex.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -21,13 +21,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -99,4 +99,4 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
</Project>

27
DryadVertex/VertexHost/vertex/vertexHost/VertexHost.vcxproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -18,19 +18,20 @@
<Keyword>Win32Proj</Keyword>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir>
<RestorePackages>true</RestorePackages>
<ProjectName>VertexHostBody</ProjectName>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -56,6 +57,13 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<NuGetPackageImportStamp>26f18b05</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<TargetName>Microsoft.Research.Dryad.$(ProjectName)</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<TargetName>Microsoft.Research.Dryad.$(ProjectName)</TargetName>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl>
@ -112,6 +120,7 @@
</PostBuildEvent>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="version.cpp" />
<ClCompile Include="vertexHost.cpp" />
</ItemGroup>
<ItemGroup>
@ -124,13 +133,13 @@
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
</ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
</Target>
</Project>

1
DryadVertex/VertexHost/vertex/vertexHost/VertexHost.vcxproj.filters
View File

@ -2,6 +2,7 @@
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<ClCompile Include="vertexHost.cpp" />
<ClCompile Include="version.cpp" />
</ItemGroup>
<ItemGroup>
<None Include="app.config" />

36
DryadVertex/VertexHost/vertex/vertexHost/app.config
View File

@ -1,27 +1,43 @@
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<runtime>
<gcServer enabled="true"></gcServer>
<gcConcurrent enabled="false"></gcConcurrent>
<generatePublisherEvidence enabled="false"/>
<gcAllowVeryLargeObjects enabled="true" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

2
DryadVertex/VertexHost/vertex/vertexHost/packages.config
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
</packages>

40
DryadVertex/VertexHost/vertex/vertexHost/version.cpp Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
#ifdef _MANAGED
#pragma managed
using namespace System::Reflection;
using namespace System::Runtime::InteropServices;
[assembly: AssemblyTitle("Microsoft.Research.Dryad.VertexHost")];
[assembly: AssemblyDescription("")];
[assembly: AssemblyConfiguration("")];
[assembly: AssemblyCompany("Microsoft Corporation")];
[assembly: AssemblyProduct("Microsoft.Research.Dryad.VertexHost")];
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
[assembly: AssemblyTrademark("")];
[assembly: AssemblyCulture("")];
[assembly: ComVisible(false)];
[assembly: AssemblyVersion("0.2.1.0")];
[assembly: AssemblyFileVersion("0.2.1.0")];
#endif

209
DryadVertex/VertexHost/vertex/vertexHost/vertexHost.cpp
View File

@ -174,27 +174,27 @@ void SetLoggingLevel()
GetLoggingFileName(logFileName);
DrLogging::Initialize(logFileName);
WCHAR traceLevel [MAX_PATH];
HRESULT hr = DrGetEnvironmentVariable(L"DRYAD_TRACE_LEVEL", traceLevel);
WCHAR loggingLevel [MAX_PATH];
HRESULT hr = DrGetEnvironmentVariable(L"DRYAD_LOGGING_LEVEL", loggingLevel);
if(hr == DrError_OK)
{
if(wcscmp(traceLevel, L"OFF") == 0)
if(wcscmp(loggingLevel, L"OFF") == 0)
{
DrLogging::SetLoggingLevel(LogLevel_Off);
}
else if(wcscmp(traceLevel, L"CRITICAL") == 0)
else if(wcscmp(loggingLevel, L"CRITICAL") == 0)
{
DrLogging::SetLoggingLevel(LogLevel_Assert);
}
else if(wcscmp(traceLevel, L"ERROR") == 0)
else if(wcscmp(loggingLevel, L"ERROR") == 0)
{
DrLogging::SetLoggingLevel(LogLevel_Error);
}
else if(wcscmp(traceLevel, L"WARN") == 0)
else if(wcscmp(loggingLevel, L"WARN") == 0)
{
DrLogging::SetLoggingLevel(LogLevel_Warning);
}
else if(wcscmp(traceLevel, L"INFO") == 0)
else if(wcscmp(loggingLevel, L"INFO") == 0)
{
DrLogging::SetLoggingLevel(LogLevel_Info);
}
@ -249,104 +249,119 @@ static void ExceptionHandler(System::Object^ sender, System::UnhandledExceptionE
//
// Start up vertex host
//
[System::Security::SecurityCriticalAttribute]
[System::Runtime::ExceptionServices::HandleProcessCorruptedStateExceptionsAttribute]
#if defined(_AMD64_)
int wmain(int argc, wchar_t* wargv[])
#else
int __cdecl wmain(int argc, wchar_t* wargv[])
#endif
public ref class VertexHost
{
try
public:
[System::Security::SecurityCriticalAttribute]
[System::Runtime::ExceptionServices::HandleProcessCorruptedStateExceptionsAttribute]
static int Main(array<System::String^>^ managedArgs)
{
//
// Enable logging based on environment variable
//
SetLoggingLevel();
DrInitErrorTable();
DrInitExitCodeTable();
DrInitLastAccessTable();
// Set unhandled exception handler to catch anything thrown from
// managed code
System::AppDomain^ currentDomain = System::AppDomain::CurrentDomain;
currentDomain->UnhandledException += gcnew System::UnhandledExceptionEventHandler(ExceptionHandler);
//
// trace for startup
//
DrLogI("Vertex Host starting");
//
// Get environment variable to know whether to break into debugger
//
BreakForDebugger();
//
// We call Register on the Managed Wrapper vertex factory to force its library to be linked.
// Registration actually occurs during static initialization.
//
s_factoryHWrapper.Register();
//
// Get command line arguments
//
char** argv;
DrGetUtf8CommandArgs(argc, wargv, &argv);
//
// Initialize the dryad communication layer with the command line arguments
//
int nOpts;
DrError e;
e = DryadInitialize(argc, argv, &nOpts);
if (e != DrError_OK)
try
{
int argc = managedArgs->Length;
wchar_t** wargv = new wchar_t*[argc+1];
for (int i=0; i<argc; ++i)
{
pin_ptr<const wchar_t> wch = PtrToStringChars(managedArgs[i]);
wargv[i] = _wcsdup(wch);
}
wargv[argc] = NULL;
//
// Report error in initializing cluster layer
// Enable logging based on environment variable
//
DrLogE("Couldn't initialise Cluster");
SetLoggingLevel();
DrInitErrorTable();
DrInitExitCodeTable();
DrInitLastAccessTable();
// Set unhandled exception handler to catch anything thrown from
// managed code
System::AppDomain^ currentDomain = System::AppDomain::CurrentDomain;
currentDomain->UnhandledException += gcnew System::UnhandledExceptionEventHandler(ExceptionHandler);
//
// trace for startup
//
DrLogE("Vertex Host starting");
//
// Get environment variable to know whether to break into debugger
//
BreakForDebugger();
//
// We call Register on the Managed Wrapper vertex factory to force its library to be linked.
// Registration actually occurs during static initialization.
//
s_factoryHWrapper.Register();
//
// Get command line arguments
//
char** argv;
DrGetUtf8CommandArgs(argc, wargv, &argv);
for (int i=0; i<managedArgs->Length; ++i)
{
free(wargv[i]);
}
delete [] wargv;
//
// Initialize the dryad communication layer with the command line arguments
//
int nOpts;
DrError e;
e = DryadInitialize(argc, argv, &nOpts);
if (e != DrError_OK)
{
//
// Report error in initializing cluster layer
//
DrLogE("Couldn't initialise Cluster");
return 1;
}
//
// Update the argument list to just those parameters that weren't used by cluster init
//
EliminateArguments(&argc, argv, 1, nOpts);
//
// Call main function to continue execution of vertex
//
int exitCode = DryadVertexMain(argc, argv, NULL);
//
// Close the cluster connection after dryadvertexmain returns
//
e = DryadShutdown();
if (e == DrError_OK)
{
//
// Report success
//
DrLogI("Completed uninitialise cluster");
}
else
{
//
// Report failure
//
DrLogE("Couldn't uninitialise cluster");
}
return exitCode;
}
catch (System::Exception^ e)
{
DrLogA("Unhandled exception: %s", DrString(e->ToString()).GetChars());
return 1;
}
//
// Update the argument list to just those parameters that weren't used by cluster init
//
EliminateArguments(&argc, argv, 1, nOpts);
//
// Call main function to continue execution of vertex
//
int exitCode = DryadVertexMain(argc, argv, NULL);
//
// Close the cluster connection after dryadvertexmain returns
//
e = DryadShutdown();
if (e == DrError_OK)
{
//
// Report success
//
DrLogI("Completed uninitialise cluster");
}
else
{
//
// Report failure
//
DrLogE("Couldn't uninitialise cluster");
}
return exitCode;
}
catch (System::Exception^ e)
{
DrLogA("Unhandled exception: %s", DrString(e->ToString()).GetChars());
return 1;
}
}
};
//
// Simple data class which contains the byte array and its length.

27
GraphManager/GraphManager.vcxproj
View File

@ -1,6 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -25,13 +28,13 @@
<ConfigurationType>DynamicLibrary</ConfigurationType>
<CLRSupport>Pure</CLRSupport>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet>
<CLRSupport>Pure</CLRSupport>
<PlatformToolset>v110</PlatformToolset>
<PlatformToolset>v120</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
@ -59,6 +62,7 @@
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad</TargetName>
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad</TargetName>
<NuGetPackageImportStamp>46822e65</NuGetPackageImportStamp>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl>
@ -223,13 +227,22 @@
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
</ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
</Target>
</Project>

2
GraphManager/filesystem/DrAzureBlobClient.h
View File

@ -20,7 +20,7 @@ limitations under the License.
#pragma once
using namespace Microsoft::Research::Peloponnese::Storage;
using namespace Microsoft::Research::Peloponnese::Azure;
DRCLASS(DrAzureInputStream) : public DrInputStream
{

267
GraphManager/filesystem/DrHdfsClient.cpp
View File

@ -129,73 +129,42 @@ DrHdfsInputStream::DrHdfsInputStream()
m_hdfsInstance = DrNull;
}
HRESULT DrHdfsInputStream::Open(DrUniversePtr universe, DrNativeString streamUri)
HRESULT DrHdfsInputStream::Open(DrUniversePtr universe, DrNativeString streamUri, DrNativeString recordType)
{
DrString uri = DrString(streamUri);
DrString record = DrString(recordType);
DrLogI("Opening instance for %s", uri.GetChars());
DrLogI("Opening instance for %s record type %s", uri.GetChars(), record.GetChars());
return OpenInternal(universe, uri);
return OpenInternal(universe, uri, record);
}
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri)
#ifdef _MANAGED
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType)
{
m_streamUri = streamUri;
HRESULT err = S_OK;
#ifdef _MANAGED
try
{
#endif
DrLogI("Opening instance for %s", streamUri.GetChars());
DrLogI("Opening instance for %s: %s", streamUri.GetChars(), recordType.GetChars());
m_hdfsInstance = GetHdfsServiceInstance(streamUri);
#ifdef _MANAGED
DrLogI("Getting file info for %s", streamUri.GetChars());
HdfsFileInfo^ stream = m_hdfsInstance->GetFileInfo(streamUri.GetString(), true);
m_fileNameArray = stream->fileNameArray;
UInt32 totalPartitionCount = static_cast<UInt32>(stream->blockArray->Length);
UInt32 totalPartitionCount;
#else
bool ret = HdfsBridgeNative::Initialize();
if (!ret)
if (recordType.Compare("Microsoft.Research.DryadLinq.LineRecord") == 0)
{
DrLogE("Error calling HdfsBridgeNative::Initialize()");
return E_FAIL;
DrLogI("Getting block-level file info for %s", streamUri.GetChars());
totalPartitionCount = static_cast<UInt32>(stream->blockArray->Length);
}
if (m_hdfsInstance == NULL)
else
{
DrLogE("Error calling GetHdfsServiceInstance(streamUri)");
return E_FAIL;
DrLogI("Getting file info for %s", streamUri.GetChars());
totalPartitionCount = m_fileNameArray->Length;
}
URL_COMPONENTSA UrlComponents = {0};
UrlComponents.dwStructSize = sizeof(UrlComponents);
UrlComponents.dwUrlPathLength = 1;
UrlComponents.dwHostNameLength = 1;
BOOL fOK = InternetCrackUrlA(streamUri.GetChars(), streamUri.GetCharsLength(), 0, &UrlComponents);
if (!fOK)
{
DrLogE("Error getting stream path from HDFS URI.");
return E_FAIL;
}
m_hostname.Set(UrlComponents.lpszHostName);
m_portNum = UrlComponents.nPort;
InstanceAccessor ia(m_hdfsInstance);
FileStat* fileStat = NULL;
ia.OpenFileStat(UrlComponents.lpszUrlPath, true, &fileStat);
UINT32 totalPartitionCount = 0;
HdfsBridgeNative::FileStatAccessor fs(fileStat);
totalPartitionCount = fs.GetNumberOfBlocks();
m_fileNameArray = (const char **)fs.GetFileNameArray();
#endif
DrLogI("Partition count %d", totalPartitionCount);
@ -205,35 +174,97 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
m_partOffsets = DrNew DrUINT64Array(totalPartitionCount);
m_partFileIds = DrNew DrUINT32Array(totalPartitionCount);
for (UINT32 i=0; i<totalPartitionCount; ++i)
if (recordType.Compare("Microsoft.Research.DryadLinq.LineRecord") == 0)
{
#ifdef _MANAGED
HdfsBlockInfo^ partition = stream->blockArray[i];
#else
HdfsBridgeNative::HdfsBlockLocInfo* partition = fs.GetBlockInfo(i);
#endif
m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(partition->Size);
m_partOffsets[i] = partition->Offset;
m_partFileIds[i] = partition->fileIndex;
#ifdef _MANAGED
for (int j = 0; j < partition->Hosts->Length; ++j)
#else
for (int j = 0; j < partition->numberOfHosts; ++j)
#endif
for (UINT32 i = 0; i < totalPartitionCount; ++i)
{
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
HdfsBlockInfo^ partition = stream->blockArray[i];
m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(partition->Size);
m_partOffsets[i] = partition->Offset;
m_partFileIds[i] = partition->fileIndex;
for (int j = 0; j < partition->Hosts->Length; ++j)
{
m_affinity[i]->AddLocality(location);
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
m_affinity[i]->AddLocality(location);
}
}
}
#ifndef _MANAGED
delete partition;
#endif
}
#ifdef _MANAGED
}
else
{
int fileBlockIndex = 0;
for (UINT32 i = 0; i < totalPartitionCount; ++i)
{
m_partOffsets[i] = 0;
m_partFileIds[i] = i;
HdfsBlockInfo^ partition = stream->blockArray[fileBlockIndex];
DrAssert(partition->fileIndex == i);
long long fileSize = partition->Size;
HashSet<DrResourceRef>^ locations = DrNew HashSet<DrResourceRef>();
for (int j = 0; j < partition->Hosts->Length; ++j)
{
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
locations->Add(location);
}
}
++fileBlockIndex;
while (fileBlockIndex < stream->blockArray->Length && stream->blockArray[fileBlockIndex]->fileIndex == i)
{
partition = stream->blockArray[fileBlockIndex];
fileSize += partition->Size;
if (locations->Count > 0)
{
HashSet<DrResourceRef>^ newLocations = DrNew HashSet<DrResourceRef>();
for (int j = 0; j < partition->Hosts->Length; ++j)
{
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
newLocations->Add(location);
}
}
locations->IntersectWith(newLocations);
}
}
m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(fileSize);
System::Text::StringBuilder^ locationText;
if (locations->Count > 0)
{
locationText = gcnew System::Text::StringBuilder("File " + m_fileNameArray[i] + " merged locations:");
}
else
{
locationText = gcnew System::Text::StringBuilder("File " + m_fileNameArray[i] + " no shared locations");
}
HashSet<DrResourceRef>::Enumerator^ enumerator = locations->GetEnumerator();
while (enumerator->MoveNext())
{
m_affinity[i]->AddLocality(enumerator->Current);
locationText->Append(" ");
locationText->Append(enumerator->Current->GetName().GetString());
}
DrString locationLog(locationText->ToString());
DrLogI("%s", locationLog.GetChars());
}
}
}
catch (System::Exception ^e)
{
@ -247,10 +278,85 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
// TODO: How do we clean this up?
//hdfsInstance->Discard();
}
#endif
return err;
}
#else
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType)
{
m_streamUri = streamUri;
HRESULT err = S_OK;
DrLogI("Opening instance for %s", streamUri.GetChars());
m_hdfsInstance = GetHdfsServiceInstance(streamUri);
bool ret = HdfsBridgeNative::Initialize();
if (!ret)
{
DrLogE("Error calling HdfsBridgeNative::Initialize()");
return E_FAIL;
}
if (m_hdfsInstance == NULL)
{
DrLogE("Error calling GetHdfsServiceInstance(streamUri)");
return E_FAIL;
}
URL_COMPONENTSA UrlComponents = {0};
UrlComponents.dwStructSize = sizeof(UrlComponents);
UrlComponents.dwUrlPathLength = 1;
UrlComponents.dwHostNameLength = 1;
BOOL fOK = InternetCrackUrlA(streamUri.GetChars(), streamUri.GetCharsLength(), 0, &UrlComponents);
if (!fOK)
{
DrLogE("Error getting stream path from HDFS URI.");
return E_FAIL;
}
m_hostname.Set(UrlComponents.lpszHostName);
m_portNum = UrlComponents.nPort;
InstanceAccessor ia(m_hdfsInstance);
FileStat* fileStat = NULL;
ia.OpenFileStat(UrlComponents.lpszUrlPath, true, &fileStat);
UINT32 totalPartitionCount = 0;
HdfsBridgeNative::FileStatAccessor fs(fileStat);
totalPartitionCount = fs.GetNumberOfBlocks();
m_fileNameArray = (const char **)fs.GetFileNameArray();
DrLogI("Partition count %d", totalPartitionCount);
/* Allocate these arrays even if they're size 0, to avoid
NullReferenceException later */
m_affinity = DrNew DrAffinityArray(totalPartitionCount);
m_partOffsets = DrNew DrUINT64Array(totalPartitionCount);
m_partFileIds = DrNew DrUINT32Array(totalPartitionCount);
for (UINT32 i=0; i<totalPartitionCount; ++i)
{
HdfsBridgeNative::HdfsBlockLocInfo* partition = fs.GetBlockInfo(i);
m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(partition->Size);
m_partOffsets[i] = partition->Offset;
m_partFileIds[i] = partition->fileIndex;
for (int j = 0; j < partition->numberOfHosts; ++j)
{
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
m_affinity[i]->AddLocality(location);
}
}
delete partition;
}
return err;
}
#endif
DrNativeString DrHdfsInputStream::GetError()
{
@ -468,6 +574,25 @@ HRESULT DrHdfsOutputStream::FinalizeSuccessfulParts(DrOutputPartitionArrayRef pa
errorText.SetF("%s", DrString(m_error).GetChars());
return E_FAIL;
}
String^ userName = Environment::GetEnvironmentVariable("USER");
if (userName == nullptr)
{
userName = Environment::UserName;
}
try
{
m_hdfsInstance->SetOwnerAndPermission(m_baseUri, userName, nullptr, Convert::ToInt16("0755", 8));
}
catch (Exception^ e)
{
DrString drDst(m_baseUri);
DrString err(e->ToString());
DrLogE("Can't set %s permissions finalizing HDFS output: %s", drDst.GetChars(), err.GetChars());
m_error = "Can't set " + m_baseUri + " permissions finalizing HDFS output: " + e->ToString();
errorText.SetF("%s", DrString(m_error).GetChars());
return E_FAIL;
}
}
else
{

4
GraphManager/filesystem/DrHdfsClient.h
View File

@ -30,8 +30,8 @@ DRCLASS(DrHdfsInputStream) : public DrInputStream
{
public:
DrHdfsInputStream();
HRESULT Open(DrUniversePtr universe, DrNativeString streamUri);
HRESULT OpenInternal(DrUniversePtr universe, DrString streamUri);
HRESULT Open(DrUniversePtr universe, DrNativeString streamUri, DrNativeString recordType);
HRESULT OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType);
DrNativeString GetError();
virtual DrString GetStreamName() DROVERRIDE;

32
GraphManager/filesystem/DrPartitionFile.cpp
View File

@ -79,6 +79,7 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
DrStringR remoteName,
DrPartitionInputStream::OverridePtr over,
bool mustOverride,
bool pathIsRooted,
DrString line,
DrUniversePtr universe)
{
@ -112,8 +113,16 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
sep = lineCopy.IndexOfChar(',');
if (sep == DrStr_InvalidIndex)
{
DrLogW("Malformed line %s: no list of machines", line.GetChars());
return false;
int n = sscanf_s(lineCopy.GetChars(), "%I64u", &parsedSize);
if (n != 1)
{
DrLogW("Malformed line %s: can't parse size", line.GetChars());
return false;
}
affinity->SetWeight(parsedSize);
lineCopy = DrString("");
}
else
{
@ -135,8 +144,14 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
if (lineCopy.GetCharsLength() == 0)
{
DrLogW("Malformed line %s: no partition machines", line.GetChars());
return false;
if (!pathIsRooted || mustOverride)
{
DrLogW("Malformed line %s: no partition machines", line.GetChars());
return false;
}
remoteName.Set(" %Invalid% ");
return true;
}
int numberOfReplicas = 0;
@ -204,7 +219,7 @@ HRESULT DrPartitionInputStream::Open(DrUniversePtr universe, DrNativeString stre
HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString streamName)
{
HRESULT err = S_OK;
DrLogI("Opening input file %s", streamName.GetChars(), DRERRORSTRING(err));
DrLogI("Opening input file %s", streamName.GetChars(), DRERRORSTRING(err));
FILE* f;
errno_t ferr = fopen_s(&f, streamName.GetChars(), "rb");
if (ferr != 0)
@ -231,6 +246,12 @@ HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString st
mustOverride = true;
}
bool pathIsRooted = false;
if (m_pathNameOnComputer.IndexOfChar(':') != DrStr_InvalidIndex)
{
pathIsRooted = true;
}
int numberOfParts;
int n = sscanf_s(partitionSizeLine.GetChars(), "%d", &numberOfParts);
if (n != 1)
@ -280,6 +301,7 @@ HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString st
remoteName,
m_override[i],
mustOverride,
pathIsRooted,
partitionLine,
universe) == false)
{

19
GraphManager/jobmanager/version.cpp
View File

@ -19,5 +19,22 @@ limitations under the License.
*/
#ifdef _MANAGED
[assembly:System::Runtime::InteropServices::ComVisible(false)];
#pragma managed
using namespace System::Reflection;
using namespace System::Runtime::InteropServices;
[assembly: AssemblyTitle("Microsoft.Research.Dryad")];
[assembly: AssemblyDescription("")];
[assembly: AssemblyConfiguration("")];
[assembly: AssemblyCompany("Microsoft Corporation")];
[assembly: AssemblyProduct("Microsoft.Research.Dryad")];
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
[assembly: AssemblyTrademark("")];
[assembly: AssemblyCulture("")];
[assembly: ComVisible(false)];
[assembly: AssemblyVersion("1.0.0.0")];
[assembly: AssemblyFileVersion("1.0.0.0")];
#endif

5
GraphManager/packages.config
View File

@ -1,4 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="Native" />
</packages>

5
GraphManager/reporting/DrCalypsoReporting.cpp
View File

@ -24,7 +24,8 @@ limitations under the License.
#include <msclr\lock.h>
using namespace System::IO;
using namespace Microsoft::Research::Peloponnese::Storage;
using namespace Microsoft::Research::Peloponnese::Azure;
using namespace Microsoft::Research::Peloponnese::Hdfs;
using namespace msclr;
//
@ -126,7 +127,7 @@ public:
DrCalypsoReporter::DrCalypsoReporter(DrNativeString uriString)
{
System::Uri^ uri = DrNew System::Uri(uriString);
if (uri->Scheme == AzureUtils::BlobScheme)
if (uri->Scheme == Utils::BlobScheme)
{
m_logStream = DrNew AzureLogAppendStream(uri, 0x20, false, false, gcnew PeloponneseLogger());
m_flushInterval = 1000;

2
GraphManager/vertex/DrVertex.cpp
View File

@ -1743,7 +1743,7 @@ void DrTeeVertex::ReactToFailedVertex(DrVertexOutputGeneratorPtr failedGenerator
DrAssert(m_inputEdges->GetNumberOfEdges() == 1);
DrEdge e = m_inputEdges->GetEdge(0);
DrLogI("Tee vertex %d.%d: calling ReactToFailedVertex on remote edge", this->m_id, GetVersion());
DrLogI("Tee vertex %d.%d: calling ReactToFailedVertex on remote edge", this->m_id, failedGenerator->GetVersion());
e.m_remoteVertex->ReactToFailedVertex(failedGenerator, DrNull, DrNull, DrNull, originalReason);
/* fill in a new generator if it's already there, e.g. if the upstream vertex is a DrStorageVertex */

56
JobBrowser/Calypso/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,56 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("DryadAnalysis")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("MSIT")]
[assembly: AssemblyProduct("DryadAnalysis")]
[assembly: AssemblyCopyright("Copyright © MSIT 2008")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("286845bc-d060-49ff-9de8-4381471a0dc6")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

1290
JobBrowser/JOM/ClusterConfiguration.cs
View File

File diff suppressed because it is too large Load Diff

377
JobBrowser/JOM/ClusterStatus.cs
View File

@ -19,12 +19,22 @@ limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using Microsoft.Research.Peloponnese.Storage;
using System.Web;
using System.Xml;
using System.Xml.Linq;
using Microsoft.Research.Peloponnese.Hdfs;
using Microsoft.Research.Peloponnese.Shared;
using Microsoft.Research.Peloponnese.Yarn;
using Microsoft.Research.Tools;
using System.Text.RegularExpressions;
using System.Text;
using JobStatus = Microsoft.Research.Peloponnese.ClusterUtils.JobStatus;
namespace Microsoft.Research.JobObjectModel
{
@ -68,12 +78,16 @@ namespace Microsoft.Research.JobObjectModel
/// <summary>
/// See if a status is already cached.
/// </summary>
/// <param name="clusterName">Name of cluster.</param>
/// <param name="config">Cluster configuration.</param>
/// <returns>The cached status.</returns>
public static ClusterStatus LookupStatus(string clusterName)
public static ClusterStatus LookupStatus(ClusterConfiguration config)
{
if (ClusterStatuses.ContainsKey(clusterName))
return ClusterStatuses[clusterName];
if (ClusterStatuses.ContainsKey(config.Name))
{
var retval = ClusterStatuses[config.Name];
if (retval.Config.Equals(config))
return retval;
}
return null;
}
@ -172,8 +186,8 @@ namespace Microsoft.Research.JobObjectModel
/// <param name="manager">Communication manager.</param>
public virtual void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
{
// refresh the whole list
this.RecomputeClusterJobList(summary.VirtualCluster, manager);
// refresh the whole list: too expensive
// this.RecomputeClusterJobList(summary.VirtualCluster, manager);
ClusterJobInformation info = this.DiscoverClusterJob(summary, manager);
if (info == null)
{
@ -193,6 +207,7 @@ namespace Microsoft.Research.JobObjectModel
/// <summary>
/// A fake cluster keeps some information about past jobs on a local filesystem, to allow post-mortem debugging.
/// </summary>
@ -442,20 +457,17 @@ namespace Microsoft.Research.JobObjectModel
/// <summary>
/// Status of an Azure DFS cluster.
/// </summary>
public class AzureDfsClusterStatus : ClusterStatus
public abstract class DfsClusterStatus : ClusterStatus
{
private AzureDfsClusterConfiguration config;
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="config">Configuration for the local machine.</param>
public AzureDfsClusterStatus(ClusterConfiguration config)
protected DfsClusterStatus(ClusterConfiguration config)
: base(config)
{
if (!(config is AzureDfsClusterConfiguration))
throw new ArgumentException("Expected a AzureYarnClusterConfiguration, got a " + config.GetType());
this.config = config as AzureDfsClusterConfiguration;
if (!(config is DfsClusterConfiguration))
throw new ArgumentException("Expected a DfsClusterConfiguration, got a " + config.GetType());
}
/// <summary>
@ -468,31 +480,6 @@ namespace Microsoft.Research.JobObjectModel
return result;
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var jobs = this.config.AzureClient.EnumerateDirectory("").ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(job);
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(job, info);
}
manager.Progress(100*done++/jobs.Count);
}
manager.Progress(100);
}
/// <summary>
/// Discover the (unique) dryadlinq job corresponding to a cluster job.
/// </summary>
@ -503,7 +490,7 @@ namespace Microsoft.Research.JobObjectModel
{
DryadLinqJobSummary result = new DryadLinqJobSummary(
clusterJob.Cluster,
this.config.TypeOfCluster,
this.Config.TypeOfCluster,
"", // virtual cluster
"", // machine
clusterJob.ClusterJobID, // jobId
@ -529,6 +516,78 @@ namespace Microsoft.Research.JobObjectModel
throw new InvalidOperationException();
}
/// <summary>
/// Cancel the specified job.
/// </summary>
/// <param name="job">Job whose execution is cancelled.</param>
/// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job)
{
return false;
}
}
/// <summary>
/// Status of an Azure DFS cluster.
/// </summary>
public class AzureDfsClusterStatus : DfsClusterStatus
{
private AzureDfsClusterConfiguration config;
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="config">Configuration for the local machine.</param>
public AzureDfsClusterStatus(ClusterConfiguration config)
: base(config)
{
if (!(config is AzureDfsClusterConfiguration))
throw new ArgumentException("Expected a AzureDfsClusterConfiguration, got a " + config.GetType());
this.config = config as AzureDfsClusterConfiguration;
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var jobs = this.config.AzureClient.ExpandFileOrDirectory(AzureDfsFile.UriFromPath(this.config, "")).ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
string jobRootFolder = AzureDfsFile.PathFromUri(this.config, job);
ClusterJobInformation info = this.GetJobInfo(jobRootFolder);
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(job.AbsolutePath, info);
}
manager.Progress(100*done++/jobs.Count);
}
manager.Progress(100);
}
/// <summary>
/// Extract blob name from a path.
/// </summary>
/// <param name="container">Container name.</param>
/// <param name="path">Path.</param>
/// <returns>The blob part of path.</returns>
public static string GetBlobName(string container, string path)
{
if (path.StartsWith("/" + container + "/"))
path = path.Substring(container.Length + 2);
int q = path.IndexOf('?');
if (q > 0)
path = path.Substring(0, q);
return path;
}
/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
@ -540,14 +599,19 @@ namespace Microsoft.Research.JobObjectModel
DateTime lastHeartBeat = DateTime.MinValue;
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
bool found = false;
string jobName = jobRootFolder;
var jobsFolders = this.config.AzureClient.EnumerateDirectory(jobRootFolder).ToList();
Uri uri = AzureDfsFile.UriFromPath(this.config, jobRootFolder);
var jobsFolders = this.config.AzureClient.ExpandFileOrDirectory(uri).ToList();
jobRootFolder = GetBlobName(this.config.Container, jobRootFolder);
string jobName = jobRootFolder;
foreach (var file in jobsFolders)
{
if (file.EndsWith("heartbeat"))
if (file.AbsolutePath.EndsWith("heartbeat"))
{
var blob = this.config.AzureClient.Container.GetPageBlobReference(file);
string blobName = GetBlobName(this.config.Container, file.AbsolutePath);
var blob = this.config.AzureClient.Container.GetPageBlobReference(blobName);
blob.FetchAttributes();
var props = blob.Metadata;
if (props.ContainsKey("status"))
@ -597,11 +661,11 @@ namespace Microsoft.Research.JobObjectModel
found = true;
}
else if (file.Contains("DryadLinqProgram__") &&
else if (file.AbsolutePath.Contains("DryadLinqProgram__") &&
// newer heartbeats contain the date
date != DateTime.MinValue)
{
var blob = this.config.AzureClient.Container.GetBlockBlobReference(file);
var blob = this.config.AzureClient.Container.GetBlockBlobReference(AzureDfsFile.PathFromUri(this.config, file));
blob.FetchAttributes();
var props = blob.Properties;
if (props.LastModified.HasValue)
@ -628,7 +692,6 @@ namespace Microsoft.Research.JobObjectModel
/// <param name="manager">Communication manager.</param>
public override void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
{
// refresh the whole list
ClusterJobInformation info = this.GetJobInfo(summary.JobID);
if (info == null)
{
@ -645,8 +708,224 @@ namespace Microsoft.Research.JobObjectModel
/// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job)
{
AzureUtils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId);
return true;
Microsoft.Research.Peloponnese.Azure.Utils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId);
return false;
}
}
/// <summary>
/// Cluster status of a WebHdfs cluster.
/// </summary>
public class WebHdfsClusterStatus : DfsClusterStatus
{
private WebHdfsClusterConfiguration config;
/// <summary>
/// Yarn client to access job status.
/// </summary>
private NativeYarnClient yarnClient;
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public WebHdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is WebHdfsClusterConfiguration))
throw new ArgumentException("Expected a WebHdfsClusterConfiguration, got a " + conf.GetType());
this.config = conf as WebHdfsClusterConfiguration;
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
private ClusterJobInformation GetJobInfo(string jobRootFolder)
{
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
long time;
long size;
this.config.DfsClient.GetFileStatus(uri, out time, out size);
DateTime date = DfsFile.TimeFromLong(time);
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
string jobName = Path.GetFileName(jobRootFolder);
string errorMsg = "";
try
{
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
var jobstatus = jobinfo.GetStatus();
errorMsg = jobinfo.ErrorMsg;
switch (jobstatus)
{
case JobStatus.NotSubmitted:
case JobStatus.Waiting:
status = ClusterJobInformation.ClusterJobStatus.Unknown;
break;
case JobStatus.Running:
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case JobStatus.Success:
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
break;
case JobStatus.Cancelled:
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
case JobStatus.Failure:
status = ClusterJobInformation.ClusterJobStatus.Failed;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception)
{
}
TimeSpan running = TimeSpan.Zero;
var info = new ClusterJobInformation(this.config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
return info;
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable once UnusedParameter.Global
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
var jobsEnum = this.config.DfsClient.EnumerateSubdirectories(uri);
List<Uri> jobs = jobsEnum != null ? jobsEnum.ToList() : new List<Uri>();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(DfsFile.PathFromUri(this.config.JobsFolderUri, job));
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(info.ClusterJobID, info);
}
manager.Progress(100 * done++ / jobs.Count);
}
manager.Progress(100);
}
}
/// <summary>
/// Cluster status of a WebHdfs cluster.
/// </summary>
public class HdfsClusterStatus : DfsClusterStatus
{
private HdfsClusterConfiguration config;
/// <summary>
/// Yarn client to access job status.
/// </summary>
private NativeYarnClient yarnClient;
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public HdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is HdfsClusterConfiguration))
throw new ArgumentException("Expected an HdfsClusterConfiguration, got a " + conf.GetType());
this.config = conf as HdfsClusterConfiguration;
// make a fake call to initialize the cluster on the foreground thread
// HDFS does not work if initialized on the background thread.
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
this.config.DfsClient.IsFileExists(uri); // ignore result
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
private ClusterJobInformation GetJobInfo(string jobRootFolder)
{
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
long time;
long size;
this.config.DfsClient.GetFileStatus(uri, out time, out size);
DateTime date = DfsFile.TimeFromLong(time);
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
string jobName = Path.GetFileName(jobRootFolder);
string errorMsg = "";
try
{
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
var jobstatus = jobinfo.GetStatus();
errorMsg = jobinfo.ErrorMsg;
switch (jobstatus)
{
case JobStatus.NotSubmitted:
case JobStatus.Waiting:
status = ClusterJobInformation.ClusterJobStatus.Unknown;
break;
case JobStatus.Running:
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case JobStatus.Success:
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
break;
case JobStatus.Cancelled:
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
case JobStatus.Failure:
status = ClusterJobInformation.ClusterJobStatus.Failed;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception)
{
}
TimeSpan running = TimeSpan.Zero;
var info = new ClusterJobInformation(config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
return info;
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable once UnusedParameter.Global
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
var jobs = this.config.DfsClient.EnumerateSubdirectories(uri).ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(DfsFile.PathFromUri(this.config.JobsFolderUri, job));
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(info.ClusterJobID, info);
}
manager.Progress(100 * done++ / jobs.Count);
}
manager.Progress(100);
}
}
}

266
JobBrowser/JOM/JobObjectModel.csproj
View File

@ -1,6 +1,12 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.6-alpha042\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.6-alpha042\build\Microsoft.Research.Peloponnese.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -9,39 +15,17 @@
<ProjectGuid>{77739535-7FAC-4487-887F-FEBA197E7572}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>DistributedDataCollection</RootNamespace>
<AssemblyName>DistributedDataCollection</AssemblyName>
<RootNamespace>JobObjectModel</RootNamespace>
<AssemblyName>JobObjectModel</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<StartupObject>
</StartupObject>
<FileUpgradeFlags>
</FileUpgradeFlags>
<UpgradeBackupLocation>
</UpgradeBackupLocation>
<OldToolsVersion>3.5</OldToolsVersion>
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
<NuGetPackageImportStamp>b735dff6</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DocumentationFile>bin\Debug\DistributedDataCollection.XML</DocumentationFile>
<DocumentationFile>bin\Debug\JobObjectModel.XML</DocumentationFile>
<DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
@ -56,108 +40,13 @@
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.DataSetExtensions">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="ClusterAccess.cs" />
<Compile Include="ClusterConfiguration.cs" />
<Compile Include="ClusterStatus.cs" />
<Compile Include="dryadlog.cs" />
<Compile Include="jobinfo.cs" />
<Compile Include="JobSummary.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Storage.cs" />
</ItemGroup>
@ -168,40 +57,121 @@
</ProjectReference>
</ItemGroup>
<ItemGroup>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<None Include="app.config" />
<None Include="packages.config">
<SubType>Designer</SubType>
</None>
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
<None Include="packages.config" />
<Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.HDInsight.Net.Http.Formatting, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.HDInsight.Net.Http.Formatting.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.3.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System">
<HintPath>..\..\..\..\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5\System.dll</HintPath>
</Reference>
<Reference Include="System.Data" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.XML" />
<Reference Include="System.Xml.Linq" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

488
JobBrowser/JOM/JobSummary.cs Normal file
View File

@ -0,0 +1,488 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System;
using System.IO;
using System.Text;
using Microsoft.Research.Tools;
namespace Microsoft.Research.JobObjectModel
{
/// <summary>
/// Error during conversation with cluster.
/// </summary>
public sealed class ClusterException : Exception
{
/// <summary>
/// Create an exception about handling a cluster.
/// </summary>
/// <param name="message">Exception message.</param>
public ClusterException(string message) : base(message) { }
}
/// <summary>
/// Identifier for a Dryad process; for now we are using globally-unique process GUID.
/// A Dryad Job identifier is always the same as the job manager process guid.
/// </summary>
[Serializable]
public class DryadProcessIdentifier : IEquatable<DryadProcessIdentifier>
{
/// <summary>
/// Process identifier.
/// </summary>
private string processIdentifier;
/// <summary>
/// Used just for XML serialization.
/// </summary>
public DryadProcessIdentifier()
{ }
/// <summary>
/// create an indentifier struct.
/// Be sure to initialize all fields before use;
/// </summary>
/// <param name="pid">The id of the process (platform-dependent).</param>
public DryadProcessIdentifier(string pid)
{
this.processIdentifier = pid;
}
/// <summary>
/// Process identifier; public for serialization only.
/// </summary>
public string ProcessIdentifier
{
get { return this.processIdentifier; }
set { this.processIdentifier = value; }
}
/// <summary>
/// If true the process identifier is not known.
/// </summary>
public bool IsUnknown { get { return this.ProcessIdentifier == null; } }
/// <summary>
/// Human-readable description of the process identifier.
/// </summary>
/// <returns>An empty string if the pid is not set.</returns>
public override string ToString()
{
if (this.ProcessIdentifier != null) return this.ProcessIdentifier;
return "";
}
/// <summary>
/// Equality test.
/// </summary>
/// <param name="obj">Object to compare to.</param>
/// <returns>True if both objects represent the same process id.</returns>
public override bool Equals(object obj)
{
if (!(obj is DryadProcessIdentifier))
return false;
return this.Equals((DryadProcessIdentifier)obj);
}
#region IEquatable<DryadProcessIdentifier> Members
/// <summary>
/// Equality test.
/// </summary>
/// <param name="other">Process id to compare to.</param>
/// <returns>True if the id's represent the same process.</returns>
public bool Equals(DryadProcessIdentifier other)
{
if (this.IsUnknown)
return other.IsUnknown;
if (other.IsUnknown)
return false;
return this.ProcessIdentifier.Equals(other.ProcessIdentifier);
}
/// <summary>
/// Overriden implementation of getHashCode.
/// </summary>
/// <returns>The hashcode of the process id.</returns>
public override int GetHashCode()
{
// ReSharper disable once BaseObjectGetHashCodeCallInGetHashCode
return base.GetHashCode();
}
#endregion
}
/// <summary>
/// Brief summary of an executed DryadLINQ job.
/// </summary>
[Serializable]
public sealed class DryadLinqJobSummary : IEquatable<DryadLinqJobSummary>
{
/// <summary>
/// Empty constructor for XML serialization.
/// </summary>
public DryadLinqJobSummary()
{ }
/// <summary>
/// Initialize a job summary.
/// </summary>
/// <param name="cluster">Cluster where the job ran.</param>
/// <param name="clusterType">A string corresponding to the type of ClusterConfiguration.</param>
/// <param name="machine">Machine where job manager ran.</param>
/// <param name="jobId">Id of job.</param>
/// <param name="jmProcessGuid">Guid of job manager process.</param>
/// <param name="clusterJobId">Id of job on the cluster.</param>
/// <param name="friendlyname">Friendly name used.</param>
/// <param name="username">Who ran the job.</param>
/// <param name="date">Start date (not completion date).</param>
/// <param name="status">Job status.</param>
/// <param name="endTime">Estimated end running time.</param>
/// <param name="virtualcluster">Virtual cluster where job ran.</param>
public DryadLinqJobSummary(
string cluster,
ClusterConfiguration.ClusterType clusterType,
string virtualcluster,
string machine,
string jobId,
string clusterJobId,
DryadProcessIdentifier jmProcessGuid,
string friendlyname,
string username,
DateTime date,
DateTime endTime,
ClusterJobInformation.ClusterJobStatus status)
{
this.VirtualCluster = virtualcluster;
this.Cluster = cluster;
this.ClusterType = clusterType;
this.Machine = machine;
this.Name = friendlyname;
this.User = username;
this.Date = date;
this.EndTime = endTime;
this.Status = status;
this.ManagerProcessGuid = jmProcessGuid;
this.JobID = jobId;
this.ClusterJobId = clusterJobId;
}
/// <summary>
/// Cluster where the job ran.
/// </summary>
public string Cluster { get; /*private*/ set; }
/// <summary>
/// Id of cluster job that originated this DryadLinq job (can be used to find the cluster job from the dryadlinq job).
/// </summary>
public string ClusterJobId { get; /*private*/ set; }
/// <summary>
/// Cluster where the job ran.
/// </summary>
public DateTime EndTime { get; /*private*/ set; }
/// <summary>
/// String describing cluster type.
/// </summary>
public ClusterConfiguration.ClusterType ClusterType { get; /*private*/ set; }
/// <summary>
/// Virtual cluster where job ran.
/// </summary>
public string VirtualCluster { get; /*private*/ set; }
/// <summary>
/// (Friendly) name of the job.
/// </summary>
public string Name { get; /*private*/ set; }
/// <summary>
/// User who submitted job.
/// </summary>
public string User { get; /*private*/ set; }
/// <summary>
/// ID of job on the cluster.
/// </summary>
public string JobID { get; /*private*/ set; }
/// <summary>
/// The Guid of the job manager process.
/// </summary>
public DryadProcessIdentifier ManagerProcessGuid { set; get; }
/// <summary>
/// User who submitted job.
/// </summary>
public string GetAlias()
{
int pos = User.IndexOf(@"\");
return User.Substring(pos + 1);
}
/// <summary>
/// Date when job was submitted.
/// </summary>
public DateTime Date { get; /*private*/ set; }
/// <summary>
/// Did the job fail?
/// </summary>
public ClusterJobInformation.ClusterJobStatus Status { get; /*internal*/ set; }
/// <summary>
/// Machine where the job manager ran.
/// </summary>
public string Machine { get; /*private*/ set; }
/// <summary>
/// Get a short name for this job summary.
/// </summary>
/// <returns>Short name of job summary.</returns>
public string ShortName()
{
// we use the starting time to uniquify the job name
return this.Date.ToString("s") + "-" + this.Name;
}
/// <summary>
/// True if these two summaries are the same. The status and end time do not matter, since the job may still be running.
/// </summary>
/// <param name="other">Summary to compare against.</param>
/// <returns>True if they are equal.</returns>
public bool Equals(DryadLinqJobSummary other)
{
return this.Cluster == other.Cluster &&
this.ClusterJobId == other.ClusterJobId &&
this.Date == other.Date &&
this.Machine == other.Machine &&
this.Name == other.Name &&
this.User == other.User;
}
/// <summary>
/// Hashcode proper for the equality test.
/// </summary>
/// <returns>The object hashcode.</returns>
public override int GetHashCode()
{
return this.ClusterJobId.GetHashCode() ^ this.ClusterJobId.GetHashCode() ^ this.Date.GetHashCode() ^ this.Machine.GetHashCode() ^ this.Name.GetHashCode() ^ this.User.GetHashCode();
}
/// <summary>
/// A string describing the unique identifying part of the summary.
/// Two different summaries may represent the same job at different times.
/// </summary>
/// <returns>The part common to all jobs.</returns>
public string AsIdentifyingString()
{
StringBuilder builder = new StringBuilder();
builder.AppendFormat("Cluster={0} ClusterJobID={1} Date={2} Machine={3} Name={4} User={5}",
this.Cluster, this.ClusterJobId, this.Date, this.Machine, this.Name, this.User);
return builder.ToString();
}
}
/// <summary>
/// This class is an abstraction of a cluster-level job, as opposed to a DryadLINQ job.
/// In Cosmos that's called a task, in HPC that's called a Job.
/// (In cosmos a task is a recurring job. In DryadLINQ running on top of cosmos, a task always contains exactly one job.)
/// </summary>
public class ClusterJobInformation : IEquatable<ClusterJobInformation>
{
/// <summary>
/// Status of a cluster job.
/// </summary>
public enum ClusterJobStatus
{
/// <summary>
/// Job is still running.
/// </summary>
Running,
/// <summary>
/// Job has finished successfully.
/// </summary>
Succeeded,
/// <summary>
/// Job has finished and has failed.
/// </summary>
Failed,
/// <summary>
/// Job has been cancelled. Not precise on cosmos clusters.
/// </summary>
Cancelled,
/// <summary>
/// Could not determine job status.
/// </summary>
Unknown,
};
/// <summary>
/// True if job is finished, false if not, or unknown.
/// </summary>
/// <param name="status">Job status.</param>
/// <returns>True if the job is no longer running.</returns>
public static bool JobIsFinished(ClusterJobStatus status)
{
switch (status)
{
case ClusterJobInformation.ClusterJobStatus.Failed:
case ClusterJobInformation.ClusterJobStatus.Succeeded:
case ClusterJobInformation.ClusterJobStatus.Cancelled:
return true;
case ClusterJobInformation.ClusterJobStatus.Running:
case ClusterJobInformation.ClusterJobStatus.Unknown:
return false;
default:
throw new InvalidDataException("Invalid job status " + status);
}
}
/// <summary>
/// Create a cluster job structure from a bunch of information.
/// </summary>
/// <param name="cluster">Cluster where the job is running.</param>
/// <param name="clusterJobGuid">Cluster job guid.</param>
/// <param name="jobName">Name of the cluster job.</param>
/// <param name="username">User who submitted cluster job.</param>
/// <param name="date">Last execution of cluster job.</param>
/// <param name="status">Execution status.</param>
/// <param name="runningTime">Time the job ran.</param>
/// <param name="virtualCluster">Cluster where the job has run.</param>
public ClusterJobInformation(
string cluster,
string virtualCluster,
string clusterJobGuid,
string jobName,
string username,
DateTime date,
TimeSpan runningTime,
ClusterJobStatus status)
{
this.VirtualCluster = virtualCluster;
this.Cluster = cluster;
this.ClusterJobID = clusterJobGuid;
this.Name = jobName;
this.User = username;
this.Date = date;
this.EstimatedRunningTime = runningTime;
this.Status = status;
this.JobSummary = null;
}
/// <summary>
/// Name of cluster job.
/// </summary>
public string Name { get; set; }
/// <summary>
/// Uset who submitted cluster job.
/// </summary>
public string User { get; set; }
/// <summary>
/// Date when job was submitted.
/// </summary>
public DateTime Date { get; set; }
/// <summary>
/// ID of Job on cluster.
/// </summary>
public string ClusterJobID { get; set; }
/// <summary>
/// Status of the execution.
/// </summary>
public ClusterJobStatus Status { get; set; }
/// <summary>
/// Cluster where the job ran.
/// </summary>
public string Cluster { get; set; }
/// <summary>
/// In some installations a cluster is composed of multiple virtual clusters.
/// </summary>
public string VirtualCluster { get; set; }
/// <summary>
/// Is the cluster job information still available on the cluster?
/// </summary>
public bool IsUnavailable { get; set; }
/// <summary>
/// Cache here the associated job, if available. Null if not cached.
/// </summary>
private DryadLinqJobSummary JobSummary { get; set; }
/// <summary>
/// Estimated time the job ran.
/// </summary>
public TimeSpan EstimatedRunningTime { get; set; }
/// <summary>
/// If known, set the associated job summary.
/// </summary>
/// <param name="summary">Job summary for this cluster job.</param>
public void SetAssociatedSummary(DryadLinqJobSummary summary)
{
this.JobSummary = summary;
}
/// <summary>
/// Discover the dryadlinq job associated with a cluster job.
/// </summary>
/// <param name="status">Cluster configuration.</param>
/// <returns>The job, if any</returns>
/// <param name="reporter">Delegate used to report errors.</param>
public DryadLinqJobSummary DiscoverDryadLinqJob(ClusterStatus status, StatusReporter reporter)
{
if (this.IsUnavailable)
return null;
if (this.JobSummary != null)
return this.JobSummary;
DryadLinqJobSummary j = status.DiscoverDryadLinqJobFromClusterJob(this, reporter);
if (j == null)
{
this.IsUnavailable = true;
}
return this.JobSummary = j;
}
/// <summary>
/// Copy the content of a cluster job.
/// </summary>
/// <param name="refresh">The value to copy from.</param>
internal void Copy(ClusterJobInformation refresh)
{
this.Name = refresh.Name;
this.Status = refresh.Status;
this.User = refresh.User;
this.JobSummary = refresh.JobSummary;
this.ClusterJobID = refresh.ClusterJobID;
this.Date = refresh.Date;
this.IsUnavailable = refresh.IsUnavailable;
this.VirtualCluster = refresh.VirtualCluster;
this.EstimatedRunningTime = refresh.EstimatedRunningTime;
}
/// <summary>
/// True if these two records represent the same job.
/// </summary>
/// <param name="other"></param>
/// <returns></returns>
public bool Equals(ClusterJobInformation other)
{
return
this.Cluster == other.Cluster &&
this.VirtualCluster == other.VirtualCluster &&
this.Name == other.Name &&
this.User == other.User &&
this.ClusterJobID == other.ClusterJobID &&
this.Date == other.Date;
}
}
}

24
JobBrowser/JOM/app.config
View File

@ -4,15 +4,15 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
@ -20,7 +20,23 @@
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
<bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>

372
JobBrowser/JOM/clusteraccess.cs
View File

@ -19,14 +19,20 @@ limitations under the License.
*/
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System;
using System.Collections.Generic;
using System.IO;
using System.Diagnostics;
using Microsoft.Research.Peloponnese.Storage;
using Microsoft.Research.Peloponnese.Azure;
using Microsoft.Research.Tools;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.Research.Peloponnese.Shared;
using Microsoft.Research.Peloponnese.WebHdfs;
using Microsoft.Research.Peloponnese.Hdfs;
namespace Microsoft.Research.JobObjectModel
{
@ -43,7 +49,8 @@ namespace Microsoft.Research.JobObjectModel
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary>
/// <returns>A stream that can be used to access the object contents.</returns>
ISharedStreamReader GetStream();
/// <param name="keepNewline">If true the streamReader will not strip the newlines.</param>
ISharedStreamReader GetStream(bool keepNewline);
/// <summary>
/// If the current object is a folder, it returns the contained objects.
/// </summary>
@ -203,12 +210,13 @@ namespace Microsoft.Research.JobObjectModel
/// A stream to the local cache, or null if the file is not cached.
/// </summary>
/// <returns>A stream to access the file.</returns>
public virtual ISharedStreamReader GetStream()
/// <param name="keepNewline">If true keep newlines.</param>
public virtual ISharedStreamReader GetStream(bool keepNewline)
{
if (this.LocalCachePath != null && File.Exists(this.LocalCachePath))
{
CachedClusterResidentObject.Record(this);
return new FileSharedStreamReader(this.LocalCachePath);
return new FileSharedStreamReader(this.LocalCachePath, keepNewline);
}
return null;
}
@ -355,14 +363,15 @@ namespace Microsoft.Research.JobObjectModel
/// The stream with the file contents.
/// </summary>
/// <returns>A stream reder.</returns>
public override ISharedStreamReader GetStream()
/// <param name="keepNewlines">If true keep the newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewlines)
{
try
{
if (!this.RepresentsAFolder)
{
//this.LocalCachePath = this.CachePath(this.Pathname);
ISharedStreamReader baseStream = base.GetStream();
ISharedStreamReader baseStream = base.GetStream(keepNewlines);
if (baseStream != null)
{
// file is cached
@ -378,12 +387,12 @@ namespace Microsoft.Research.JobObjectModel
throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream();
return new FileSharedStreamReader(this.Pathname.ToString(), writer, this.OnClose);
return new FileSharedStreamReader(this.Pathname.ToString(), writer, keepNewlines, this.OnClose);
}
else
{
// dont cache it
return new FileSharedStreamReader(this.Pathname.ToString());
return new FileSharedStreamReader(this.Pathname.ToString(), keepNewlines);
}
}
catch (Exception ex)
@ -650,9 +659,10 @@ namespace Microsoft.Research.JobObjectModel
/// The contents of the folder.
/// </summary>
/// <returns>The contents of the folder.</returns>
public override ISharedStreamReader GetStream()
/// <param name="keepNewline">If true keep newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewline)
{
return this.OriginalFolder.GetStream();
return this.OriginalFolder.GetStream(keepNewline);
}
/// <summary>
@ -720,9 +730,10 @@ namespace Microsoft.Research.JobObjectModel
/// A stream returning the contents.
/// </summary>
/// <returns>The contents of this object.</returns>
public ISharedStreamReader GetStream()
/// <param name="keepNewlines">If true keep newlines.</param>
public ISharedStreamReader GetStream(bool keepNewlines)
{
return new StringIteratorStreamReader(this.contents);
return new StringIteratorStreamReader(this.contents, keepNewlines);
}
/// <summary>
@ -835,9 +846,10 @@ namespace Microsoft.Research.JobObjectModel
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary>
/// <returns>A stream that can be used to access the object contents.</returns>
public ISharedStreamReader GetStream()
/// <param name="keepNewlines">If true keep the newlines.</param>
public ISharedStreamReader GetStream(bool keepNewlines)
{
return new FileSharedStreamReader(this.path);
return new FileSharedStreamReader(this.path, keepNewlines);
}
/// <summary>
@ -969,8 +981,8 @@ namespace Microsoft.Research.JobObjectModel
this.RepresentsAFolder = isFolder;
this.size = -1;
if (!string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
this.LocalCachePath = Path.Combine(CachedClusterResidentObject.CacheDirectory, this.path);
if (!String.IsNullOrEmpty(CacheDirectory))
this.LocalCachePath = Path.Combine(CacheDirectory, this.path);
}
/// <summary>
@ -982,14 +994,46 @@ namespace Microsoft.Research.JobObjectModel
protected set;
}
/// <summary>
/// Create a File URI from a file path.
/// </summary>
/// <param name="path">Path to file.</param>
/// <returns>The File uri.</returns>
/// <param name="config">Azure cluster configuration.</param>
public static Uri UriFromPath(AzureDfsClusterConfiguration config, string path)
{
if (path.StartsWith(config.Container))
throw new InvalidOperationException("Path contains container name");
var retval = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(config.AccountName, config.Container, path, null, config.AccountKey);
//Console.WriteLine("Uri {0}", retval);
return retval;
}
/// <summary>
/// Create a path from a URI.
/// </summary>
/// <param name="uri">URI of an HDFS file.</param>
/// <returns>The path to the file.</returns>
/// <param name="config">Azure cluster configuration.</param>
public static string PathFromUri(AzureDfsClusterConfiguration config, Uri uri)
{
string path = uri.LocalPath;
path = path.Trim('/');
if (path.StartsWith(config.Container))
path = path.Substring(config.Container.Length);
path = path.Trim('/');
return path;
}
/// <summary>
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary>
/// <returns>A stream that can be used to access the object contents.</returns>
public override ISharedStreamReader GetStream()
/// <param name="keepNewlines">If true keep the newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewlines)
{
ISharedStreamReader baseStream = base.GetStream();
ISharedStreamReader baseStream = base.GetStream(keepNewlines);
if (baseStream != null)
{
// file is cached
@ -1000,29 +1044,30 @@ namespace Microsoft.Research.JobObjectModel
Stream stream;
if (this.IsDfsStream)
{
var dfsFileStream = this.client.GetDfsFileStream(this.path);
stream = dfsFileStream.Stream;
Uri uri = UriFromPath(this.Config as AzureDfsClusterConfiguration, this.path);
stream = this.client.GetDfsStreamReader(uri);
}
else
{
string p = AzureDfsClusterStatus.GetBlobName(this.client.ContainerName,this.path);
stream = new AzureLogReaderStream(
this.client.AccountName,
this.client.AccountKey,
this.client.ContainerName,
this.path);
p);
}
long size = this.Size;
long sz = this.Size;
int bufferSize = 1024*1024;
if (size >= 0)
if (sz >= 0)
{
bufferSize = (int)(size/10);
bufferSize = (int)(sz/10);
if (bufferSize < 1024*1024)
bufferSize = 1024*1024;
if (bufferSize > 20*1024*1024)
bufferSize = 20*1024*1024;
}
StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, false, bufferSize);
SimpleStreamReader reader = new SimpleStreamReader(stream, true, Encoding.UTF8, false, bufferSize);
if (this.ShouldCacheLocally && this.LocalCachePath != null)
{
@ -1030,12 +1075,12 @@ namespace Microsoft.Research.JobObjectModel
if (this.RepresentsAFolder)
throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream();
return new SharedStreamReader(reader, writer, this.OnClose);
return new SharedStreamReader(reader, writer, keepNewlines, this.OnClose);
}
else
{
// dont cache it
return new SharedStreamReader(reader);
return new SharedStreamReader(reader, keepNewlines);
}
}
@ -1060,7 +1105,7 @@ namespace Microsoft.Research.JobObjectModel
if (item is CloudBlockBlob)
{
CloudBlockBlob blob = (CloudBlockBlob)item;
blocks.Add(blob.Name, blob.Properties.Length);
this.blocks.Add(blob.Name, blob.Properties.Length);
}
else if (item is CloudPageBlob)
{
@ -1072,8 +1117,8 @@ namespace Microsoft.Research.JobObjectModel
if (metadata.ContainsKey("writePosition"))
{
long sz;
if (long.TryParse(metadata["writePosition"], out sz))
pages.Add(pageBlob.Name, sz);
if (Int64.TryParse(metadata["writePosition"], out sz))
this.pages.Add(pageBlob.Name, sz);
}
}
else if (item is CloudBlobDirectory)
@ -1094,29 +1139,28 @@ namespace Microsoft.Research.JobObjectModel
this.PopulateCache();
long length = -1;
foreach (var child in this.client.EnumerateDirectory(this.path))
Uri uri = UriFromPath(this.Config as AzureDfsClusterConfiguration, this.path);
Console.WriteLine("AzureDfsFile.GetFileAndFolders({0}) -> {1}", this.path, uri);
foreach (var child in this.client.ExpandFileOrDirectory(uri))
{
Regex re = Utilities.RegexFromSearchPattern(match);
if (!re.IsMatch(child)) continue;
if (!re.IsMatch(child.AbsolutePath)) continue;
bool isFolder = false;
bool isDfsStream = false;
if (blocks.ContainsKey(child))
if (this.blocks.ContainsKey(child.AbsolutePath))
{
isDfsStream = true;
length = blocks[child];
length = this.blocks[child.AbsolutePath];
}
else if (pages.ContainsKey(child))
else if (this.pages.ContainsKey(child.AbsolutePath))
{
isDfsStream = false;
length = pages[child];
length = this.pages[child.AbsolutePath];
}
else if (this.client != null)
// otherwise this information may be incorrect
isFolder = true;
var file = new AzureDfsFile(this.Config, this.Job, this.client, child, this.ShouldCacheLocally, isFolder);
var file = new AzureDfsFile(this.Config, this.Job, this.client, PathFromUri(this.Config as AzureDfsClusterConfiguration, child), this.ShouldCacheLocally, isFolder);
file.IsDfsStream = isDfsStream;
file.size = length;
@ -1158,30 +1202,24 @@ namespace Microsoft.Research.JobObjectModel
{
this.PopulateCache();
string filepath;
if (this.client != null)
filepath = this.client.Combine(this.path, filename);
else
filepath = Path.Combine(this.path, filename);
string combined = Path.Combine(this.path, filename);
Uri filepath = UriFromPath(this.Config as AzureDfsClusterConfiguration, combined);
bool isFolder = false;
bool isDfsStream = false;
long sz = -1;
if (blocks.ContainsKey(filepath))
if (this.blocks.ContainsKey(combined))
{
isDfsStream = true;
sz = blocks[filepath];
sz = this.blocks[filepath.AbsolutePath];
}
else if (pages.ContainsKey(filepath))
else if (this.pages.ContainsKey(combined))
{
isDfsStream = false;
sz = pages[filepath];
sz = this.pages[filepath.AbsolutePath];
}
else if (this.client != null)
// if the client is null the information may be incorrect
isFolder = true;
var file = new AzureDfsFile(this.Config, this.Job, this.client, filepath, this.ShouldCacheLocally, isFolder);
var file = new AzureDfsFile(this.Config, this.Job, this.client, combined, this.ShouldCacheLocally, isFolder);
file.IsDfsStream = isDfsStream;
file.size = sz;
return file;
@ -1213,4 +1251,232 @@ namespace Microsoft.Research.JobObjectModel
return this.path;
}
}
/// <summary>
/// A file residing on HDFS (accessed either using HDFS or WebHdfs).
/// </summary>
public class DfsFile : CachedClusterResidentObject
{
private Uri baseUri;
private Uri uri;
private HdfsClientBase client;
/// <summary>
/// A file with the specified path.
/// </summary>
/// <param name="path">Path to the file.</param>
/// <param name="client">Azure client.</param>
/// <param name="config">Cluster configuration.</param>
/// <param name="job">Job accessing this file.</param>
/// <param name="jobFolderUri">Uri to base folder.</param>
/// <param name="isFolder">If true this must be a folder.</param>
/// <param name="canCache">True if the file can be cached (it is immutable for sure).</param>
public DfsFile(ClusterConfiguration config, Uri jobFolderUri, DryadLinqJobSummary job, HdfsClientBase client, string path, bool canCache, bool isFolder)
: base(config, job)
{
this.client = client;
this.Exception = null;
this.baseUri = jobFolderUri;
this.uri = UriFromPath(jobFolderUri, path);
this.ShouldCacheLocally = canCache;
this.RepresentsAFolder = isFolder;
this.size = -1;
Console.WriteLine("DfsFile Uri={0}", this.uri);
if (!string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
this.LocalCachePath = Path.Combine(CachedClusterResidentObject.CacheDirectory, PathFromUri(this.baseUri, this.uri));
}
/// <summary>
/// Create a File URI from a file path.
/// </summary>
/// <param name="baseUri">Base URI point to jobs folder.</param>
/// <param name="path">Path to file.</param>
/// <returns>The File uri.</returns>
public static Uri UriFromPath(Uri baseUri, string path)
{
UriBuilder builder = new UriBuilder(baseUri);
builder.Path = builder.Path.TrimEnd('/') + "/" + path.TrimStart('/');
return builder.Uri;
}
/// <summary>
/// Create a path from a URI.
/// </summary>
/// <param name="uri">URI of an HDFS file.</param>
/// <param name="baseUri">Base URI (should be a prefix of the other uri).</param>
/// <returns>The path to the file.</returns>
public static string PathFromUri(Uri baseUri, Uri uri)
{
string path = uri.PathAndQuery;
if (path.StartsWith(baseUri.PathAndQuery))
path = path.Substring(baseUri.PathAndQuery.Length);
path = path.Trim('/');
return path;
}
private DfsFile(ClusterConfiguration config, DryadLinqJobSummary job, Exception ex)
: base(config, job)
{
this.Exception = ex;
}
/// <summary>
/// True if the object is a folder.
/// </summary>
public override bool RepresentsAFolder
{
get;
protected set;
}
/// <summary>
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary>
/// <returns>A stream that can be used to access the object contents.</returns>
/// <param name="keepNewlines">If true keep the newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewlines)
{
ISharedStreamReader baseStream = base.GetStream(keepNewlines);
if (baseStream != null)
{
// file is cached
Trace.TraceInformation("Reading from local cache {0}", baseStream);
return baseStream;
}
Stream stream = this.client.GetDfsStreamReader(this.uri);
long sz = this.Size;
int bufferSize = 1024 * 1024;
if (sz >= 0)
{
bufferSize = (int)(sz / 10);
if (bufferSize < 1024 * 1024)
bufferSize = 1024 * 1024;
if (bufferSize > 20 * 1024 * 1024)
bufferSize = 20 * 1024 * 1024;
}
SimpleStreamReader reader = new SimpleStreamReader(stream, true, Encoding.UTF8, false, bufferSize);
if (this.ShouldCacheLocally && this.LocalCachePath != null)
{
// cache it
if (this.RepresentsAFolder)
throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream();
return new SharedStreamReader(reader, writer, keepNewlines, this.OnClose);
}
else
{
// dont cache it
return new SharedStreamReader(reader, keepNewlines);
}
}
/// <summary>
/// If the current object is a folder, it returns the contained objects.
/// </summary>
/// <returns>An iterator over all contained objects that match the specified string.</returns>
/// <param name="match">A shell expression (similar to the argument of Directory.GetFiles()).</param>
public override IEnumerable<IClusterResidentObject> GetFilesAndFolders(string match)
{
HashSet<Uri> folders = new HashSet<Uri>();
foreach (var child in this.client.EnumerateSubdirectories(this.uri))
{
folders.Add(child);
}
Regex re = Utilities.RegexFromSearchPattern(match);
foreach (var child in this.client.ExpandFileOrDirectory(this.uri))
{
if (!re.IsMatch(PathFromUri(this.baseUri, child))) continue;
bool isFolder = folders.Contains(child);
var file = new DfsFile(this.Config, this.baseUri, this.Job, this.client, PathFromUri(this.baseUri, child), this.ShouldCacheLocally, isFolder);
long length;
long time;
this.client.GetFileStatus(child, out time, out length);
file.size = length;
file.CreationTime = TimeFromLong(time);
yield return file;
}
}
private long size;
/// <summary>
/// Size of the object in bytes (if not a folder). The size can be -1 when it is unknown.
/// </summary>
public override long Size
{
get { return this.size; }
}
/// <summary>
/// Short name of the object.
/// </summary>
public override string Name
{
get { return Path.GetFileName(PathFromUri(this.baseUri, this.uri)); }
}
private static DateTime origin = new DateTime(1970, 1, 1);
/// <summary>
/// Convert file status time into a DateTime.
/// </summary>
/// <param name="fileTime">File time obtained from client.</param>
/// <returns>A DateTime object.</returns>
public static DateTime TimeFromLong(long fileTime)
{
return origin + TimeSpan.FromMilliseconds(fileTime);
}
/// <summary>
/// Date when object was created.
/// </summary>
public override DateTime CreationTime
{
get { return DateTime.Now; }
}
/// <summary>
/// For a folder object, returns the contained file with the specified name.
/// </summary>
/// <param name="filename">File name within the folder.</param>
/// <returns>The file within the folder.</returns>
public override IClusterResidentObject GetFile(string filename)
{
var matchingFiles = this.GetFilesAndFolders(filename).ToList();
if (matchingFiles.Count == 1)
return matchingFiles[0];
return new DfsFile(this.Config, this.Job, new Exception("Ambiguous name " + filename + " in " + this));
}
/// <summary>
/// For a folder object, returns the contained folder with the specified name.
/// </summary>
/// <param name="foldername">Folder name within the folder.</param>
/// <returns>The subfolder within the folder.</returns>
public override IClusterResidentObject GetFolder(string foldername)
{
var file = this.GetFile(foldername);
if (!file.RepresentsAFolder) throw new InvalidOperationException(foldername + " is not a folder");
return file;
}
/// <summary>
/// Returns a string that represents the current object.
/// </summary>
/// <returns>
/// A string that represents the current object.
/// </returns>
/// <filterpriority>2</filterpriority>
public override string ToString()
{
return this.uri.ToString();
}
}
}

47
JobBrowser/JOM/jobinfo.cs
View File

@ -19,8 +19,6 @@ limitations under the License.
*/
#undef USE_LINQ_TO_DRYAD
#undef USE_HPC
using System;
using System.Collections.Generic;
@ -660,6 +658,13 @@ namespace Microsoft.Research.JobObjectModel
return 0;
}
/// <summary>
/// Sometimes the StreamReader can read a partial line only.
/// We cache here the previously read line and if it looks like the current line is just a fragment, then we concatentate it
/// to the previous line. The line is a fragment if it does not start with 'logtimelocal'.
/// </summary>
private string previousLine = "";
/// <summary>
/// New JM stdout parsing code, for YARN-based DryadLINQ.
/// </summary>
@ -668,6 +673,17 @@ namespace Microsoft.Research.JobObjectModel
private bool ParseStdoutLineNew(string line)
{
if (string.IsNullOrWhiteSpace(line)) return true;
// The line should start with logtimelocal, otherwise it's probably a fragment of the previous line
this.previousLine += line;
if (!(line.EndsWith("\r") || line.EndsWith("\n")))
{
// line is incomplete, return now, parse later
return true;
}
line = this.previousLine.Trim();
this.previousLine = "";
Dictionary<string, string> kvp = Utilities.ParseCSVKVP(line);
if (kvp == null) return false;
@ -848,7 +864,7 @@ namespace Microsoft.Research.JobObjectModel
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
long totalRead = TryGetNumeric(kvp, "totalRead");
long tempRead = TryGetNumeric(kvp, "tempRead");
//long tempRead = TryGetNumeric(kvp, "tempRead");
long tempReadInRack = TryGetNumeric(kvp, "tempReadInRack");
long tempReadCrossRack = TryGetNumeric(kvp, "tempReadCrossRack");
long localRead = TryGetNumeric(kvp, "localRead");
@ -1456,7 +1472,7 @@ namespace Microsoft.Research.JobObjectModel
manager.Status(message, StatusKind.LongOp);
if (this.cachedStdoutReader == null)
this.cachedStdoutReader = file.GetStream();
this.cachedStdoutReader = file.GetStream(true);
if (this.cachedStdoutReader.Exception != null)
{
manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
@ -1472,11 +1488,15 @@ namespace Microsoft.Research.JobObjectModel
while (true)
{
manager.Token.ThrowIfCancellationRequested();
int startLine = currentLine;
bool completeLine = true;
try
{
completeLine = this.ParseStdoutLineNew(line);
if (this.ClusterConfiguration is DfsClusterConfiguration)
completeLine = this.ParseStdoutLineNew(line);
else
{
this.ParseStdoutLine(line);
}
}
catch (Exception ex)
{
@ -1486,9 +1506,8 @@ namespace Microsoft.Research.JobObjectModel
if (!completeLine)
{
if (this.cachedStdoutReader.EndOfStream)
{
throw new Exception("File ended while scanning for closing quote started on line " + startLine);
}
// no exception, the log may be truncated
break;
string extraline = this.cachedStdoutReader.ReadLine();
line += "\n" + extraline;
@ -1599,7 +1618,7 @@ namespace Microsoft.Research.JobObjectModel
{
bool success = true;
ISharedStreamReader sr = logfile.GetStream();
ISharedStreamReader sr = logfile.GetStream(false);
if (sr.Exception != null)
{
statusReporter("Exception while opening file " + logfile + ":" + sr.Exception.Message, StatusKind.Error);
@ -2042,7 +2061,7 @@ namespace Microsoft.Research.JobObjectModel
{
DryadJobStaticPlan retval;
{
retval = new DryadLinqJobStaticPlan(config, file.GetStream());
retval = new DryadLinqJobStaticPlan(config, file.GetStream(false));
}
retval.ParseQueryPlan(manager);
return retval;
@ -2723,7 +2742,7 @@ namespace Microsoft.Research.JobObjectModel
{
uri = uri.Substring(0, option);
}
this.LocalPath = uri.Substring(sepindex + 3);
this.LocalPath = uri.Substring(sepindex + 3).Trim();
if (uripathprefix != null) {
// Unfortunately the uri is absolute, although it should be relative sometimes. We fix this here.
@ -3293,7 +3312,7 @@ namespace Microsoft.Research.JobObjectModel
if (this.InputChannels != null)
// skip discovery
inputs = false;
ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream();
ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream(false);
var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, manager);
if (channels == null)
{
@ -3336,7 +3355,7 @@ namespace Microsoft.Research.JobObjectModel
manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error);
return false;
}
ISharedStreamReader sr = files.First().GetStream();
ISharedStreamReader sr = files.First().GetStream(false);
if (sr.Exception != null)
{
manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);

38
JobBrowser/JOM/packages.config
View File

@ -1,21 +1,25 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.21" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.3.2.0" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.28" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Client" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.WebHdfs" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.3.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.1.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.3.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="3.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
</packages>

2
JobBrowser/JOM/storage.cs
View File

@ -19,8 +19,6 @@ limitations under the License.
*/
#undef USE_DSC
#undef USE_TIDYFS
using System;
using System.Collections.Generic;

40
JobBrowser/JobBrowser/ClusterBrowser.cs
View File

@ -19,6 +19,7 @@ limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.ComponentModel;
@ -142,6 +143,8 @@ namespace Microsoft.Research.DryadAnalysis
ClusterConfiguration config = ClusterConfiguration.KnownClusterByName(clusterName);
this.Configuration = config;
this.diagnoseToolStripMenuItem.Visible = config.SupportsDiagnosis;
this.diagnoseToolStripMenuItem1.Visible = config.SupportsDiagnosis;
}
catch (Exception ex)
{
@ -602,12 +605,8 @@ namespace Microsoft.Research.DryadAnalysis
ToolStripItem strip = item.OwnerItem;
string clus = strip.Text;
var config = this.EditCluster(clus);
if (config != null)
{
ClusterConfiguration.AddKnownCluster(config);
this.AddClusterNameToMenu(config.Name);
}
var conf = this.EditCluster(clus);
this.ConfigurationChanged(conf);
}
void selItem_Click(object sender, EventArgs e)
@ -620,9 +619,8 @@ namespace Microsoft.Research.DryadAnalysis
this.ClusterSelected(clus);
}
void AddNewCluster(object sender, EventArgs e)
private void ConfigurationChanged(ClusterConfiguration conf)
{
ClusterConfiguration conf = this.EditCluster(null);
if (conf == null) return;
// you cannot have two cache clusters at the same time
@ -650,6 +648,13 @@ namespace Microsoft.Research.DryadAnalysis
ClusterConfiguration.AddKnownCluster(conf);
this.AddClusterNameToMenu(conf.Name);
this.Status("Added cluster " + conf.Name, StatusKind.OK);
}
void AddNewCluster(object sender, EventArgs e)
{
ClusterConfiguration conf = this.EditCluster(null);
this.ConfigurationChanged(conf);
}
/// <summary>
@ -661,10 +666,23 @@ namespace Microsoft.Research.DryadAnalysis
{
ClusterConfigEditor editor = new ClusterConfigEditor();
if (clusterName != null)
try
{
var config = ClusterConfiguration.KnownClusterByName(clusterName);
editor.SetConfigToEdit(config);
if (clusterName != null)
{
var config = ClusterConfiguration.KnownClusterByName(clusterName);
editor.SetConfigToEdit(config);
}
else
{
editor.SetConfigToEdit(null);
}
}
catch (Exception)
{
// This can happen when the cluster serialization has changed
// and we can no longer read the saved properties
editor.SetConfigToEdit(null);
}
DialogResult res = editor.ShowDialog();
if (res == System.Windows.Forms.DialogResult.OK)

133
JobBrowser/JobBrowser/ClusterConfigEditor.Designer.cs generated
View File

@ -49,13 +49,14 @@ namespace Microsoft.Research.JobObjectModel
private void InitializeComponent()
{
this.tableLayoutPanel_buttons = new System.Windows.Forms.TableLayoutPanel();
this.buttonOK = new System.Windows.Forms.Button();
this.buttonCancel = new System.Windows.Forms.Button();
this.buttonOK = new System.Windows.Forms.Button();
this.tableLayoutPanel_properties = new System.Windows.Forms.TableLayoutPanel();
this.label_name = new System.Windows.Forms.Label();
this.textBox_name = new System.Windows.Forms.TextBox();
this.label1 = new System.Windows.Forms.Label();
this.label2 = new System.Windows.Forms.Label();
this.label_type = new System.Windows.Forms.Label();
this.comboBox_clusterType = new System.Windows.Forms.ComboBox();
this.label_description = new System.Windows.Forms.Label();
this.panel1 = new System.Windows.Forms.Panel();
this.tableLayoutPanel_buttons.SuspendLayout();
this.tableLayoutPanel_properties.SuspendLayout();
@ -69,14 +70,26 @@ namespace Microsoft.Research.JobObjectModel
this.tableLayoutPanel_buttons.Controls.Add(this.buttonCancel, 1, 0);
this.tableLayoutPanel_buttons.Controls.Add(this.buttonOK, 0, 0);
this.tableLayoutPanel_buttons.Dock = System.Windows.Forms.DockStyle.Bottom;
this.tableLayoutPanel_buttons.Location = new System.Drawing.Point(0, 143);
this.tableLayoutPanel_buttons.Location = new System.Drawing.Point(0, 216);
this.tableLayoutPanel_buttons.Name = "tableLayoutPanel_buttons";
this.tableLayoutPanel_buttons.RowCount = 1;
this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100F));
this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 20F));
this.tableLayoutPanel_buttons.Size = new System.Drawing.Size(381, 33);
this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 33F));
this.tableLayoutPanel_buttons.Size = new System.Drawing.Size(389, 33);
this.tableLayoutPanel_buttons.TabIndex = 0;
//
// buttonCancel
//
this.buttonCancel.DialogResult = System.Windows.Forms.DialogResult.Cancel;
this.buttonCancel.Dock = System.Windows.Forms.DockStyle.Fill;
this.buttonCancel.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.buttonCancel.Location = new System.Drawing.Point(197, 3);
this.buttonCancel.Name = "buttonCancel";
this.buttonCancel.Size = new System.Drawing.Size(189, 27);
this.buttonCancel.TabIndex = 1;
this.buttonCancel.Text = "Cancel";
this.buttonCancel.UseVisualStyleBackColor = true;
//
// buttonOK
//
this.buttonOK.DialogResult = System.Windows.Forms.DialogResult.OK;
@ -84,101 +97,104 @@ namespace Microsoft.Research.JobObjectModel
this.buttonOK.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.buttonOK.Location = new System.Drawing.Point(3, 3);
this.buttonOK.Name = "buttonOK";
this.buttonOK.Size = new System.Drawing.Size(184, 27);
this.buttonOK.Size = new System.Drawing.Size(188, 27);
this.buttonOK.TabIndex = 0;
this.buttonOK.Text = "OK";
this.buttonOK.UseVisualStyleBackColor = true;
//
// buttonCancel
//
this.buttonCancel.DialogResult = System.Windows.Forms.DialogResult.Cancel;
this.buttonCancel.Dock = System.Windows.Forms.DockStyle.Fill;
this.buttonCancel.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.buttonCancel.Location = new System.Drawing.Point(193, 3);
this.buttonCancel.Name = "buttonCancel";
this.buttonCancel.Size = new System.Drawing.Size(185, 27);
this.buttonCancel.TabIndex = 1;
this.buttonCancel.Text = "Cancel";
this.buttonCancel.UseVisualStyleBackColor = true;
//
// tableLayoutPanel_properties
//
this.tableLayoutPanel_properties.AutoSize = true;
this.tableLayoutPanel_properties.ColumnCount = 2;
this.tableLayoutPanel_properties.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle());
this.tableLayoutPanel_properties.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100F));
this.tableLayoutPanel_properties.Controls.Add(this.label2, 0, 0);
this.tableLayoutPanel_properties.Controls.Add(this.textBox_name, 1, 0);
this.tableLayoutPanel_properties.Controls.Add(this.label1, 0, 1);
this.tableLayoutPanel_properties.Controls.Add(this.comboBox_clusterType, 1, 1);
this.tableLayoutPanel_properties.Controls.Add(this.comboBox_clusterType, 1, 2);
this.tableLayoutPanel_properties.Controls.Add(this.textBox_name, 1, 1);
this.tableLayoutPanel_properties.Controls.Add(this.label_name, 0, 1);
this.tableLayoutPanel_properties.Controls.Add(this.label_description, 0, 0);
this.tableLayoutPanel_properties.Controls.Add(this.label_type, 0, 2);
this.tableLayoutPanel_properties.Dock = System.Windows.Forms.DockStyle.Top;
this.tableLayoutPanel_properties.Location = new System.Drawing.Point(0, 0);
this.tableLayoutPanel_properties.Name = "tableLayoutPanel_properties";
this.tableLayoutPanel_properties.RowCount = 2;
this.tableLayoutPanel_properties.RowCount = 3;
this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle());
this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle());
this.tableLayoutPanel_properties.Size = new System.Drawing.Size(381, 54);
this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle());
this.tableLayoutPanel_properties.Size = new System.Drawing.Size(389, 66);
this.tableLayoutPanel_properties.TabIndex = 1;
//
// label_name
//
this.label_name.AutoSize = true;
this.label_name.Dock = System.Windows.Forms.DockStyle.Fill;
this.label_name.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.label_name.Location = new System.Drawing.Point(3, 19);
this.label_name.Name = "label_name";
this.label_name.Size = new System.Drawing.Size(80, 27);
this.label_name.TabIndex = 5;
this.label_name.Text = "Cluster name";
this.label_name.TextAlign = System.Drawing.ContentAlignment.MiddleRight;
//
// textBox_name
//
this.textBox_name.Dock = System.Windows.Forms.DockStyle.Fill;
this.textBox_name.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.textBox_name.Location = new System.Drawing.Point(89, 3);
this.textBox_name.Location = new System.Drawing.Point(89, 22);
this.textBox_name.Name = "textBox_name";
this.textBox_name.Size = new System.Drawing.Size(289, 21);
this.textBox_name.Size = new System.Drawing.Size(297, 21);
this.textBox_name.TabIndex = 3;
//
// label1
// label_type
//
this.label1.AutoSize = true;
this.label1.Dock = System.Windows.Forms.DockStyle.Fill;
this.label1.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.label1.Location = new System.Drawing.Point(3, 27);
this.label1.Name = "label1";
this.label1.Size = new System.Drawing.Size(80, 27);
this.label1.TabIndex = 4;
this.label1.Text = "Cluster type";
this.label1.TextAlign = System.Drawing.ContentAlignment.MiddleRight;
//
// label2
//
this.label2.AutoSize = true;
this.label2.Dock = System.Windows.Forms.DockStyle.Fill;
this.label2.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.label2.Location = new System.Drawing.Point(3, 0);
this.label2.Name = "label2";
this.label2.Size = new System.Drawing.Size(80, 27);
this.label2.TabIndex = 5;
this.label2.Text = "Cluster name";
this.label2.TextAlign = System.Drawing.ContentAlignment.MiddleRight;
this.label_type.AutoSize = true;
this.label_type.Dock = System.Windows.Forms.DockStyle.Fill;
this.label_type.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.label_type.Location = new System.Drawing.Point(3, 46);
this.label_type.Name = "label_type";
this.label_type.Size = new System.Drawing.Size(80, 20);
this.label_type.TabIndex = 4;
this.label_type.Text = "Cluster type";
this.label_type.TextAlign = System.Drawing.ContentAlignment.MiddleRight;
//
// comboBox_clusterType
//
this.comboBox_clusterType.Dock = System.Windows.Forms.DockStyle.Fill;
this.comboBox_clusterType.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.comboBox_clusterType.FormattingEnabled = true;
this.comboBox_clusterType.Location = new System.Drawing.Point(89, 30);
this.comboBox_clusterType.IntegralHeight = false;
this.comboBox_clusterType.Location = new System.Drawing.Point(89, 49);
this.comboBox_clusterType.Name = "comboBox_clusterType";
this.comboBox_clusterType.Size = new System.Drawing.Size(289, 23);
this.comboBox_clusterType.Size = new System.Drawing.Size(297, 23);
this.comboBox_clusterType.TabIndex = 6;
this.comboBox_clusterType.SelectedIndexChanged += new System.EventHandler(this.comboBox_clusterType_SelectedIndexChanged);
//
// label_description
//
this.label_description.AutoSize = true;
this.tableLayoutPanel_properties.SetColumnSpan(this.label_description, 2);
this.label_description.Dock = System.Windows.Forms.DockStyle.Fill;
this.label_description.Location = new System.Drawing.Point(3, 3);
this.label_description.Margin = new System.Windows.Forms.Padding(3);
this.label_description.Name = "label_description";
this.label_description.Size = new System.Drawing.Size(383, 13);
this.label_description.TabIndex = 7;
this.label_description.Text = "Description";
this.label_description.TextAlign = System.Drawing.ContentAlignment.MiddleCenter;
//
// panel1
//
this.panel1.Dock = System.Windows.Forms.DockStyle.Fill;
this.panel1.Location = new System.Drawing.Point(0, 54);
this.panel1.Location = new System.Drawing.Point(0, 0);
this.panel1.Name = "panel1";
this.panel1.Size = new System.Drawing.Size(381, 89);
this.panel1.Size = new System.Drawing.Size(389, 216);
this.panel1.TabIndex = 2;
//
// ClusterConfigEditor
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(381, 176);
this.Controls.Add(this.panel1);
this.ClientSize = new System.Drawing.Size(389, 249);
this.Controls.Add(this.tableLayoutPanel_properties);
this.Controls.Add(this.panel1);
this.Controls.Add(this.tableLayoutPanel_buttons);
this.Name = "ClusterConfigEditor";
this.Text = "Cluster Editor";
@ -196,11 +212,12 @@ namespace Microsoft.Research.JobObjectModel
private System.Windows.Forms.Button buttonCancel;
private System.Windows.Forms.Button buttonOK;
private System.Windows.Forms.TableLayoutPanel tableLayoutPanel_properties;
private System.Windows.Forms.Label label2;
private System.Windows.Forms.Label label_name;
private System.Windows.Forms.TextBox textBox_name;
private System.Windows.Forms.Label label1;
private System.Windows.Forms.Label label_type;
private System.Windows.Forms.ComboBox comboBox_clusterType;
private System.Windows.Forms.Panel panel1;
private System.Windows.Forms.Label label_description;
}
}

18
JobBrowser/JobBrowser/ClusterConfigEditor.cs
View File

@ -23,7 +23,6 @@ using System;
using System.Collections.Generic;
using System.Drawing;
using System.Windows.Forms;
using Microsoft.Research.Tools;
namespace Microsoft.Research.JobObjectModel
{
@ -43,9 +42,9 @@ namespace Microsoft.Research.JobObjectModel
this.addedControls = new List<Control>();
this.propEditor = new Dictionary<string, TextBox>();
for (var i = ClusterConfiguration.ClusterType.Unknown+1; i < ClusterConfiguration.ClusterType.MaxUnused; i++)
foreach (var type in ClusterConfiguration.Available)
{
this.comboBox_clusterType.Items.Add(i.ToString());
this.comboBox_clusterType.Items.Add(type.ToString());
}
}
@ -72,6 +71,7 @@ namespace Microsoft.Research.JobObjectModel
this.textBox_name.Text = props.Name;
this.comboBox_clusterType.SelectedItem = props.Type.ToString();
this.propEditor.Clear();
this.AddPropertiesToEdit(props.Properties);
}
@ -84,6 +84,8 @@ namespace Microsoft.Research.JobObjectModel
/// <param name="props">Properties to add controls for.</param>
private void AddPropertiesToEdit(List<PropertySetting> props)
{
this.label_description.Text = this.config.Description;
foreach (var prop in props)
{
Label labl = new Label();
@ -125,7 +127,7 @@ namespace Microsoft.Research.JobObjectModel
{
var type = (ClusterConfiguration.ClusterType)Enum.Parse(typeof(ClusterConfiguration.ClusterType), this.comboBox_clusterType.Text);
ClusterConfigurationSerialization ser = new ClusterConfigurationSerialization()
ClusterConfigurationSerialization ser = new ClusterConfigurationSerialization
{
Name = this.textBox_name.Text,
Type = type,
@ -150,13 +152,15 @@ namespace Microsoft.Research.JobObjectModel
private void comboBox_clusterType_SelectedIndexChanged(object sender, EventArgs e)
{
// initial setting for an existing configuration
if (this.config != null) return;
if (!this.canChangeType)
// we are just editing the cluster properties
return;
this.RemoveAddedProperties();
var type = (ClusterConfiguration.ClusterType)Enum.Parse(typeof(ClusterConfiguration.ClusterType), this.comboBox_clusterType.Text);
this.config = ClusterConfiguration.CreateConfiguration(type);
this.AddPropertiesToEdit(this.config.ExtractData().Properties);
var serialization = this.config.ExtractData();
this.AddPropertiesToEdit(serialization.Properties);
}
}
}

31
JobBrowser/JobBrowser/Diagnosis.cs
View File

@ -19,6 +19,7 @@ limitations under the License.
*/
using System;
using System.IO;
using System.Collections.Generic;
@ -413,16 +414,24 @@ namespace Microsoft.Research.DryadAnalysis
bool success = int.TryParse(m.Groups[3].Value, out channelNo);
if (!success)
return null;
this.Vertex.DiscoverChannels(true, false, true, manager);
var channels = this.Vertex.InputChannels;
if (channels == null)
return null;
if (channels.Count < channelNo)
try
{
this.Vertex.DiscoverChannels(true, false, true, manager);
var channels = this.Vertex.InputChannels;
if (channels == null)
return null;
if (channels.Count < channelNo)
{
this.Log(DiagnosisMessage.Importance.Error, "Could not discover channel " + channelNo, this.VertexName);
return null;
}
return channels[channelNo];
}
catch (Exception)
{
this.Log(DiagnosisMessage.Importance.Error, "Could not discover channel " + channelNo, this.VertexName);
return null;
}
return channels[channelNo];
}
/// <summary>
@ -434,7 +443,7 @@ namespace Microsoft.Research.DryadAnalysis
IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
if (stdout.Exception != null)
return Decision.Dontknow;
ISharedStreamReader sr = stdout.GetStream();
ISharedStreamReader sr = stdout.GetStream(false);
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
@ -458,7 +467,7 @@ namespace Microsoft.Research.DryadAnalysis
IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
if (stdout.Exception != null)
return false;
ISharedStreamReader sr = stdout.GetStream();
ISharedStreamReader sr = stdout.GetStream(false);
// only look for the error in the first 10 lines
for (int i = 0; i < 10; i++)
{
@ -496,7 +505,7 @@ namespace Microsoft.Research.DryadAnalysis
{
IClusterResidentObject logdir = this.Job.ClusterConfiguration.ProcessWorkDirectory(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
IClusterResidentObject stackTrace = logdir.GetFile(this.stackTraceFile);
ISharedStreamReader sr = stackTrace.GetStream();
ISharedStreamReader sr = stackTrace.GetStream(false);
if (sr.Exception == null)
{
@ -676,7 +685,7 @@ namespace Microsoft.Research.DryadAnalysis
return Decision.Dontknow;
}
ISharedStreamReader sr = jmstdout.GetStream();
ISharedStreamReader sr = jmstdout.GetStream(false);
if (sr.Exception != null)
{
this.Log(DiagnosisMessage.Importance.Tracing, "Could not read job manager standard output", sr.Exception.Message);

14
JobBrowser/JobBrowser/JobBrowser.cs
View File

@ -393,6 +393,10 @@ namespace Microsoft.Research.DryadAnalysis
this.graphViewer.InsertingEdge = false;
this.staticGraphZoomLevel = 0;
this.diagnoseToolStripMenuItem.Visible = job.ClusterConfiguration.SupportsDiagnosis;
this.diagnoseToolStripMenuItem1.Visible = job.ClusterConfiguration.SupportsDiagnosis;
this.diagnoseToolStripMenuItem2.Visible = job.ClusterConfiguration.SupportsDiagnosis;
this.planDrawSurface = new DrawingSurface2D(this.panel_jobSchedule);
this.planDrawSurface.SetMargins(4, 4, 4, 4);
this.panel_jobSchedule.MouseDoubleClick += this.panel_jobSchedule_MouseDoubleClick;
@ -1671,7 +1675,7 @@ namespace Microsoft.Research.DryadAnalysis
else
{
manager.Status("Extracting contents of " + path, StatusKind.LongOp);
ISharedStreamReader sr = path.GetStream();
ISharedStreamReader sr = path.GetStream(false);
if (sr.Exception != null)
{
error += " [Error " + sr.Exception.Message + "]";
@ -3010,7 +3014,7 @@ namespace Microsoft.Research.DryadAnalysis
long done = 0;
foreach (var file in files)
{
ISharedStreamReader sr = file.GetStream();
ISharedStreamReader sr = file.GetStream(false);
if (sr.Exception != null)
{
logViewer.Status("Error opening file: " + sr.Exception.Message, StatusKind.Error);
@ -3075,7 +3079,7 @@ namespace Microsoft.Research.DryadAnalysis
return false;
}
ISharedStreamReader sr = stdout.GetStream();
ISharedStreamReader sr = stdout.GetStream(false);
if (sr.Exception != null)
{
logViewer.Status("Error opening JM stdout: " + sr.Exception.Message, StatusKind.Error);
@ -3310,7 +3314,7 @@ namespace Microsoft.Research.DryadAnalysis
}
this.Status("Caching " + file.Name, StatusKind.LongOp);
ISharedStreamReader reader = file.GetStream();
ISharedStreamReader reader = file.GetStream(false);
// ReSharper disable UnusedVariable
foreach (string line in reader.ReadAllLines())
// ReSharper restore UnusedVariable
@ -3433,7 +3437,7 @@ namespace Microsoft.Research.DryadAnalysis
continue;
}
ISharedStreamReader reader = file.GetStream();
ISharedStreamReader reader = file.GetStream(false);
// ReSharper disable once UnusedVariable
foreach (string line in reader.ReadAllLines())
{

207
JobBrowser/JobBrowser/JobBrowser.csproj
View File

@ -1,6 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -10,7 +15,7 @@
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>JobBrowser</RootNamespace>
<AssemblyName>DryadLinqBrowser</AssemblyName>
<AssemblyName>JobBrowser</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<StartupObject>
@ -18,26 +23,7 @@
<IsWebBootstrapper>false</IsWebBootstrapper>
<ApplicationIcon>icons\DryadAnalysis.ico</ApplicationIcon>
<ApplicationManifest>app.manifest</ApplicationManifest>
<FileUpgradeFlags>
</FileUpgradeFlags>
<UpgradeBackupLocation>
</UpgradeBackupLocation>
<OldToolsVersion>3.5</OldToolsVersion>
<TargetFrameworkProfile />
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<NuGetPackageImportStamp>3753ab50</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
@ -62,130 +48,99 @@
<RunPostBuildEvent>OnBuildSuccess</RunPostBuildEvent>
</PropertyGroup>
<ItemGroup>
<Reference Include="EnvDTE, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE.dll</HintPath>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="EnvDTE80, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE80.dll</HintPath>
<HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="EnvDTE90, Version=9.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE90.dll</HintPath>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.Hadoop.Client, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.HDInsight.Net.Http.Formatting, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.HDInsight.Net.Http.Formatting.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hpc.Scheduler, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\Daphne\Dependences\Microsoft.Hpc.Scheduler.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Hpc.Scheduler.Properties, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\Daphne\Dependences\Microsoft.Hpc.Scheduler.Properties.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Msagl, Version=3.0.0.0, Culture=neutral, PublicKeyToken=7f84f738e1fc245c, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<Reference Include="Microsoft.Msagl">
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Msagl.Drawing, Version=3.0.0.0, Culture=neutral, PublicKeyToken=8a3d7c21d5fa1306, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<Reference Include="Microsoft.Msagl.Drawing">
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Msagl.GraphViewerGdi, Version=3.0.0.0, Culture=neutral, PublicKeyToken=fffc27ea4058b3a1, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<Reference Include="Microsoft.Msagl.GraphViewerGdi">
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.3.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.2.0\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.3.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Core" />
<Reference Include="System.Data.Linq" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
<HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
<HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.ServiceModel" />
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.Windows.Forms" />
<Reference Include="System.Xml.Linq">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.DataSetExtensions">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="System.Data" />
<Reference Include="System.Deployment" />
<Reference Include="System.Drawing" />
@ -281,52 +236,32 @@
<Content Include="icons\DryadAnalysis.ico" />
<Content Include="icons\JobBrowser.ico" />
</ItemGroup>
<ItemGroup>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.2.0">
<Visible>False</Visible>
<ProductName>.NET Framework 2.0 %28x86%29</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.0">
<Visible>False</Visible>
<ProductName>.NET Framework 3.0 %28x86%29</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.1-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.1-beta\build\Microsoft.Research.Peloponnese.Client.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

1
JobBrowser/JobBrowser/LocalDebuggingAndProfiling.cs
View File

@ -19,6 +19,7 @@ limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.IO;

2
JobBrowser/JobBrowser/LogViewer.cs
View File

@ -127,7 +127,7 @@ namespace Microsoft.Research.DryadAnalysis
this.Initialize(text, basefilename);
//ISharedStreamReader sr = new FileSharedStreamReader(filename);
ISharedStreamReader sr = file.GetStream();
ISharedStreamReader sr = file.GetStream(false);
long lineno = 0;
long bytes = 0;

10
JobBrowser/JobBrowser/Properties/AssemblyInfo.cs
View File

@ -25,12 +25,12 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("DryadLinqBrowser")]
[assembly: AssemblyTitle("JobBrowser")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("MSIT")]
[assembly: AssemblyProduct("DryadLinqBrowser")]
[assembly: AssemblyCopyright("Copyright © Microsoft")]
[assembly: AssemblyProduct("JobBrowser")]
[assembly: AssemblyCopyright("Copyright © MSIT 2008")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.1.0")]
[assembly: AssemblyFileVersion("1.0.1.0")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

20
JobBrowser/JobBrowser/app.config
View File

@ -7,23 +7,23 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.0.7" newVersion="1.1.0.7" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.0.7" newVersion="1.1.0.7" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
@ -31,7 +31,15 @@
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
<bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.3.2.0" newVersion="1.3.2.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>

38
JobBrowser/JobBrowser/packages.config
View File

@ -1,22 +1,26 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.21" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.3.2.0" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.28" targetFramework="net45" />
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Client" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.WebHdfs" version="0.8.1-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.3.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.1.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.3.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="3.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
</packages>

41
JobBrowser/Tools/Tools.csproj
View File

@ -8,36 +8,16 @@
<ProjectGuid>{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Artemis.Tools</RootNamespace>
<RootNamespace>Microsoft.Research.Calypso.Tools</RootNamespace>
<AssemblyName>Microsoft.Research.Calypso.Tools</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<FileUpgradeFlags>
</FileUpgradeFlags>
<OldToolsVersion>3.5</OldToolsVersion>
<UpgradeBackupLocation />
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DocumentationFile>bin\Debug\Microsoft.Hpc.Daphe.Tools.xml</DocumentationFile>
<DocumentationFile>bin\Debug\Microsoft.Research.Calypso.Tools.xml</DocumentationFile>
<DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
@ -71,23 +51,6 @@
<Compile Include="tools.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.

631
JobBrowser/Tools/tools.cs
View File

@ -1241,7 +1241,7 @@ namespace Microsoft.Research.Tools
/// <param name="url">Url to reach.</param>
/// <returns>A streamreader that returns the loaded web page.</returns>
/// <param name="credentials">Credentials to use.</param>
public static StreamReader Navigate(string url, ICredentials credentials)
public static Stream Navigate(string url, ICredentials credentials)
{
CookieContainer cookiejar = new CookieContainer();
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
@ -1258,9 +1258,7 @@ namespace Microsoft.Research.Tools
System.Diagnostics.Trace.Assert(resp.StatusCode == HttpStatusCode.OK);
Trace.TraceInformation("Received response");
// ReSharper disable once AssignNullToNotNullAttribute
StreamReader respReader = new StreamReader(resp.GetResponseStream());
return respReader;
return resp.GetResponseStream();
}
/// <summary>
@ -2887,55 +2885,10 @@ namespace Microsoft.Research.Tools
}
}
/// <summary>
/// An abstract class for streaming through files piece by piece.
/// </summary>
public interface IFileStreamer
{
/// <summary>
/// Done reading file.
/// </summary>
void Close();
/// <summary>
/// Read one "line" from the file.
/// </summary>
/// <returns>The line parsed into pieces.</returns>
string[] ReadLine();
/// <summary>
/// Read the file header.
/// </summary>
/// <returns>The line parsed into pieces.</returns>
string[] ReadHeader();
/// <summary>
/// Write the file header.
/// </summary>
/// <param name="header">Header to write.</param>
void WriteHeader(IEnumerable<string> header);
/// <summary>
/// Write a line in the file.
/// </summary>
/// <param name="line">Line to write.</param>
void WriteLine(IEnumerable<string> line);
/// <summary>
/// Start the stream from the beginning.
/// </summary>
void Reset();
/// <summary>
/// The whole contents of the file.
/// </summary>
/// <returns>The complete file contents.</returns>
IEnumerable<string[]> ReadFile();
/// <summary>
/// Line being parsed.
/// </summary>
long CurrentLineNumber { get; }
}
/// <summary>
/// Base implementation of file streamer.
/// </summary>
public abstract class BaseFileStreamer : IFileStreamer
public abstract class BaseFileStreamer
{
/// <summary>
/// True if the file is expected to contain a header.
@ -2984,7 +2937,8 @@ namespace Microsoft.Research.Tools
/// <param name="filename">File to access.</param>
/// <param name="mode">File mode.</param>
/// <param name="statusReporter">Delegate used to report errors.</param>
protected BaseFileStreamer(string filename, FileMode mode, StatusReporter statusReporter)
/// <param name="keepNewlines">If true keep newlines.</param>
protected BaseFileStreamer(string filename, FileMode mode, bool keepNewlines, StatusReporter statusReporter)
{
this.reader = null;
this.writer = null;
@ -2992,19 +2946,19 @@ namespace Microsoft.Research.Tools
this.mode = mode;
this.statusReporter = statusReporter;
// ReSharper disable once DoNotCallOverridableMethodsInConstructor
this.Reset();
this.Reset(keepNewlines);
}
/// <summary>
/// Go to the beginning.
/// </summary>
public virtual void Reset()
public virtual void Reset(bool keepNewlines)
{
this.Close();
switch (this.mode)
{
case FileMode.Open:
this.reader = new FileSharedStreamReader(filename);
this.reader = new FileSharedStreamReader(filename, keepNewlines);
if (this.reader.Exception != null)
throw this.reader.Exception;
this.writer = null;
@ -3150,7 +3104,7 @@ namespace Microsoft.Research.Tools
/// <param name="mode">Mode of access.</param>
/// <param name="statusReporter">Delegate used to report errors.</param>
public KVPFileStreamer(string filename, FileMode mode, StatusReporter statusReporter)
: base(filename, mode, statusReporter)
: base(filename, mode, false, statusReporter)
{
}
@ -3213,7 +3167,7 @@ namespace Microsoft.Research.Tools
/// <param name="hasHeader">True if the file contains a header.</param>
/// <param name="statusReporter">Delegate used to report errors.</param>
public CSVFileStreamer(string filename, FileMode mode, bool hasHeader, StatusReporter statusReporter)
: base(filename, mode, statusReporter)
: base(filename, mode, false, statusReporter)
{
this.HasHeader = hasHeader;
this.header = null;
@ -3580,6 +3534,10 @@ namespace Microsoft.Research.Tools
/// Delegate to call when stream is closed if caching.
/// </summary>
Action onClose;
/// <summary>
/// If true the reader keeps all newlines.
/// </summary>
private bool readerKeepsNewlines;
/// <summary>
/// A shared stream reader representing an exception.
@ -3594,16 +3552,19 @@ namespace Microsoft.Research.Tools
/// <summary>
/// Actual stream where the data is being read from.
/// </summary>
protected StreamReader actualReader;
protected ISimpleStreamReader actualReader;
/// <summary>
/// Create a stream reader for the specified stream.
/// </summary>
/// <param name="reader">Stream to read.</param>
public SharedStreamReader(StreamReader reader)
/// <param name="keepNewlines">Keep newlines in the input stream.</param>
public SharedStreamReader(ISimpleStreamReader reader, bool keepNewlines)
{
this.actualReader = reader;
this.cacheWriter = null;
this.readerKeepsNewlines = keepNewlines;
this.readerKeepsNewlines = false;
}
/// <summary>
@ -3612,8 +3573,10 @@ namespace Microsoft.Research.Tools
/// <param name="reader">Stream to read.</param>
/// <param name="cacheWriter">Use this stream to copy the file to a cache.</param>
/// <param name="onClose">Delegate to call when stream is completely read.</param>
public SharedStreamReader(StreamReader reader, StreamWriter cacheWriter, Action onClose)
/// <param name="readerKeepsNewlines">If true the reader keeps all newlines.</param>
public SharedStreamReader(ISimpleStreamReader reader, StreamWriter cacheWriter, bool readerKeepsNewlines, Action onClose)
{
this.readerKeepsNewlines = readerKeepsNewlines;
this.actualReader = reader;
this.cacheWriter = cacheWriter;
this.onClose = onClose;
@ -3624,8 +3587,10 @@ namespace Microsoft.Research.Tools
/// </summary>
/// <param name="cw">Stream used to write to the cache.</param>
/// <param name="onCl">Action to invoke on close.</param>
public void SetCacheWriter(StreamWriter cw, Action onCl)
/// <param name="keepNewlines">If true keep newlines.</param>
public void SetCacheWriter(StreamWriter cw, bool keepNewlines, Action onCl)
{
this.readerKeepsNewlines = keepNewlines;
this.cacheWriter = cw;
this.onClose = onCl;
}
@ -3635,8 +3600,9 @@ namespace Microsoft.Research.Tools
/// </summary>
public SharedStreamReader()
{
this.actualReader = StreamReader.Null;
this.actualReader = new WrapperSimpleStreamReader(StreamReader.Null);
this.cacheWriter = null;
this.readerKeepsNewlines = false;
}
/// <summary>
@ -3653,12 +3619,17 @@ namespace Microsoft.Research.Tools
/// <summary>
/// Read one line from the stream.
/// </summary>
/// <returns></returns>
/// <returns>The read line.</returns>
public override string ReadLine()
{
string line = this.actualReader.ReadLine();
if (this.cacheWriter != null)
this.cacheWriter.WriteLine(line);
{
if (this.readerKeepsNewlines)
this.cacheWriter.Write(line);
else
this.cacheWriter.WriteLine(line);
}
return line;
}
@ -3667,7 +3638,6 @@ namespace Microsoft.Research.Tools
/// </summary>
public override void Close()
{
if (this.actualReader != StreamReader.Null)
this.actualReader.Close();
if (this.cacheWriter != null)
this.cacheWriter.Close();
@ -3690,10 +3660,10 @@ namespace Microsoft.Research.Tools
/// <param name="token">Can be used to cancel the reading.</param>
public override string ReadToEnd(CancellationToken token)
{
string result = this.actualReader.ReadToEnd();
token.ThrowIfCancellationRequested();
string result = this.actualReader.ReadToEnd(token);
if (this.cacheWriter != null)
{
token.ThrowIfCancellationRequested();
this.cacheWriter.Write(result);
this.cacheWriter.Close();
if (this.onClose != null)
@ -3725,13 +3695,14 @@ namespace Microsoft.Research.Tools
/// Create a stream reader for the specified file.
/// </summary>
/// <param name="file">File to read.</param>
public FileSharedStreamReader(string file)
/// <param name="keepNewline">If true keep newlines.</param>
public FileSharedStreamReader(string file, bool keepNewline)
{
try
{
this.file = file;
Stream rd = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
this.actualReader = new StreamReader(rd);
Stream rd = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
this.actualReader = new SimpleStreamReader(rd, keepNewline);
}
catch (Exception ex)
@ -3749,13 +3720,14 @@ namespace Microsoft.Research.Tools
/// <param name="file">File to read from.</param>
/// <param name="cache">Cache here the contents read from the file.</param>
/// <param name="onClose">Action to invoke when file is closed.</param>
public FileSharedStreamReader(string file, StreamWriter cache, Action onClose)
/// <param name="keepNewlines">If true keep newlines.</param>
public FileSharedStreamReader(string file, StreamWriter cache, bool keepNewlines, Action onClose)
{
try
{
Stream rd = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
this.actualReader = new StreamReader(rd);
this.SetCacheWriter(cache, onClose);
this.actualReader = new SimpleStreamReader(rd, true);
this.SetCacheWriter(cache, keepNewlines, onClose);
}
catch (Exception ex)
{
@ -3788,6 +3760,7 @@ namespace Microsoft.Research.Tools
/// True if we have reached the end of the stream.
/// </summary>
bool endOfStream;
private bool keepNewlines;
/// <summary>
/// End of the stream.
@ -3810,9 +3783,11 @@ namespace Microsoft.Research.Tools
/// <summary>
/// A string iterator reading from this data.
/// </summary>
/// <param name="data"></param>
public StringIteratorStreamReader(IEnumerable<string> data)
/// <param name="data">Strings to read from; each is a "line".</param>
/// <param name="keepNewlines">If true keep the newlines.</param>
public StringIteratorStreamReader(IEnumerable<string> data, bool keepNewlines)
{
this.keepNewlines = keepNewlines;
this.contents = data.GetEnumerator();
this.endOfStream = !this.contents.MoveNext();
}
@ -3820,7 +3795,7 @@ namespace Microsoft.Research.Tools
/// <summary>
/// Read one line from the stream.
/// </summary>
/// <returns></returns>
/// <returns>One line from the set of strings.</returns>
public override string ReadLine()
{
string line = this.contents.Current;
@ -3844,4 +3819,510 @@ namespace Microsoft.Research.Tools
this.contents = null;
}
}
/// <summary>
/// A completely stripped-down StreamReader; only 3 public methods.
/// </summary>
public interface ISimpleStreamReader
: IDisposable
{
/// <summary>
/// Read one line of text.
/// </summary>
/// <returns>The read line.</returns>
string ReadLine();
/// <summary>
/// True if we have reached the end of stream.
/// </summary>
bool EndOfStream { get; }
/// <summary>
/// Read all the data from the stream.
/// </summary>
/// <param name="token">Cancellation token.</param>
/// <returns>All the data in the stream.</returns>
string ReadToEnd(CancellationToken token);
/// <summary>
/// Close the stream.
/// </summary>
void Close();
}
/// <summary>
/// A simple stream reader which wraps a true StreamReader.
/// This class is only here for compatibility purposes, it should be unused.
/// </summary>
public class WrapperSimpleStreamReader : ISimpleStreamReader
{
private StreamReader reader;
/// <summary>
/// Create a Wrapper around a stream reader.
/// </summary>
/// <param name="reader">Stream reader to wrap.</param>
public WrapperSimpleStreamReader(StreamReader reader)
{
this.reader = reader;
}
/// <summary>
/// Read one line of text.
/// </summary>
/// <returns>The read line.</returns>
public string ReadLine()
{
return this.reader.ReadLine();
}
/// <summary>
/// True if we have reached the end of stream.
/// </summary>
public bool EndOfStream
{
get { return this.reader.EndOfStream; }
}
/// <summary>
/// Read all the data from the stream.
/// </summary>
/// <param name="token">Cancellation token.</param>
/// <returns>All the data in the stream.</returns>
public string ReadToEnd(CancellationToken token)
{
token.ThrowIfCancellationRequested();
return this.reader.ReadToEnd();
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
/// <filterpriority>2</filterpriority>
public void Dispose()
{
this.reader.Dispose();
}
/// <summary>
/// Close the stream.
/// </summary>
public void Close()
{
this.reader.Close();
}
}
/// <summary>
/// A simple stream reader. This is almost like a standard StreamReader, but
/// has the option not to strip the end-of-line characters in ReadLine.
/// </summary>
public class SimpleStreamReader : ISimpleStreamReader
{
/// <summary>
/// Stream we are reading from.
/// </summary>
private Stream stream;
/// <summary>
/// If true do not strip the End of line characters.
/// </summary>
private bool keepEndOfLine;
/// <summary>
/// FIFO buffer.
/// </summary>
/// <typeparam name="T">Type of data stored in buffer.</typeparam>
private class Buffer<T>
{
/// <summary>
/// Data in buffer.
/// </summary>
public T[] Data;
/// <summary>
/// Last index of data available in the buffer.
/// </summary>
public int EndOfData;
/// <summary>
/// Current position for reading from buffer.
/// </summary>
public int CurrentPosition;
private const int minBufferSize = 4096;
/// <summary>
/// Allocate a buffer.
/// </summary>
/// <param name="size">Buffer size.</param>
public Buffer(int size)
{
if (size < minBufferSize)
size = minBufferSize;
this.Data = new T[size];
this.EndOfData = 0;
this.CurrentPosition = 0;
}
/// <summary>
/// Actual size of allocated buffer.
/// </summary>
public int Size
{
get
{
return this.Data.Length;
}
}
/// <summary>
/// True if the buffer is empty.
/// </summary>
/// <returns>A boolean indicating whether there is any data in the buffer.</returns>
public bool IsEmpty()
{
return this.CurrentPosition == this.EndOfData;
}
/// <summary>
/// Items available in the buffer.
/// </summary>
public int ItemsAvailable
{
get { return this.EndOfData - this.CurrentPosition; }
}
/// <summary>
/// Value at specified index in buffer.
/// </summary>
/// <param name="index">Index of value in buffer.</param>
/// <returns>The value in the buffer.</returns>
public T this[int index]
{
get
{
if (index < 0 || index > this.EndOfData)
throw new ArgumentOutOfRangeException("index", "must be between 0 and " + this.EndOfData);
return this.Data[index];
}
}
/// <summary>
/// Delete first items from the buffer.
/// </summary>
/// <param name="items">Number of items to delete.</param>
public void DeletePreamble(int items)
{
if (this.EndOfData < items)
throw new ArgumentException("Cannot delete " + items + " from buffer since only " + this.EndOfData + " are present");
Buffer.BlockCopy(this.Data, items, this.Data, 0, this.EndOfData - items);
this.EndOfData -= items;
}
}
/// <summary>
/// Buffer for read data.
/// </summary>
private Buffer<byte> byteBuffer;
/// <summary>
/// Buffer for decoded data.
/// </summary>
private Buffer<char> charBuffer;
private bool checkPreamble;
/// <summary>
/// Files may have a preamble which indicates the data encoding.
/// </summary>
private byte[] preamble;
/// <summary>
/// If true encoding must be detected.
/// </summary>
private bool encodingUnknown;
// ReSharper disable once PrivateFieldCanBeConvertedToLocalVariable
private Encoding dataEncoding;
private Decoder dataDecoder;
/// <summary>
/// Create a SimpleStream reader to read from a stream.
/// </summary>
/// <param name="Stream">Stream to read from.</param>
/// <param name="keepEndOfLine">If true do not strip the end of line characters.</param>
public SimpleStreamReader(Stream Stream, bool keepEndOfLine = false)
: this(Stream, keepEndOfLine, Encoding.UTF8, true)
{
}
/// <summary>
/// Create a SimpleStream reader reading from a stream.
/// </summary>
/// <param name="stream">Stream to read from.</param>
/// <param name="keepEndOfLine">If true do not strip the end of line characters.</param>
/// <param name="bufferSize">Size of buffer to use when reading from the underlying stream.</param>
/// <param name="encoding">Character encoding to use.</param>
/// <param name="detectEncoding">If true detect the encoding.</param>
public SimpleStreamReader(Stream stream, bool keepEndOfLine, Encoding encoding, bool detectEncoding, int bufferSize = 4096)
{
if (stream == null)
throw new ArgumentNullException("stream");
if (!stream.CanRead)
throw new ArgumentException("StreamNotReadable");
if (bufferSize <= 0)
throw new ArgumentOutOfRangeException("bufferSize");
this.dataEncoding = encoding;
this.dataDecoder = this.dataEncoding.GetDecoder();
this.stream = stream;
this.keepEndOfLine = keepEndOfLine;
this.byteBuffer = new Buffer<byte>(bufferSize);
int charSize = encoding.GetMaxCharCount(this.byteBuffer.Size);
this.charBuffer = new Buffer<char>(charSize);
this.encodingUnknown = detectEncoding;
this.preamble = encoding.GetPreamble();
this.checkPreamble = (this.preamble.Length > 0);
}
/// <summary>
/// Read one line of text.
/// </summary>
/// <returns>The read line.</returns>
public string ReadLine()
{
if (this.stream == null)
throw new InvalidOperationException("Reader closed");
if (this.charBuffer.IsEmpty())
{
if (this.ReadAndConvert() == 0) return null;
}
StringBuilder sb = new StringBuilder();
do
{
int i = this.charBuffer.CurrentPosition;
do
{
char ch = this.charBuffer[i];
if (ch == '\r' || ch == '\n')
{
int offset = this.keepEndOfLine ? 1 : 0;
sb.Append(this.charBuffer.Data, this.charBuffer.CurrentPosition, i + offset - this.charBuffer.CurrentPosition);
this.charBuffer.CurrentPosition = i + 1;
if (ch == '\r')
{
// check for \r\n
if (this.charBuffer.CurrentPosition < this.charBuffer.EndOfData || // one more character available
this.ReadAndConvert() > 0) // read next batch
{
if (this.charBuffer[this.charBuffer.CurrentPosition] == '\n')
this.charBuffer.CurrentPosition++;
if (this.keepEndOfLine)
sb.Append('\n');
}
}
return sb.ToString();
}
i++;
} while (i < this.charBuffer.EndOfData);
sb.Append(this.charBuffer.Data, this.charBuffer.CurrentPosition, this.charBuffer.ItemsAvailable);
}
while (this.ReadAndConvert() > 0);
return sb.ToString();
}
/// <summary>
/// True if we have reached the end of stream.
/// </summary>
public bool EndOfStream
{
get
{
if (this.stream == null)
throw new InvalidOperationException("Reader closed");
if (!this.charBuffer.IsEmpty())
return false;
int numRead = this.ReadAndConvert();
return numRead == 0;
}
}
private bool CheckForPreamble()
{
if (this.checkPreamble)
{
int len = (this.byteBuffer.EndOfData >= (this.preamble.Length)) ?
(this.preamble.Length - this.byteBuffer.CurrentPosition) : this.byteBuffer.ItemsAvailable;
for (int i = 0; i < len; i++)
{
if (this.byteBuffer.Data[this.byteBuffer.CurrentPosition + i] != this.preamble[this.byteBuffer.CurrentPosition + i])
{
this.checkPreamble = false;
return false;
}
}
// preamble found
this.byteBuffer.DeletePreamble(this.preamble.Length);
this.byteBuffer.CurrentPosition = 0;
this.checkPreamble = false;
this.encodingUnknown = false;
}
return this.checkPreamble;
}
/// <summary>
/// Read one buffer of bytes, and transfer them to a byte of chars.
/// Return number of chars transferred.
/// </summary>
/// <returns>The number of chars read.</returns>
private int ReadAndConvert()
{
this.charBuffer.EndOfData = this.charBuffer.CurrentPosition = 0;
if (!this.checkPreamble)
this.byteBuffer.EndOfData = 0;
do
{
if (this.checkPreamble)
{
int len = stream.Read(this.byteBuffer.Data, this.byteBuffer.CurrentPosition, this.byteBuffer.Size - this.byteBuffer.CurrentPosition);
if (len == 0)
{
if (this.byteBuffer.EndOfData > 0)
{
this.charBuffer.EndOfData += this.dataDecoder.GetChars(this.byteBuffer.Data, 0, this.byteBuffer.EndOfData, this.charBuffer.Data, this.charBuffer.EndOfData);
this.byteBuffer.CurrentPosition = this.byteBuffer.EndOfData = 0;
}
return this.charBuffer.EndOfData;
}
this.byteBuffer.EndOfData += len;
}
else
{
this.byteBuffer.EndOfData = stream.Read(this.byteBuffer.Data, 0, this.byteBuffer.Size);
if (this.byteBuffer.EndOfData == 0) // EOF
return this.charBuffer.EndOfData;
}
if (this.CheckForPreamble())
continue;
if (this.encodingUnknown && this.byteBuffer.EndOfData >= 2)
this.DetectEncoding();
this.charBuffer.EndOfData += this.dataDecoder.GetChars(this.byteBuffer.Data, 0, this.byteBuffer.EndOfData, this.charBuffer.Data, this.charBuffer.EndOfData);
} while (this.charBuffer.EndOfData == 0);
return this.charBuffer.EndOfData;
}
private void DetectEncoding()
{
// first few bytes in the buffer, starting at 0
int len = this.byteBuffer.EndOfData;
if (len < 2)
return;
this.encodingUnknown = false;
bool changed = false;
if (this.byteBuffer[0] == 0xFE && this.byteBuffer[1] == 0xFF)
{
// Big Endian Unicode
this.dataEncoding = new UnicodeEncoding(true, true);
this.byteBuffer.DeletePreamble(2);
changed = true;
}
else if (this.byteBuffer[0] == 0xFF && this.byteBuffer[1] == 0xFE)
{
if (len < 4 || this.byteBuffer[2] != 0 || this.byteBuffer[3] != 0)
{
this.dataEncoding = new UnicodeEncoding(false, true);
this.byteBuffer.DeletePreamble(2);
changed = true;
}
else
{
this.dataEncoding = new UTF32Encoding(false, true);
this.byteBuffer.DeletePreamble(4);
changed = true;
}
}
else if (len >= 3 && this.byteBuffer[0] == 0xEF && this.byteBuffer[1] == 0xBB && this.byteBuffer[2] == 0xBF)
{
this.dataEncoding = Encoding.UTF8;
this.byteBuffer.DeletePreamble(3);
changed = true;
}
else if (len >= 4 && this.byteBuffer[0] == 0 && this.byteBuffer[1] == 0 &&
this.byteBuffer[2] == 0xFE && this.byteBuffer[3] == 0xFF)
{
this.dataEncoding = new UTF32Encoding(true, true);
this.byteBuffer.DeletePreamble(4);
changed = true;
}
else if (len == 2)
this.encodingUnknown = true;
if (changed)
{
this.dataDecoder = this.dataEncoding.GetDecoder();
int maxCharsPerBuffer = this.dataEncoding.GetMaxCharCount(byteBuffer.Size);
this.charBuffer = new Buffer<char>(maxCharsPerBuffer);
}
}
/// <summary>
/// Read all the data from the stream.
/// </summary>
/// <param name="token">Cancellation token.</param>
/// <returns>All the data in the stream.</returns>
public string ReadToEnd(CancellationToken token)
{
if (this.stream == null)
throw new InvalidOperationException("Stream closed");
StringBuilder builder = new StringBuilder();
do
{
token.ThrowIfCancellationRequested();
builder.Append(this.charBuffer.Data, this.charBuffer.CurrentPosition, this.charBuffer.ItemsAvailable);
this.charBuffer.CurrentPosition = this.charBuffer.EndOfData;
this.ReadAndConvert();
} while (this.charBuffer.EndOfData > 0);
return builder.ToString();
}
/// <summary>
/// Close the stream.
/// </summary>
public void Close()
{
this.Dispose(true);
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
/// <filterpriority>2</filterpriority>
public void Dispose()
{
this.Dispose(false);
}
/// <summary>
/// Internal dispose method.
/// </summary>
/// <param name="disposing"></param>
protected void Dispose(bool disposing)
{
this.stream.Dispose();
this.stream = null;
this.charBuffer = null;
this.byteBuffer = null;
}
}
}

43
JobBrowser/UsefulForms/UsefulForms.csproj
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -12,26 +12,6 @@
<AssemblyName>Microsoft.Research.Calypso.UsefulForms</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<FileUpgradeFlags>
</FileUpgradeFlags>
<OldToolsVersion>3.5</OldToolsVersion>
<UpgradeBackupLocation />
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
@ -56,9 +36,7 @@
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Core" />
<Reference Include="System.Data" />
<Reference Include="System.Drawing" />
<Reference Include="System.Windows.Forms" />
@ -109,23 +87,6 @@
<Name>Tools</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.

12
LinqToDryad/CodeGenHelper.cs
View File

@ -57,7 +57,8 @@ namespace Microsoft.Research.DryadLinq.Internal
DynamicMethod dm = new DynamicMethod("GetObjField",
typeof(S),
new Type[] { typeT },
typeT);
typeT,
true);
ILGenerator ilgen = dm.GetILGenerator();
ilgen.Emit(OpCodes.Ldarg_0);
ilgen.Emit(OpCodes.Ldfld, finfo);
@ -83,7 +84,8 @@ namespace Microsoft.Research.DryadLinq.Internal
DynamicMethod dm = new DynamicMethod("SetObjField",
typeof(void),
new Type[] { typeT, typeof(S) },
typeT);
typeT,
true);
ILGenerator ilgen = dm.GetILGenerator();
ilgen.Emit(OpCodes.Ldarg_0);
ilgen.Emit(OpCodes.Ldarg_1);
@ -110,7 +112,8 @@ namespace Microsoft.Research.DryadLinq.Internal
DynamicMethod dm = new DynamicMethod("GetStructField",
typeof(S),
new Type[] { typeT.MakeByRefType() },
typeT);
typeT,
true);
ILGenerator ilgen = dm.GetILGenerator();
ilgen.Emit(OpCodes.Ldarg_0);
ilgen.Emit(OpCodes.Ldfld, finfo);
@ -136,7 +139,8 @@ namespace Microsoft.Research.DryadLinq.Internal
DynamicMethod dm = new DynamicMethod("SetStructField",
typeof(void),
new Type[] { typeT.MakeByRefType(), typeof(S) },
typeT);
typeT,
true);
ILGenerator ilgen = dm.GetILGenerator();
ilgen.Emit(OpCodes.Ldarg_0);
ilgen.Emit(OpCodes.Ldarg_1);

53
LinqToDryad/Constants.cs
View File

@ -69,20 +69,47 @@ namespace Microsoft.Research.DryadLinq
public const string debugAzure = "DEBUG_AZURE";
// Recognized values are: OFF, CRITICAL, ERROR, WARN, INFO, VERBOSE
public const string TraceLevelEnvVar = "CCP_DRYADTRACELEVEL";
public const string TraceOff = "OFF";
public const string TraceCritical = "CRITICAL";
public const string TraceError = "ERROR";
public const string TraceWarning = "WARN";
public const string TraceInfo = "INFO";
public const string TraceVerbose = "VERBOSE";
public const string LoggingLevelEnvVar = "DRYAD_LOGGING_LEVEL";
public const string LoggingOff = "OFF";
public const string LoggingCritical = "CRITICAL";
public const string LoggingError = "ERROR";
public const string LoggingWarning = "WARN";
public const string LoggingInfo = "INFO";
public const string LoggingVerbose = "VERBOSE";
public const int TraceOffLevel = 0;
public const int TraceCriticalLevel = 1;
public const int TraceErrorLevel = 3;
public const int TraceWarningLevel = 7;
public const int TraceInfoLevel = 15;
public const int TraceVerboseLevel = 31;
public const int LoggingOffLevel = 0;
public const int LoggingCriticalLevel = 1;
public const int LoggingErrorLevel = 3;
public const int LoggingWarningLevel = 7;
public const int LoggingInfoLevel = 15;
public const int LoggingVerboseLevel = 31;
public static string LoggingStringFromLevel(int level)
{
switch (level)
{
case LoggingOffLevel:
return LoggingOff;
case LoggingCriticalLevel:
return LoggingCritical;
case LoggingErrorLevel:
return LoggingError;
case LoggingWarningLevel:
return LoggingWarning;
case LoggingInfoLevel:
return LoggingInfo;
case LoggingVerboseLevel:
return LoggingVerbose;
default:
throw new ApplicationException("Unknown logging level " + level);
}
}
// SchedulerHelper environment variables
public const string clusterNameEnvVar = "CCP_CLUSTER_NAME";

2
LinqToDryad/DataPath.cs
View File

@ -34,7 +34,7 @@ namespace Microsoft.Research.DryadLinq
{
internal static class DataPath
{
internal const string TEMPORARY_STREAM_NAME_PREFIX = "DryadLinqTemp/";
internal const string TEMPORARY_STREAM_NAME_PREFIX = "DryadLinqTemp";
internal const string DSC_URI_SCHEME = "hpcdsc";
internal const string HDFS_URI_SCHEME = "hdfs";

176
LinqToDryad/DataProvider.cs
View File

@ -35,7 +35,7 @@ using System.Diagnostics;
using Microsoft.Research.DryadLinq.Internal;
using System.IO.Compression;
using Microsoft.Research.Peloponnese.Storage;
using Microsoft.Research.Peloponnese.Azure;
namespace Microsoft.Research.DryadLinq
{
@ -126,12 +126,14 @@ namespace Microsoft.Research.DryadLinq
/// <param name="metaData">The metadata for the collection.</param>
/// <param name="outputScheme">The compression scheme used to store the collection.</param>
/// <param name="isTemp">true to only store the collection temporarily with a time lease.</param>
/// <param name="serializer">A stream-based serializer.</param>
public abstract void Ingress<T>(DryadLinqContext context,
IEnumerable<T> source,
Uri dataSetUri,
DryadLinqMetaData metaData,
CompressionScheme outputScheme,
bool isTemp = false);
bool isTemp,
Expression<Action<IEnumerable<T>, Stream>> serializer);
/// <summary>
/// Creates an instance of Stream for a dataset at a specified location. This is
@ -197,28 +199,35 @@ namespace Microsoft.Research.DryadLinq
/// <typeparam name="T">The record type of the dataset.</typeparam>
/// <param name="context">An instance of <see cref="DryadLinqContext"/></param>
/// <param name="dataSetUri">The URI of the dataset</param>
/// <param name="deserializer">A stream-based deserializer</param>
/// <returns>A query object representing the specified dataset.</returns>
internal static DryadLinqQuery<T> GetPartitionedTable<T>(DryadLinqContext context, Uri dataSetUri)
internal static DryadLinqQuery<T>
GetPartitionedTable<T>(DryadLinqContext context,
Uri dataSetUri,
Expression<Func<Stream, IEnumerable<T>>> deserializer)
{
string scheme = DataPath.GetScheme(dataSetUri);
DataProvider dataProvider = DataProvider.GetDataProvider(scheme);
dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri);
return new DryadLinqQuery<T>(context, dataProvider, dataSetUri);
return new DryadLinqQuery<T>(context, dataProvider, dataSetUri, deserializer);
}
/// <summary>
/// Reads a specified dataset.
/// Reads the dataset specified by a URI.
/// </summary>
/// <typeparam name="T">The record type of the dataset.</typeparam>
/// <typeparam name="T">The record type of the dataset</typeparam>
/// <param name="context">An instance of <see cref="DryadLinqContext"/></param>
/// <param name="dataSetUri">The URI of the dataset.</param>
/// <returns>A sequence of records as IEnumerable{T}.</returns>
public static IEnumerable<T> ReadData<T>(DryadLinqContext context, Uri dataSetUri)
/// <param name="dataSetUri">The URI of the dataset</param>
/// <param name="deserializer">A stream-based deserializer</param>
/// <returns>A sequence of records as IEnumerable{T}</returns>
internal static IEnumerable<T> ReadData<T>(DryadLinqContext context,
Uri dataSetUri,
Expression<Func<Stream, IEnumerable<T>>> deserializer)
{
string scheme = DataPath.GetScheme(dataSetUri);
DataProvider dataProvider = DataProvider.GetDataProvider(scheme);
dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri);
return new DryadLinqQueryEnumerable<T>(context, dataProvider, dataSetUri);
return new DryadLinqQueryEnumerable<T>(context, dataProvider, dataSetUri, deserializer);
}
/// <summary>
@ -231,19 +240,23 @@ namespace Microsoft.Research.DryadLinq
/// <param name="metaData">The metadata of the data.</param>
/// <param name="outputScheme">The compression scheme.</param>
/// <param name="isTemp">true if the data is only stored temporarily.</param>
/// <param name="serializer">A stream-based serializer</param>
/// <param name="deserializer">A stream-based deserializer</param>
/// <returns>An instance of IQueryable{T} for the data.</returns>
internal static DryadLinqQuery<T> StoreData<T>(DryadLinqContext context,
IEnumerable<T> source,
Uri dataSetUri,
DryadLinqMetaData metaData,
CompressionScheme outputScheme,
bool isTemp = false)
bool isTemp,
Expression<Action<IEnumerable<T>, Stream>> serializer,
Expression<Func<Stream, IEnumerable<T>>> deserializer)
{
string scheme = DataPath.GetScheme(dataSetUri);
DataProvider dataProvider = DataProvider.GetDataProvider(scheme);
dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri);
dataProvider.Ingress(context, source, dataSetUri, metaData, outputScheme, isTemp);
DryadLinqQuery<T> res = DataProvider.GetPartitionedTable<T>(context, dataSetUri);
dataProvider.Ingress(context, source, dataSetUri, metaData, outputScheme, isTemp, serializer);
DryadLinqQuery<T> res = DataProvider.GetPartitionedTable<T>(context, dataSetUri, deserializer);
res.CheckAndInitialize(); // must initialize
return res;
}
@ -285,7 +298,7 @@ namespace Microsoft.Research.DryadLinq
public override Uri GetTemporaryStreamUri(DryadLinqContext context, string path)
{
return context.Cluster.MakeDefaultUri(DataPath.TEMPORARY_STREAM_NAME_PREFIX + path);
return context.Cluster.MakeInternalClusterUri("tmp", DataPath.TEMPORARY_STREAM_NAME_PREFIX, path);
}
public override DryadLinqMetaData GetMetaData(DryadLinqContext context, Uri dataSetUri)
@ -294,11 +307,28 @@ namespace Microsoft.Research.DryadLinq
return null;
}
public override Uri RewriteUri<T>(DryadLinqContext context, Uri dataSetUri, FileAccess access)
{
UriBuilder builder = new UriBuilder(dataSetUri);
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(builder.Query);
if (access != FileAccess.Write &&
typeof(T) == typeof(Microsoft.Research.DryadLinq.LineRecord))
{
query["seekBoundaries"] = "Microsoft.Research.DryadLinq.LineRecord";
}
builder.Query = query.ToString();
return builder.Uri;
}
public override DryadLinqStreamInfo GetStreamInfo(DryadLinqContext context, Uri dataSetUri)
{
Int32 parCnt = 0;
Int64 size = -1;
context.Cluster.DfsClient.GetContentSummary(dataSetUri.AbsolutePath, ref size, ref parCnt);
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(dataSetUri.Query);
bool expandBlocks = (query["seekboundaries"] == "Microsoft.Research.DryadLinq.LineRecord");
context.GetHdfsClient.GetDirectoryContentSummary(dataSetUri, expandBlocks, ref size, ref parCnt);
if (parCnt == 0)
{
throw new DryadLinqException("Got 0 partition count for " + dataSetUri.AbsoluteUri);
@ -311,26 +341,36 @@ namespace Microsoft.Research.DryadLinq
Uri dataSetUri,
DryadLinqMetaData metaData,
CompressionScheme outputScheme,
bool isTemp = false)
bool isTemp,
Expression<Action<IEnumerable<T>, Stream>> serializer)
{
throw new DryadLinqException("TBA");
DryadLinqFactory<T> factory = (DryadLinqFactory<T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
using (Stream stream = context.GetHdfsClient.GetDfsStreamWriter(dataSetUri))
{
DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream);
DryadLinqRecordWriter<T> writer = factory.MakeWriter(nativeStream);
foreach (T rec in source)
{
writer.WriteRecordSync(rec);
}
writer.Close();
}
}
public override Stream Egress(DryadLinqContext context, Uri dataSetUri)
{
throw new DryadLinqException("TBA");
return context.GetHdfsClient.GetDfsDirectoryStreamReader(dataSetUri);
}
public override void CheckExistence(DryadLinqContext context, Uri dataSetUri, bool deleteIfExists)
{
WebHdfsClient client = new WebHdfsClient(dataSetUri.Host, 8033, 50070);
if (client.IsFileExists(dataSetUri.AbsolutePath))
if (context.GetHdfsClient.IsFileExists(dataSetUri))
{
if (!deleteIfExists)
{
throw new DryadLinqException("Can't output to existing HDFS collection " + dataSetUri.AbsoluteUri);
}
client.DeleteDfsFile(dataSetUri.AbsolutePath);
context.GetHdfsClient.DeleteDfsFile(dataSetUri, true);
}
}
}
@ -380,48 +420,58 @@ namespace Microsoft.Research.DryadLinq
Uri dataSetUri,
DryadLinqMetaData metaData,
CompressionScheme compressionScheme,
bool isTemp = false)
bool isTemp,
Expression<Action<IEnumerable<T>, Stream>> serializer)
{
string fileName = dataSetUri.LocalPath;
if (!String.IsNullOrEmpty(dataSetUri.Host))
{
fileName = @"\\" + dataSetUri.Host + fileName;
}
// Write the partition:
string partDir = context.PartitionUncPath;
if (partDir == null)
{
partDir = Path.GetDirectoryName(dataSetUri.LocalPath);
}
if (!Path.IsPathRooted(partDir))
{
partDir = Path.Combine("/", partDir);
}
string partDir = Path.GetDirectoryName(fileName);
partDir = Path.Combine(partDir, DryadLinqUtil.MakeUniqueName());
Directory.CreateDirectory(partDir);
string partPath = Path.Combine(partDir, "Part");
string partFilePath = partPath + ".00000000";
string uncPath = Path.Combine(partDir, "Part");
string partitionPath = uncPath + ".00000000";
DryadLinqFactory<T> factory = (DryadLinqFactory<T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
using (FileStream fstream = new FileStream(partFilePath, FileMode.CreateNew, FileAccess.Write))
using (FileStream fstream = new FileStream(partitionPath, FileMode.CreateNew, FileAccess.Write))
{
DryadLinqFileBlockStream nativeStream = new DryadLinqFileBlockStream(fstream, compressionScheme);
DryadLinqRecordWriter<T> writer = factory.MakeWriter(nativeStream);
foreach (T rec in source)
if (serializer == null)
{
writer.WriteRecordSync(rec);
DryadLinqFileBlockStream nativeStream = new DryadLinqFileBlockStream(fstream, compressionScheme);
DryadLinqRecordWriter<T> writer = factory.MakeWriter(nativeStream);
foreach (T rec in source)
{
writer.WriteRecordSync(rec);
}
writer.Close();
}
else
{
Action<IEnumerable<T>, Stream> serializerFunc = serializer.Compile();
serializerFunc(source, fstream);
}
writer.Close();
}
// Write the partfile:
FileInfo finfo = new FileInfo(partFilePath);
using (StreamWriter writer = File.CreateText(dataSetUri.LocalPath))
long partSize = new FileInfo(partitionPath).Length;
using (StreamWriter writer = File.CreateText(fileName))
{
writer.WriteLine(partPath);
writer.WriteLine(uncPath);
writer.WriteLine("1");
writer.WriteLine("{0},{1},{2}", 0, finfo.Length, Environment.MachineName);
writer.WriteLine("{0},{1}", 0, partSize);
}
}
public override Stream Egress(DryadLinqContext context, Uri dataSetUri)
{
string fileName = dataSetUri.LocalPath;
if (!String.IsNullOrEmpty(dataSetUri.Host))
{
fileName = @"\\" + dataSetUri.Host + fileName;
}
var lines = File.ReadAllLines(fileName);
if (lines.Length < 3)
{
@ -434,6 +484,10 @@ namespace Microsoft.Research.DryadLinq
public override void CheckExistence(DryadLinqContext context, Uri dataSetUri, bool deleteIfExists)
{
string fileName = dataSetUri.LocalPath;
if (!String.IsNullOrEmpty(dataSetUri.Host))
{
fileName = @"\\" + dataSetUri.Host + fileName;
}
if (File.Exists(fileName))
{
if (!deleteIfExists)
@ -460,18 +514,17 @@ namespace Microsoft.Research.DryadLinq
private string[] GetPartitionPaths(string[] lines)
{
bool isLocalPath = lines[0].Contains(':');
string[] filePathArray = new string[lines.Length - 2];
for (int i = 2; i < lines.Length; i++)
{
int idx = i - 2;
string[] fields = lines[i].Split(',');
if (fields[2].Contains(':'))
if (fields.Length > 2 && fields[2].Contains(':'))
{
string[] parts = fields[2].Split(':');
filePathArray[idx] = String.Format(@"\\{0}\{1}", parts[0], parts[1]);
}
else if (isLocalPath)
else if (Path.IsPathRooted(lines[0]))
{
filePathArray[idx] = String.Format("{0}.{1:X8}", lines[0], idx);
}
@ -501,13 +554,13 @@ namespace Microsoft.Research.DryadLinq
public override Uri GetTemporaryStreamUri(DryadLinqContext context, string path)
{
return context.Cluster.MakeDefaultUri(DataPath.TEMPORARY_STREAM_NAME_PREFIX + path);
return context.Cluster.MakeInternalClusterUri(DataPath.TEMPORARY_STREAM_NAME_PREFIX, path);
}
public override Uri RewriteUri<T>(DryadLinqContext context, Uri dataSetUri, FileAccess access)
{
string account, key, container, blob;
AzureUtils.FromAzureUri(dataSetUri, out account, out key, out container, out blob);
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dataSetUri, out account, out key, out container, out blob);
UriBuilder builder = new UriBuilder(dataSetUri);
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(builder.Query);
@ -563,25 +616,34 @@ namespace Microsoft.Research.DryadLinq
Uri dataSetUri,
DryadLinqMetaData metaData,
CompressionScheme compressionScheme,
bool isTemp = false)
bool isTemp,
Expression<Action<IEnumerable<T>, Stream>> serializer)
{
string account, key, container, blob;
AzureUtils.FromAzureUri(dataSetUri, out account, out key, out container, out blob);
Utils.FromAzureUri(dataSetUri, out account, out key, out container, out blob);
if (compressionScheme != CompressionScheme.None)
{
throw new DryadLinqException("Not implemented: writing to Azure temporary storage with compression enabled");
}
AzureDfsClient client = new AzureDfsClient(account, key, container);
DryadLinqFactory<T> factory = (DryadLinqFactory<T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
using (Stream stream = client.GetFileStreamWriterAsync(blob).Result)
using (Stream stream = client.GetDfsStreamWriterAsync(dataSetUri).Result)
{
DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream);
DryadLinqRecordWriter<T> writer = factory.MakeWriter(nativeStream);
foreach (T rec in source)
if (serializer == null)
{
writer.WriteRecordSync(rec);
DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream);
DryadLinqRecordWriter<T> writer = factory.MakeWriter(nativeStream);
foreach (T rec in source)
{
writer.WriteRecordSync(rec);
}
writer.Close();
}
else
{
Action<IEnumerable<T>, Stream> serializerFunc = serializer.Compile();
serializerFunc(source, stream);
}
writer.Close();
}
}

6
LinqToDryad/DryadLinqBinaryReader.cs
View File

@ -84,11 +84,11 @@ namespace Microsoft.Research.DryadLinq
/// </summary>
~DryadLinqBinaryReader()
{
if (!this.m_isClosed)
// Only release native resoure here
if (this.m_curDataBlockInfo.ItemHandle != IntPtr.Zero)
{
this.m_isClosed = true;
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_nativeStream.Close();
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
}
}

12
LinqToDryad/DryadLinqBinaryWriter.cs
View File

@ -101,15 +101,11 @@ namespace Microsoft.Research.DryadLinq
/// </summary>
~DryadLinqBinaryWriter()
{
if (!this.m_isClosed)
// Only release native resoure here
if (this.m_curDataBlockInfo.ItemHandle != IntPtr.Zero)
{
this.m_isClosed = true;
this.Flush();
if (this.m_curBlockSize > 0)
{
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
}
this.m_nativeStream.Close();
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
}
}

16
LinqToDryad/DryadLinqCodeGen.cs
View File

@ -138,6 +138,7 @@ namespace Microsoft.Research.DryadLinq.Internal
s_BuiltinTypeToSerializerName.Add(typeof(string), "StringDryadLinqSerializer");
s_BuiltinTypeToSerializerName.Add(typeof(Guid), "GuidDryadLinqSerializer");
s_BuiltinTypeToSerializerName.Add(typeof(SqlDateTime), "SqlDateTimeDryadLinqSerializer");
s_BuiltinTypeToSerializerName.Add(typeof(LineRecord), "LineRecordDryadLinqSerializer");
// Initialize the mapping from type to its factory
s_TypeToFactory = new Dictionary<Type, object>(20);
@ -1901,8 +1902,7 @@ namespace Microsoft.Research.DryadLinq.Internal
CodeExpression arg1 = new CodeArgumentReferenceExpression("args");
CodeExpression arg2 = new CodeArgumentReferenceExpression(VertexParamName);
CodeExpression
denvInitExpr = new CodeObjectCreateExpression("VertexEnv", arg1, arg2);
CodeExpression denvInitExpr = new CodeObjectCreateExpression("VertexEnv", arg1, arg2);
return new CodeVariableDeclarationStatement("VertexEnv", VertexEnvName, denvInitExpr);
}
@ -1940,7 +1940,6 @@ namespace Microsoft.Research.DryadLinq.Internal
CodeMethodInvokeExpression debuggerCheckExpr = new CodeMethodInvokeExpression(
new CodeMethodReferenceExpression(new CodeTypeReferenceExpression(HelperClassName),
DebugHelperMethodName));
vertexMethod.Statements.Add(new CodeExpressionStatement(debuggerCheckExpr));
}
@ -1971,14 +1970,7 @@ namespace Microsoft.Research.DryadLinq.Internal
vertexMethod.Statements.Add(new CodeExpressionStatement(setParamsExpr));
}
// Push the parallel-code settings into DryadLinqVertex
bool multiThreading = this.m_context.EnableMultiThreadingInVertex;
vertexMethod.Statements.Add(SetVertexParamField("MultiThreading", multiThreading));
vertexMethod.Statements.Add(
new CodeAssignStatement(
new CodeFieldReferenceExpression(DLVTypeExpr, "s_multiThreading"),
new CodePrimitiveExpression(multiThreading)));
// Initialize vertex env:
vertexMethod.Statements.Add(MakeVertexEnvDecl(node));
Type[] outputTypes = node.OutputTypes;
@ -2122,7 +2114,7 @@ namespace Microsoft.Research.DryadLinq.Internal
pipeline[pipeline.Length - 1].OutputDataSetInfo.orderByInfo.IsOrdered);
CodeExpression applyExpr;
if (this.m_context.EnableMultiThreadingInVertex)
if (this.m_context.ThreadsPerWorker > 1)
{
applyExpr = new CodeMethodInvokeExpression(
DryadLinqCodeGen.DLVTypeExpr,

Some files were not shown because too many files have changed in this diff Show More