berkus
/
Dryad
mirror of https://github.com/MicrosoftResearch/Dryad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Release 0.2 edits

This commit is contained in:
misard 2014-09-30 16:15:56 -07:00
parent 5c260ffe0b
commit efc79b749b
147 changed files with 5814 additions and 3196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
ClusterInterface/ClusterInterface.csproj
View File

@ -1,6 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -9,10 +10,11 @@
<OutputType>Library</OutputType> <OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Dryad.ClusterInterface</RootNamespace> <RootNamespace>Microsoft.Research.Dryad.ClusterInterface</RootNamespace>
<AssemblyName>DryadHttpClusterInterface</AssemblyName> <AssemblyName>Microsoft.Research.Dryad.HttpClusterInterface</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<TargetFrameworkProfile /> <TargetFrameworkProfile />
<NuGetPackageImportStamp>41254407</NuGetPackageImportStamp>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
@ -33,99 +35,12 @@
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" /> <Reference Include="System" />
<Reference Include="System.Core" /> <Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" /> <Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" /> <Reference Include="System.Net" />
<Reference Include="System.Net.Http" /> <Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" /> <Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" /> <Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" /> <Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" /> <Reference Include="System.Data.DataSetExtensions" />
@ -134,29 +49,34 @@
<Reference Include="System.Xml" /> <Reference Include="System.Xml" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Compile Include="..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="Interfaces.cs" /> <Compile Include="Interfaces.cs" />
<Compile Include="HttpScheduler.cs" /> <Compile Include="HttpScheduler.cs" />
<Compile Include="Process.cs" /> <Compile Include="Process.cs" />
<Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="app.config" /> <None Include="app.config">
<None Include="packages.config" /> <SubType>Designer</SubType>
</None>
<None Include="packages.config">
<SubType>Designer</SubType>
</None>
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
</Target> </Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild"> <Target Name="BeforeBuild">

27
ClusterInterface/Properties/AssemblyInfo.cs
View File

@ -24,32 +24,7 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following // General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information // set of attributes. Change these attribute values to modify the information
// associated with an assembly. // associated with an assembly.
[assembly: AssemblyTitle("DryadHttpClusterInterface")] [assembly: AssemblyTitle("Microsoft.Research.Dryad.HttpClusterInterface")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyProduct("Dryad")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM // The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("9d262d5f-f6c5-4f9c-a1cc-7568cb399bf9")] [assembly: Guid("9d262d5f-f6c5-4f9c-a1cc-7568cb399bf9")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

24
ClusterInterface/app.config
View File

@ -4,24 +4,40 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" /> <bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" /> <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>
</configuration> </configuration>

20
ClusterInterface/packages.config
View File

@ -1,21 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Shared" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Utils" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages> </packages>

26
Dryad.sln
View File

@ -1,6 +1,8 @@
 
Microsoft Visual Studio Solution File, Format Version 12.00 Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2012 # Visual Studio 2013
VisualStudioVersion = 12.0.30723.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DryadVertex", "DryadVertex", "{99F5E7FE-ADD4-4E9D-BA90-5D3D3409BEB3}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DryadVertex", "DryadVertex", "{99F5E7FE-ADD4-4E9D-BA90-5D3D3409BEB3}"
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "VertexHost", "VertexHost", "{37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "VertexHost", "VertexHost", "{37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D}"
@ -27,7 +29,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dprocess", "DryadVertex\Ver
EndProject EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ManagedWrapperVertex", "DryadVertex\VertexHost\vertex\managedwrappervertex\ManagedWrapperVertex.vcxproj", "{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}" Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ManagedWrapperVertex", "DryadVertex\VertexHost\vertex\managedwrappervertex\ManagedWrapperVertex.vcxproj", "{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}"
EndProject EndProject
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHost", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}" Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHostBody", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}"
ProjectSection(ProjectDependencies) = postProject ProjectSection(ProjectDependencies) = postProject
{AA529122-F51C-48D7-A8C1-C0B24F570885} = {AA529122-F51C-48D7-A8C1-C0B24F570885} {AA529122-F51C-48D7-A8C1-C0B24F570885} = {AA529122-F51C-48D7-A8C1-C0B24F570885}
{482E0741-E244-4974-97D4-3A7167581E91} = {482E0741-E244-4974-97D4-3A7167581E91} {482E0741-E244-4974-97D4-3A7167581E91} = {482E0741-E244-4974-97D4-3A7167581E91}
@ -78,6 +80,19 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalScheduler", "LocalSche
{A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6} = {A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6} {A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6} = {A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6}
EndProjectSection EndProjectSection
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VertexHostMain", "DryadVertex\VertexHost\vertex\VertexHostMain\VertexHostMain.csproj", "{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "NuSpec", "NuSpec", "{35121ABA-5FD2-4E1D-9C05-033D78D824F3}"
ProjectSection(SolutionItems) = preProject
Microsoft.Research.Dryad.nuspec = Microsoft.Research.Dryad.nuspec
Microsoft.Research.Dryad.targets = Microsoft.Research.Dryad.targets
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Misc", "Misc", "{D361F025-2F7F-4C1A-8FBC-4AD826CC656D}"
ProjectSection(SolutionItems) = preProject
SharedAssemblyInfo.cs = SharedAssemblyInfo.cs
EndProjectSection
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64 Debug|x64 = Debug|x64
@ -144,6 +159,10 @@ Global
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Debug|x64.Build.0 = Debug|x64 {FA798DE2-A442-4D2B-B31A-FA9274308290}.Debug|x64.Build.0 = Debug|x64
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.ActiveCfg = Release|x64 {FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.ActiveCfg = Release|x64
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.Build.0 = Release|x64 {FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.Build.0 = Release|x64
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Debug|x64.ActiveCfg = Debug|x64
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Debug|x64.Build.0 = Debug|x64
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Release|x64.ActiveCfg = Release|x64
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Release|x64.Build.0 = Release|x64
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE
@ -156,11 +175,12 @@ Global
{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE} {016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{57663B94-E11B-431E-BE4B-E2C61112DEC5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE} {57663B94-E11B-431E-BE4B-E2C61112DEC5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{AA529122-F51C-48D7-A8C1-C0B24F570885} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE} {AA529122-F51C-48D7-A8C1-C0B24F570885} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{A0033286-9C5F-4113-BAA5-58A1274F95C5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{BDEDD3BB-C7E2-498F-A212-F99786C8E23C} = {DC277807-506B-4F7C-BECD-345079B91044} {BDEDD3BB-C7E2-498F-A212-F99786C8E23C} = {DC277807-506B-4F7C-BECD-345079B91044}
{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37} = {DC277807-506B-4F7C-BECD-345079B91044} {0CF3D1D5-9BBE-4175-979B-EC6138EF4F37} = {DC277807-506B-4F7C-BECD-345079B91044}
{AB9EA66C-5811-49A7-B002-24203AEB9083} = {DC277807-506B-4F7C-BECD-345079B91044} {AB9EA66C-5811-49A7-B002-24203AEB9083} = {DC277807-506B-4F7C-BECD-345079B91044}
{3EE0920C-0607-4569-9EC3-5C12BB6EF244} = {DC277807-506B-4F7C-BECD-345079B91044} {3EE0920C-0607-4569-9EC3-5C12BB6EF244} = {DC277807-506B-4F7C-BECD-345079B91044}
{89C5654B-02E4-478D-A7E6-50D79F638B4F} = {DC277807-506B-4F7C-BECD-345079B91044} {89C5654B-02E4-478D-A7E6-50D79F638B4F} = {DC277807-506B-4F7C-BECD-345079B91044}
{A0033286-9C5F-4113-BAA5-58A1274F95C5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352} = {DC277807-506B-4F7C-BECD-345079B91044}
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal

113
DryadLinqGraphManager/DryadLinqGraphManager.csproj
View File

@ -1,6 +1,9 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -10,7 +13,7 @@
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Dryad.GraphManager</RootNamespace> <RootNamespace>Microsoft.Research.Dryad.GraphManager</RootNamespace>
<AssemblyName>DryadLinqGraphManager</AssemblyName> <AssemblyName>Microsoft.Research.Dryad.GraphManager</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<FileUpgradeFlags> <FileUpgradeFlags>
@ -33,6 +36,7 @@
<ApplicationVersion>1.0.0.%2a</ApplicationVersion> <ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<UseApplicationTrust>false</UseApplicationTrust> <UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled> <BootstrapperEnabled>true</BootstrapperEnabled>
<NuGetPackageImportStamp>6aa65a02</NuGetPackageImportStamp>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
@ -53,80 +57,28 @@
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath> <HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath> <HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath> <HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath> <HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath> <HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference> </Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL"> <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath> <HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference> </Reference>
<Reference Include="System" /> <Reference Include="System" />
<Reference Include="System.Core"> <Reference Include="System.Core">
@ -138,18 +90,10 @@
<Reference Include="System.Data.Services.Client" /> <Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" /> <Reference Include="System.Net" />
<Reference Include="System.Net.Http" /> <Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" /> <Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath> <HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference> </Reference>
<Reference Include="System.Web" /> <Reference Include="System.Web" />
<Reference Include="System.Xml.Linq"> <Reference Include="System.Xml.Linq">
@ -162,6 +106,9 @@
<Reference Include="System.Xml" /> <Reference Include="System.Xml" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Compile Include="..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="DryadLinqApplication.cs" /> <Compile Include="DryadLinqApplication.cs" />
<Compile Include="GraphBuilder.cs" /> <Compile Include="GraphBuilder.cs" />
<Compile Include="LinqToDryadException.cs" /> <Compile Include="LinqToDryadException.cs" />
@ -203,19 +150,23 @@
</ProjectReference> </ProjectReference>
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
</Target> </Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild"> <Target Name="BeforeBuild">

16
DryadLinqGraphManager/GraphBuilder.cs
View File

@ -92,7 +92,7 @@ namespace Microsoft.Research.Dryad.GraphManager
err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath); err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath);
if (!SUCCEEDED(err)) if (!SUCCEEDED(err))
{ {
string msg = String.Format("Could not read DSC input file {0}", info.sources[0]); string msg = String.Format("Could not read Partitioned file input {0}", info.sources[0]);
throw new LinqToDryadException(msg, err); throw new LinqToDryadException(msg, err);
} }
@ -127,7 +127,9 @@ namespace Microsoft.Research.Dryad.GraphManager
DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset"); DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");
err = input.Open(app.GetUniverse(), info.sources[0]); Uri srcUri = new Uri(info.sources[0]);
err = input.Open(app.GetUniverse(), srcUri.GetLeftPart(UriPartial.Path), info.recordType);
if (!SUCCEEDED(err)) if (!SUCCEEDED(err))
{ {
string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError()); string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError());
@ -182,10 +184,16 @@ namespace Microsoft.Research.Dryad.GraphManager
if ( info.ioType == VertexInfo.IOType.PARTITIONEDFILE ) if ( info.ioType == VertexInfo.IOType.PARTITIONEDFILE )
{ {
DrPartitionOutputStream output = new DrPartitionOutputStream(); DrPartitionOutputStream output = new DrPartitionOutputStream();
int err = output.Open(new Uri(info.sources[0]).AbsolutePath, info.partitionUncPath); Uri sourceUri = new Uri(info.sources[0]);
string sourcePath = sourceUri.AbsolutePath;
if (!String.IsNullOrEmpty(sourceUri.Host))
{
sourcePath = @"\\" + sourceUri.Host + sourcePath;
}
int err = output.Open(sourcePath, info.partitionUncPath);
if (!SUCCEEDED(err)) if (!SUCCEEDED(err))
{ {
string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]); string msg = String.Format("Could not open output fileset {0}", sourcePath);
throw new LinqToDryadException(msg, err); throw new LinqToDryadException(msg, err);
} }

58
DryadLinqGraphManager/LinqToDryadJM.cs
View File

@ -73,12 +73,12 @@ namespace Microsoft.Research.Dryad.GraphManager
public static Uri AddDfsJobDirectoryFromArgs(string arg) public static Uri AddDfsJobDirectoryFromArgs(string arg)
{ {
string jobId = Microsoft.Research.Peloponnese.Storage.AzureUtils.ApplicationIdFromEnvironment(); string jobId = Microsoft.Research.Peloponnese.Yarn.Utils.ApplicationIdFromEnvironment();
string jobDirectoryBase = Microsoft.Research.Peloponnese.Storage.AzureUtils.CmdLineDecode(arg.Substring(arg.IndexOf('=') + 1)); string jobDirectoryBase = Microsoft.Research.Peloponnese.Utils.CmdLineDecode(arg.Substring(arg.IndexOf('=') + 1));
string jobDirectorySpecific = jobDirectoryBase.Replace("_JOBID_", jobId); string jobDirectorySpecific = jobDirectoryBase.Replace("_JOBID_", jobId);
UriBuilder logLocation = new UriBuilder(jobDirectorySpecific); UriBuilder logLocation = new UriBuilder(jobDirectorySpecific);
logLocation.Path = logLocation.Path + "calypso.log"; logLocation.Path = logLocation.Path.TrimEnd('/') + "/calypso.log";
DebugHelper.AddReporter(new DrCalypsoReporter(logLocation.Uri.AbsoluteUri)); DebugHelper.AddReporter(new DrCalypsoReporter(logLocation.Uri.AbsoluteUri));
@ -94,7 +94,7 @@ namespace Microsoft.Research.Dryad.GraphManager
{ {
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory)) using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
{ {
string dfsPath = dfsDirectory.AbsolutePath + Path.GetFileName(localPath); string dfsPath = dfsDirectory.AbsolutePath.TrimEnd('/') + "/" + Path.GetFileName(localPath);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath); DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
hdfs.UploadAll(localPath, dfsPath); hdfs.UploadAll(localPath, dfsPath);
} }
@ -102,11 +102,12 @@ namespace Microsoft.Research.Dryad.GraphManager
else if (dfsDirectory.Scheme == "azureblob") else if (dfsDirectory.Scheme == "azureblob")
{ {
string account, key, container, blob; string account, key, container, blob;
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob); Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container); var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
string dfsPath = blob + Path.GetFileName(localPath); string dfsPath = blob.TrimEnd('/') + "/" + Path.GetFileName(localPath);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath); Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
azure.PutDfsFile(localPath, dfsPath); DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsUri.AbsoluteUri);
azure.PutDfsFile(dfsUri, localPath);
} }
else if (dfsDirectory.Scheme == "file") else if (dfsDirectory.Scheme == "file")
{ {
@ -138,10 +139,11 @@ namespace Microsoft.Research.Dryad.GraphManager
else if (dfsDirectory.Scheme == "azureblob") else if (dfsDirectory.Scheme == "azureblob")
{ {
string account, key, container, blob; string account, key, container, blob;
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob); Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container); var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
string dfsPath = blob + dfsName; string dfsPath = blob + dfsName;
azure.PutDfsFile(payloadBytes, dfsPath); Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
azure.PutDfsFile(dfsUri, payloadBytes);
} }
else if (dfsDirectory.Scheme == "file") else if (dfsDirectory.Scheme == "file")
{ {
@ -207,7 +209,39 @@ namespace Microsoft.Research.Dryad.GraphManager
logDir = logDir.Split(',').First().Trim(); logDir = logDir.Split(',').First().Trim();
DrLogging.Initialize(Path.Combine(logDir, "graphmanager"), false); DrLogging.Initialize(Path.Combine(logDir, "graphmanager"), false);
} }
string logLevel = Environment.GetEnvironmentVariable("DRYAD_LOGGING_LEVEL");
if (logLevel == null)
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Debug);
}
else
{
if (logLevel == "OFF")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Off);
}
else if (logLevel == "CRITICAL")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Assert);
}
else if (logLevel == "ERROR")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Error);
}
else if (logLevel == "WARN")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Warning);
}
else if (logLevel == "INFO")
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Info); DrLogging.SetLoggingLevel(DrLogTypeManaged.Info);
}
else
{
DrLogging.SetLoggingLevel(DrLogTypeManaged.Debug);
}
}
// Report start time to Artemis - must come after // Report start time to Artemis - must come after
// DrLogging is initialized so stdout is redirected // DrLogging is initialized so stdout is redirected

7
DryadLinqGraphManager/Program.cs
View File

@ -28,6 +28,13 @@ using Microsoft.Research.Dryad;
namespace Microsoft.Research.Dryad.GraphManager namespace Microsoft.Research.Dryad.GraphManager
{ {
internal class ConfigDependency : Microsoft.Research.Peloponnese.Shared.AssemblyDependencyAttribute
{
public ConfigDependency() : base("Microsoft.Research.Dryad.GraphManager.exe.config", false)
{
}
}
class Program class Program
{ {
static Uri dfsDirectory; static Uri dfsDirectory;

28
DryadLinqGraphManager/Properties/AssemblyInfo.cs
View File

@ -22,34 +22,12 @@ using System.Reflection;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
[assembly: Microsoft.Research.Dryad.GraphManager.ConfigDependency()]
// General Information about an assembly is controlled through the following // General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information // set of attributes. Change these attribute values to modify the information
// associated with an assembly. // associated with an assembly.
[assembly: AssemblyTitle("DryadLinqGraphManager")] [assembly: AssemblyTitle("Microsoft.Research.Dryad.GraphManager")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyProduct("Dryad")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM // The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("55c7f4b4-02ab-4309-b276-3c9ab5a927e0")] [assembly: Guid("55c7f4b4-02ab-4309-b276-3c9ab5a927e0")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

6
DryadLinqGraphManager/QueryParser.cs
View File

@ -237,6 +237,12 @@ namespace Microsoft.Research.Dryad.GraphManager
vertex.info = new VertexInfo(); vertex.info = new VertexInfo();
vertex.info.ioType = GetIoType(ioType); vertex.info.ioType = GetIoType(ioType);
XmlNode recordType = storageSet.SelectSingleNode("RecordType");
if (recordType != null)
{
vertex.info.recordType = recordType.InnerXml;
}
XmlNodeList storageUris = storageSet.SelectNodes("SourceURI"); XmlNodeList storageUris = storageSet.SelectNodes("SourceURI");
vertex.info.sources = new string[storageUris.Count]; vertex.info.sources = new string[storageUris.Count];
for (int indexStorageUri=0; indexStorageUri<storageUris.Count; indexStorageUri++) for (int indexStorageUri=0; indexStorageUri<storageUris.Count; indexStorageUri++)

20
DryadLinqGraphManager/app.config
View File

@ -7,19 +7,19 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" /> <bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
@ -27,11 +27,19 @@
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.0.7" newVersion="1.1.0.7" /> <bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.1.0.7" newVersion="1.1.0.7" /> <bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly> </dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>

27
DryadLinqGraphManager/packages.config
View File

@ -1,21 +1,14 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" /> <package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" /> <package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" /> <package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Shared" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Utils" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" /> <package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" /> <package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" /> <package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages> </packages>

21
DryadLinqTests/ApplyAndForkTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;

21
DryadLinqTests/BasicAPITests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;

44
DryadLinqTests/DryadLinqTests.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -12,9 +12,26 @@
<AssemblyName>DryadLinqTests</AssemblyName> <AssemblyName>DryadLinqTests</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
<RestorePackages>true</RestorePackages>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>x64</PlatformTarget> <PlatformTarget>AnyCPU</PlatformTarget>
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType> <DebugType>full</DebugType>
<Optimize>false</Optimize> <Optimize>false</Optimize>
@ -187,6 +204,29 @@
<None Include="App.config" /> <None Include="App.config" />
<None Include="packages.config" /> <None Include="packages.config" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\LinqToDryad\LinqToDryad.csproj">
<Project>{d33c34cc-6db2-417c-88b7-299830711774}</Project>
<Name>LinqToDryad</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<BootstrapperPackage Include=".NETFramework,Version=v4.5">
<Visible>False</Visible>
<ProductName>Microsoft .NET Framework 4.5 %28x86 and x64%29</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>false</Install>
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" /> <Import Project="packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''"> <Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">

21
DryadLinqTests/GroupByReduceTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;

21
DryadLinqTests/MiscBugFixTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;

19
DryadLinqTests/Properties/AssemblyInfo.cs
View File

@ -1,3 +1,22 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System.Reflection; using System.Reflection;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;

21
DryadLinqTests/RangePartitionAPICoverageTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.DryadLinq.Internal; using Microsoft.Research.DryadLinq.Internal;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;
using System; using System;

21
DryadLinqTests/SerializationTestTypes.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;

21
DryadLinqTests/SerializationTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;

21
DryadLinqTests/SimpleTests.cs
View File

@ -1,4 +1,23 @@
using System; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;

21
DryadLinqTests/TestLog.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;

21
DryadLinqTests/TypesInQueryTests.cs
View File

@ -1,4 +1,23 @@
using Microsoft.Research.DryadLinq; /*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;

19
DryadLinqTests/Utils.cs
View File

@ -1,3 +1,22 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using Microsoft.Research.DryadLinq; using Microsoft.Research.DryadLinq;
using Microsoft.Research.DryadLinq.Internal; using Microsoft.Research.DryadLinq.Internal;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Storage;

19
DryadLinqTests/WordCount.cs
View File

@ -1,3 +1,22 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
// comment the following line to run on Azure // comment the following line to run on Azure
#define local #define local

37
DryadVertex/VertexHost/system/channel/channel.vcxproj
View File

@ -1,6 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -24,13 +25,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -54,6 +55,13 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<NuGetPackageImportStamp>028fbeb3</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl> <Midl>
@ -61,7 +69,7 @@
</Midl> </Midl>
<ClCompile> <ClCompile>
<Optimization>Disabled</Optimization> <Optimization>Disabled</Optimization>
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\..\peloponnese\HadoopBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<PreprocessorDefinitions>WIN32;_DEBUG;_LIB;WIN32_LEAN_AND_MEAN;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;WIN32_LEAN_AND_MEAN;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<MinimalRebuild>false</MinimalRebuild> <MinimalRebuild>false</MinimalRebuild>
<BasicRuntimeChecks>Default</BasicRuntimeChecks> <BasicRuntimeChecks>Default</BasicRuntimeChecks>
@ -72,10 +80,6 @@
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat> <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories> <AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile> </ClCompile>
<Lib>
<AdditionalDependencies>Microsoft.Research.Peloponnese.HadoopBridge.lib</AdditionalDependencies>
<AdditionalLibraryDirectories>..\..\..\..\..\peloponnese\bin\$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Midl> <Midl>
@ -88,13 +92,9 @@
</PrecompiledHeader> </PrecompiledHeader>
<WarningLevel>Level3</WarningLevel> <WarningLevel>Level3</WarningLevel>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat> <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\..\peloponnese\HadoopBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories> <AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile> </ClCompile>
<Lib>
<AdditionalDependencies>Microsoft.Research.Peloponnese.HadoopBridge.lib</AdditionalDependencies>
<AdditionalLibraryDirectories>..\..\..\..\..\peloponnese\bin\$(Configuration)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemGroup> <ItemGroup>
<ClInclude Include="include\channelbuffer.h" /> <ClInclude Include="include\channelbuffer.h" />
@ -149,13 +149,16 @@
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
</ImportGroup> </ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
</Target> </Target>
</Project> </Project>

3
DryadVertex/VertexHost/system/channel/packages.config
View File

@ -1,4 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="Native" /> <package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.0-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.0-beta" targetFramework="Native" />
</packages> </packages>

48
DryadVertex/VertexHost/system/channel/src/channelbufferhdfs.cpp
View File

@ -929,6 +929,37 @@ RChannelBufferHdfsWriter::RChannelBufferHdfsWriter(const char* uri)
Hdfs::InstanceAccessor ia(bridge); Hdfs::InstanceAccessor ia(bridge);
ia.Discard(); ia.Discard();
} }
char envUserName[128];
DWORD ret = ::GetEnvironmentVariableA("USER", envUserName, sizeof(envUserName));
if (ret > 0 && ret < sizeof(envUserName)-1)
{
m_user.Set(envUserName);
DrLogI("Hdfs writer set user to %s from environment", m_user.GetString());
}
else
{
if (ret == 0 && GetLastError() != ERROR_ENVVAR_NOT_FOUND)
{
DrLogW("Can't get USER environment variable: %s", DRERRORSTRING(DrGetLastError()));
}
if (ret >= sizeof(envUserName)-1)
{
DrLogW("USER environment is too long: %d", ret);
}
DWORD bufSize = sizeof(envUserName);
BOOL bRet = ::GetUserNameA(envUserName, &bufSize);
if (bRet == 0)
{
DrLogW("Can't get username: %s", DRERRORSTRING(DrGetLastError()));
}
else
{
m_user.Set(envUserName);
DrLogI("Hdfs writer set user to %s from username", m_user.GetString());
}
}
} }
} }
} }
@ -1021,7 +1052,7 @@ bool RChannelBufferHdfsWriter::Open(Hdfs::Instance** pInstance,
Hdfs::InstanceAccessor ia(instance); Hdfs::InstanceAccessor ia(instance);
Hdfs::Writer* writer; Hdfs::Writer* writer;
bool openedWriter = ia.OpenWriter(filePath.GetString(), false, &writer); bool openedWriter = ia.OpenCreate(filePath.GetString(), 1024 * 1024, -1, &writer);
if (!openedWriter) if (!openedWriter)
{ {
char* errorMsg = ia.GetExceptionMessage(); char* errorMsg = ia.GetExceptionMessage();
@ -1036,6 +1067,21 @@ bool RChannelBufferHdfsWriter::Open(Hdfs::Instance** pInstance,
ia.Discard(); ia.Discard();
return false; return false;
} }
bool setPermissions = ia.SetOwnerAndPermission(filePath.GetString(), m_user.GetString(), NULL, 0644);
if (!setPermissions)
{
char* errorMsg = ia.GetExceptionMessage();
DrStr64 description;
description.SetF("Can't set HDFS file permissions '%s': %s",
m_uri.GetString(), errorMsg);
HadoopNative::DisposeString(errorMsg);
m_completionItem.Attach(MakeErrorItem(DryadError_ChannelOpenError,
description.GetString()));
ia.Discard();
return false;
}
DrLogI("Opened Hdfs writer for %s", m_uri.GetString()); DrLogI("Opened Hdfs writer for %s", m_uri.GetString());

1
DryadVertex/VertexHost/system/channel/src/channelbufferhdfs.h
View File

@ -149,6 +149,7 @@ private:
bool Open(Hdfs::Instance** pInstance, Hdfs::Writer** pWriter); bool Open(Hdfs::Instance** pInstance, Hdfs::Writer** pWriter);
bool AddToQueue(WriteEntry* entry); bool AddToQueue(WriteEntry* entry);
DrStr64 m_user;
DrStr64 m_uri; DrStr64 m_uri;
DryadBList<WriteEntry> m_queue; DryadBList<WriteEntry> m_queue;
UInt32 m_queueLength; UInt32 m_queueLength;

2
DryadVertex/VertexHost/system/channel/src/channelbuffernativewriter.cpp
View File

@ -2205,7 +2205,7 @@ void RChannelBufferWriterNativeFile::EagerCloseFile()
} }
else else
{ {
DrLogI( "Set final file pointer"); DrLogI( "Set final file pointer. Length %I64u", finalLength.QuadPart);
} }
ok = SetEndOfFile(m_bufferedFileHandle); ok = SetEndOfFile(m_bufferedFileHandle);

8
DryadVertex/VertexHost/system/classlib/classlib.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -21,13 +21,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>false</CLRSupport> <CLRSupport>false</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>false</CLRSupport> <CLRSupport>false</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">

8
DryadVertex/VertexHost/system/common/common.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -19,14 +19,14 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
<CharacterSet>MultiByte</CharacterSet> <CharacterSet>MultiByte</CharacterSet>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>MultiByte</CharacterSet> <CharacterSet>MultiByte</CharacterSet>
<CLRSupport>false</CLRSupport> <CLRSupport>false</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">

37
DryadVertex/VertexHost/system/dprocess/dprocess.vcxproj
View File

@ -1,6 +1,8 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -22,7 +24,7 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
</PropertyGroup> </PropertyGroup>
@ -30,7 +32,7 @@
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -54,6 +56,13 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<NuGetPackageImportStamp>1ed8669f</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<TargetName>$(ProjectName)</TargetName>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl> <Midl>
@ -72,10 +81,6 @@
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat> <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories> <AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile> </ClCompile>
<Lib>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<Midl> <Midl>
@ -91,10 +96,6 @@
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;..\channel\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;..\channel\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories> <AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
</ClCompile> </ClCompile>
<Lib>
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
</Lib>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemGroup> <ItemGroup>
<ClInclude Include="include\dryadvertex.h" /> <ClInclude Include="include\dryadvertex.h" />
@ -123,13 +124,19 @@
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
</ImportGroup> </ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
</Target> </Target>
</Project> </Project>

4
DryadVertex/VertexHost/system/dprocess/packages.config
View File

@ -1,4 +1,6 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" /> <package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.0-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.0-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.0-beta" targetFramework="Native" />
</packages> </packages>

16
DryadVertex/VertexHost/system/managedchannel/AzureReader.cs
View File

@ -27,7 +27,7 @@ using System.Runtime.InteropServices;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Azure;
namespace Microsoft.Research.Dryad.Channel namespace Microsoft.Research.Dryad.Channel
{ {
@ -74,7 +74,7 @@ namespace Microsoft.Research.Dryad.Channel
{ {
Log.LogInformation("Opening read for " + source.AbsoluteUri); Log.LogInformation("Opening read for " + source.AbsoluteUri);
string account, key, container, blobName; string account, key, container, blobName;
AzureUtils.FromAzureUri(source, out account, out key, out container, out blobName); Utils.FromAzureUri(source, out account, out key, out container, out blobName);
client = new AzureDfsClient(account, key, container, false, new PeloponneseLogger(Log.Logger)); client = new AzureDfsClient(account, key, container, false, new PeloponneseLogger(Log.Logger));
client.SetParallelThreadCount(4); client.SetParallelThreadCount(4);
@ -94,7 +94,7 @@ namespace Microsoft.Research.Dryad.Channel
private async Task OpenBlob() private async Task OpenBlob()
{ {
string account, key, container, blobName; string account, key, container, blobName;
AzureUtils.FromAzureUri(source, out account, out key, out container, out blobName); Utils.FromAzureUri(source, out account, out key, out container, out blobName);
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(source.Query); NameValueCollection query = System.Web.HttpUtility.ParseQueryString(source.Query);
@ -105,9 +105,11 @@ namespace Microsoft.Research.Dryad.Channel
++blobIndex; ++blobIndex;
} }
Log.LogInformation("Opening read for blob " + blobName); Uri blobUri = Utils.ToAzureUri(account, container, blobName, null, key);
readStream = (await client.GetDfsFileStreamAsync(blobName, ExecutionTimeout, new PeloponneseLogger(Log.Logger))).Stream; Log.LogInformation("Opening read for blob " + blobUri.AbsoluteUri);
readStream = (await client.GetDfsStreamReaderAsync(blobUri, ExecutionTimeout, new PeloponneseLogger(Log.Logger)));
long offset = -1; long offset = -1;
if (query["offset"] != null) if (query["offset"] != null)
@ -215,7 +217,7 @@ namespace Microsoft.Research.Dryad.Channel
int toRead = (int)Math.Min((long)managedBuffer.Length, bytesToRead); int toRead = (int)Math.Min((long)managedBuffer.Length, bytesToRead);
Log.LogInformation("About to read buffer length " + toRead); Log.LogInformation("About to read buffer length " + toRead);
readData.nRead = await AzureUtils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () => readData.nRead = await Utils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
{ {
return await readStream.ReadAsync(managedBuffer, 0, toRead); return await readStream.ReadAsync(managedBuffer, 0, toRead);
}); });
@ -301,7 +303,7 @@ namespace Microsoft.Research.Dryad.Channel
{ {
int toRead = (int)Math.Min(buffer.LongLength, endOffset - currentOffset); int toRead = (int)Math.Min(buffer.LongLength, endOffset - currentOffset);
int nRead = await AzureUtils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () => int nRead = await Utils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
{ {
return await readStream.ReadAsync(buffer, 0, toRead); return await readStream.ReadAsync(buffer, 0, toRead);
}); });

2
DryadVertex/VertexHost/system/managedchannel/AzureWriter.cs
View File

@ -25,7 +25,7 @@ using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Azure;
namespace Microsoft.Research.Dryad.Channel namespace Microsoft.Research.Dryad.Channel
{ {

26
DryadVertex/VertexHost/system/managedchannel/Properties/AssemblyInfo.cs
View File

@ -24,32 +24,8 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following // General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information // set of attributes. Change these attribute values to modify the information
// associated with an assembly. // associated with an assembly.
[assembly: AssemblyTitle("managedchannel")] [assembly: AssemblyTitle("Microsoft.Research.Dryad.ManagedChannel")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("dryad")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM // The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("f806a093-c5ae-4752-82d3-26bcc2341b29")] [assembly: Guid("f806a093-c5ae-4752-82d3-26bcc2341b29")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.5.0")]
[assembly: AssemblyFileVersion("0.1.5.0")]

28
DryadVertex/VertexHost/system/managedchannel/app.config
View File

@ -4,24 +4,44 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" /> <bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" /> <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.HDInsight.Threading" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.0" newVersion="1.2.1.0" />
</dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>
</configuration> </configuration>

109
DryadVertex/VertexHost/system/managedchannel/managedchannel.csproj
View File

@ -1,6 +1,8 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -9,11 +11,12 @@
<OutputType>Library</OutputType> <OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Dryad.Channel</RootNamespace> <RootNamespace>Microsoft.Research.Dryad.Channel</RootNamespace>
<AssemblyName>DryadManagedChannel</AssemblyName> <AssemblyName>Microsoft.Research.Dryad.ManagedChannel</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir> <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir>
<RestorePackages>true</RestorePackages> <RestorePackages>true</RestorePackages>
<NuGetPackageImportStamp>5738d8d0</NuGetPackageImportStamp>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
@ -35,98 +38,38 @@
<AllowUnsafeBlocks>false</AllowUnsafeBlocks> <AllowUnsafeBlocks>false</AllowUnsafeBlocks>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath> <HintPath>..\..\..\..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath> <HintPath>..\..\..\..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath> <HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath> <HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath> <HintPath>..\..\..\..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference> </Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL"> <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath> <HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference> </Reference>
<Reference Include="System" /> <Reference Include="System" />
<Reference Include="System.Core" /> <Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" /> <Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" /> <Reference Include="System.Net" />
<Reference Include="System.Net.Http" /> <Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" /> <Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath> <HintPath>..\..\..\..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference> </Reference>
<Reference Include="System.Web" /> <Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" /> <Reference Include="System.Xml.Linq" />
@ -136,6 +79,9 @@
<Reference Include="System.Xml" /> <Reference Include="System.Xml" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Compile Include="..\..\..\..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="AzureReader.cs" /> <Compile Include="AzureReader.cs" />
<Compile Include="AzureWriter.cs" /> <Compile Include="AzureWriter.cs" />
<Compile Include="FileWriter.cs" /> <Compile Include="FileWriter.cs" />
@ -151,19 +97,20 @@
<None Include="packages.config" /> <None Include="packages.config" />
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
</Target> </Target>
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild"> <Target Name="BeforeBuild">

26
DryadVertex/VertexHost/system/managedchannel/packages.config
View File

@ -1,21 +1,13 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" /> <package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" /> <package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" /> <package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Shared" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Utils" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.5-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" /> <package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" /> <package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" /> <package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
</packages> </packages>

43
DryadVertex/VertexHost/vertex/VertexHostMain/Program.cs Normal file
View File

@ -0,0 +1,43 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace VertexHostMain
{
internal class ConfigDependency : Microsoft.Research.Peloponnese.Shared.AssemblyDependencyAttribute
{
public
ConfigDependency()
: base("Microsoft.Research.Dryad.VertexHost.exe.config", false)
{
}
}
class Program
{
static int Main(string[] args)
{
return VertexHost.Main(args);
}
}
}

32
DryadVertex/VertexHost/vertex/VertexHostMain/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,32 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
[assembly: VertexHostMain.ConfigDependency()]
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Microsoft.Research.Dryad.VertexHost")]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("5534b858-8472-4dc3-8f24-5b59f8805156")]

79
DryadVertex/VertexHost/vertex/VertexHostMain/VertexHostMain.csproj Normal file
View File

@ -0,0 +1,79 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>VertexHostMain</RootNamespace>
<AssemblyName>Microsoft.Research.Dryad.VertexHost</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
<NuGetPackageImportStamp>21807ffd</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>..\..\..\..\bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
<OutputPath>..\..\..\..\bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="..\..\..\..\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\vertexHost\VertexHost.vcxproj">
<Project>{0cf3d1d5-9bbe-4175-979b-ec6138ef4f37}</Project>
<Name>VertexHostBody</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="app.config" />
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
</Target>
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

48
DryadVertex/VertexHost/vertex/VertexHostMain/app.config Normal file
View File

@ -0,0 +1,48 @@
<?xml version="1.0"?>
<configuration>
<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/></startup>
<runtime>
<gcServer enabled="true"></gcServer>
<gcConcurrent enabled="false"></gcConcurrent>
<generatePublisherEvidence enabled="false"/>
<gcAllowVeryLargeObjects enabled="true" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

4
DryadVertex/VertexHost/vertex/VertexHostMain/packages.config Normal file
View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.0-beta" targetFramework="net45" />
</packages>

8
DryadVertex/VertexHost/vertex/WrapperNativeInfo/WrapperNativeInfo.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -21,13 +21,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">

2
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/DryadLINQNativeChannels.def
View File

@ -1,4 +1,4 @@
LIBRARY "DryadLINQNativeChannels.dll" LIBRARY "Microsoft.Research.Dryad.DryadLinq.NativeWrapper.dll"
EXPORTS EXPORTS
GetTotalLength GetTotalLength

13
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/WrapperNativeInfoDll.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -19,14 +19,14 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -52,8 +52,8 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">DryadLINQNativeChannels</TargetName> <TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad.DryadLinq.NativeWrapper</TargetName>
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">DryadLINQNativeChannels</TargetName> <TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad.DryadLinq.NativeWrapper</TargetName>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl> <Midl>
@ -111,6 +111,7 @@
<ClInclude Include="stdafx.h" /> <ClInclude Include="stdafx.h" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ClCompile Include="version.cpp" />
<ClCompile Include="wrappernativeinfostubs.cpp" /> <ClCompile Include="wrappernativeinfostubs.cpp" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

3
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/WrapperNativeInfoDll.vcxproj.filters
View File

@ -23,6 +23,9 @@
<ClCompile Include="wrappernativeinfostubs.cpp"> <ClCompile Include="wrappernativeinfostubs.cpp">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="version.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="DryadLINQNativeChannels.def"> <None Include="DryadLINQNativeChannels.def">

40
DryadVertex/VertexHost/vertex/WrapperNativeInfoDll/version.cpp Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
#ifdef _MANAGED
#pragma managed
using namespace System::Reflection;
using namespace System::Runtime::InteropServices;
[assembly: AssemblyTitle("Microsoft.Research.Dryad.DryadLinq.NativeWrapper")];
[assembly: AssemblyDescription("")];
[assembly: AssemblyConfiguration("")];
[assembly: AssemblyCompany("Microsoft Corporation")];
[assembly: AssemblyProduct("Microsoft.Research.Dryad.DryadLinq.NativeWrapper")];
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
[assembly: AssemblyTrademark("")];
[assembly: AssemblyCulture("")];
[assembly: ComVisible(false)];
[assembly: AssemblyVersion("1.0.0.0")];
[assembly: AssemblyFileVersion("1.0.0.0")];
#endif

31
DryadVertex/VertexHost/vertex/managedwrappervertex/ManagedWrapperVertex.cpp
View File

@ -197,6 +197,14 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
logName = System::IO::Path::Combine(logDirectory, logName); logName = System::IO::Path::Combine(logDirectory, logName);
} }
int threadsPerWorker = 1;
System::String^ threadsPerWorkerStr = System::Environment::GetEnvironmentVariable("DRYAD_THREADS_PER_WORKER");
if (threadsPerWorkerStr != nullptr)
{
threadsPerWorker = Int32::Parse(threadsPerWorkerStr);
}
DrLogI("ManagedWrapperVertex: threadsPerWorker %u", threadsPerWorker);
DrLogI("ManagedWrapperVertex: %p %u %u", nativeInfo, numberOfInputChannels, numberOfOutputChannels); DrLogI("ManagedWrapperVertex: %p %u %u", nativeInfo, numberOfInputChannels, numberOfOutputChannels);
DrLogI("ManagedWrapperVertex: Calling %s.%s", GetArgument(2), GetArgument(3)); DrLogI("ManagedWrapperVertex: Calling %s.%s", GetArgument(2), GetArgument(3));
DrLogging::FlushLog(); DrLogging::FlushLog();
@ -217,7 +225,7 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
// //
System::String^ classFullName = gcnew System::String(GetArgument(2)); System::String^ classFullName = gcnew System::String(GetArgument(2));
System::String^ assemblyName = classFullName->Substring(0, classFullName->LastIndexOf('.')); System::String^ assemblyName = classFullName->Substring(0, classFullName->LastIndexOf('.'));
System::String ^bridgeAssemblyPartialName = gcnew System::String(assemblyName); System::String ^bridgeAssemblyName = gcnew System::String(assemblyName);
System::String ^bridgeClassName = gcnew System::String(assemblyName + ".Internal.VertexEnv"); System::String ^bridgeClassName = gcnew System::String(assemblyName + ".Internal.VertexEnv");
System::String ^bridgeMethodName = gcnew System::String(L"VertexBridge"); System::String ^bridgeMethodName = gcnew System::String(L"VertexBridge");
@ -240,7 +248,7 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
// "<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>" // "<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
// //
System::Text::StringBuilder ^vertexBridgeArg = gcnew System::Text::StringBuilder(); System::Text::StringBuilder ^vertexBridgeArg = gcnew System::Text::StringBuilder();
vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"c:\\HpcTemp\\user\\jobID\\Microsoft.Hpc.Linq0.dll"; vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"\\HpcTemp\\user\\jobID\\Microsoft.Research.DryadLinq0.dll";
vertexBridgeArg->Append(","); vertexBridgeArg->Append(",");
vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Research.DryadLinq.DryadLinq__Vertex"; vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Research.DryadLinq.DryadLinq__Vertex";
vertexBridgeArg->Append(","); vertexBridgeArg->Append(",");
@ -248,7 +256,6 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
vertexBridgeArg->Append(","); vertexBridgeArg->Append(",");
vertexBridgeArg->Append(vertexMethodArgs->ToString()); vertexBridgeArg->Append(vertexMethodArgs->ToString());
DrLogI("ManagedWrapperVertex: Calling into Vertex Bridge to invoke Vertex Entry: %s", GetArgument(3)); DrLogI("ManagedWrapperVertex: Calling into Vertex Bridge to invoke Vertex Entry: %s", GetArgument(3));
DrLogging::FlushLog(); DrLogging::FlushLog();
@ -259,10 +266,20 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
// //
try try
{ {
System::Console::WriteLine("Assembly name " + bridgeAssemblyPartialName); System::Reflection::Assembly ^vertexBridgeAsm;
System::Reflection::Assembly ^vertexBridgeAsm = System::Reflection::Assembly::LoadWithPartialName(bridgeAssemblyPartialName); try
System::Type ^vertexBridgeType = vertexBridgeAsm->GetType(gcnew System::String(bridgeClassName)); {
System::Reflection::MethodInfo ^vertexBridgeMethod = vertexBridgeType->GetMethod(gcnew System::String(bridgeMethodName), vertexBridgeAsm = System::Reflection::Assembly::Load(bridgeAssemblyName);
}
catch (System::Exception ^ie)
{
DrLogI("ManagedWrapperVertex: Failed to load assembly %s: %s", bridgeAssemblyName, ie->ToString());
System::String^ asmLoc = System::IO::Path::Combine("..", bridgeAssemblyName + ".dll");
vertexBridgeAsm = System::Reflection::Assembly::LoadFrom(asmLoc);
}
System::Type ^vertexBridgeType = vertexBridgeAsm->GetType(bridgeClassName);
System::Reflection::MethodInfo ^vertexBridgeMethod
= vertexBridgeType->GetMethod(bridgeMethodName,
static_cast<System::Reflection::BindingFlags>(System::Reflection::BindingFlags::NonPublic | static_cast<System::Reflection::BindingFlags>(System::Reflection::BindingFlags::NonPublic |
System::Reflection::BindingFlags::Static)); System::Reflection::BindingFlags::Static));

8
DryadVertex/VertexHost/vertex/managedwrappervertex/ManagedWrapperVertex.vcxproj
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -21,13 +21,13 @@
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType> <ConfigurationType>StaticLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">

27
DryadVertex/VertexHost/vertex/vertexHost/VertexHost.vcxproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -18,19 +18,20 @@
<Keyword>Win32Proj</Keyword> <Keyword>Win32Proj</Keyword>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir> <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir>
<RestorePackages>true</RestorePackages> <RestorePackages>true</RestorePackages>
<ProjectName>VertexHostBody</ProjectName>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>true</CLRSupport> <CLRSupport>true</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -56,6 +57,13 @@
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<NuGetPackageImportStamp>16d8f87f</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<TargetName>Microsoft.Research.Dryad.$(ProjectName)</TargetName>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<TargetName>Microsoft.Research.Dryad.$(ProjectName)</TargetName>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl> <Midl>
@ -112,6 +120,7 @@
</PostBuildEvent> </PostBuildEvent>
</ItemDefinitionGroup> </ItemDefinitionGroup>
<ItemGroup> <ItemGroup>
<ClCompile Include="version.cpp" />
<ClCompile Include="vertexHost.cpp" /> <ClCompile Include="vertexHost.cpp" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
@ -124,13 +133,13 @@
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
</ImportGroup> </ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
</Target> </Target>
</Project> </Project>

1
DryadVertex/VertexHost/vertex/vertexHost/VertexHost.vcxproj.filters
View File

@ -2,6 +2,7 @@
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup> <ItemGroup>
<ClCompile Include="vertexHost.cpp" /> <ClCompile Include="vertexHost.cpp" />
<ClCompile Include="version.cpp" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="app.config" /> <None Include="app.config" />

32
DryadVertex/VertexHost/vertex/vertexHost/app.config
View File

@ -1,26 +1,42 @@
<?xml version="1.0" encoding="utf-8" ?> <?xml version="1.0" encoding="utf-8" ?>
<configuration> <configuration>
<runtime> <runtime>
<gcServer enabled="true"></gcServer>
<gcConcurrent enabled="false"></gcConcurrent>
<generatePublisherEvidence enabled="false"/>
<gcAllowVeryLargeObjects enabled="true" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" /> <bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
</dependentAssembly> </dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>

2
DryadVertex/VertexHost/vertex/vertexHost/packages.config
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" /> <package id="Microsoft.Research.Peloponnese.Shared" version="0.8.0-beta" targetFramework="Native" />
</packages> </packages>

40
DryadVertex/VertexHost/vertex/vertexHost/version.cpp Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
#ifdef _MANAGED
#pragma managed
using namespace System::Reflection;
using namespace System::Runtime::InteropServices;
[assembly: AssemblyTitle("Microsoft.Research.Dryad.VertexHost")];
[assembly: AssemblyDescription("")];
[assembly: AssemblyConfiguration("")];
[assembly: AssemblyCompany("Microsoft Corporation")];
[assembly: AssemblyProduct("Microsoft.Research.Dryad.VertexHost")];
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
[assembly: AssemblyTrademark("")];
[assembly: AssemblyCulture("")];
[assembly: ComVisible(false)];
[assembly: AssemblyVersion("1.0.0.0")];
[assembly: AssemblyFileVersion("1.0.0.0")];
#endif

47
DryadVertex/VertexHost/vertex/vertexHost/vertexHost.cpp
View File

@ -174,27 +174,27 @@ void SetLoggingLevel()
GetLoggingFileName(logFileName); GetLoggingFileName(logFileName);
DrLogging::Initialize(logFileName); DrLogging::Initialize(logFileName);
WCHAR traceLevel [MAX_PATH]; WCHAR loggingLevel [MAX_PATH];
HRESULT hr = DrGetEnvironmentVariable(L"DRYAD_TRACE_LEVEL", traceLevel); HRESULT hr = DrGetEnvironmentVariable(L"DRYAD_LOGGING_LEVEL", loggingLevel);
if(hr == DrError_OK) if(hr == DrError_OK)
{ {
if(wcscmp(traceLevel, L"OFF") == 0) if(wcscmp(loggingLevel, L"OFF") == 0)
{ {
DrLogging::SetLoggingLevel(LogLevel_Off); DrLogging::SetLoggingLevel(LogLevel_Off);
} }
else if(wcscmp(traceLevel, L"CRITICAL") == 0) else if(wcscmp(loggingLevel, L"CRITICAL") == 0)
{ {
DrLogging::SetLoggingLevel(LogLevel_Assert); DrLogging::SetLoggingLevel(LogLevel_Assert);
} }
else if(wcscmp(traceLevel, L"ERROR") == 0) else if(wcscmp(loggingLevel, L"ERROR") == 0)
{ {
DrLogging::SetLoggingLevel(LogLevel_Error); DrLogging::SetLoggingLevel(LogLevel_Error);
} }
else if(wcscmp(traceLevel, L"WARN") == 0) else if(wcscmp(loggingLevel, L"WARN") == 0)
{ {
DrLogging::SetLoggingLevel(LogLevel_Warning); DrLogging::SetLoggingLevel(LogLevel_Warning);
} }
else if(wcscmp(traceLevel, L"INFO") == 0) else if(wcscmp(loggingLevel, L"INFO") == 0)
{ {
DrLogging::SetLoggingLevel(LogLevel_Info); DrLogging::SetLoggingLevel(LogLevel_Info);
} }
@ -249,16 +249,24 @@ static void ExceptionHandler(System::Object^ sender, System::UnhandledExceptionE
// //
// Start up vertex host // Start up vertex host
// //
[System::Security::SecurityCriticalAttribute] public ref class VertexHost
[System::Runtime::ExceptionServices::HandleProcessCorruptedStateExceptionsAttribute]
#if defined(_AMD64_)
int wmain(int argc, wchar_t* wargv[])
#else
int __cdecl wmain(int argc, wchar_t* wargv[])
#endif
{ {
public:
[System::Security::SecurityCriticalAttribute]
[System::Runtime::ExceptionServices::HandleProcessCorruptedStateExceptionsAttribute]
static int Main(array<System::String^>^ managedArgs)
{
try try
{ {
int argc = managedArgs->Length;
wchar_t** wargv = new wchar_t*[argc+1];
for (int i=0; i<argc; ++i)
{
pin_ptr<const wchar_t> wch = PtrToStringChars(managedArgs[i]);
wargv[i] = _wcsdup(wch);
}
wargv[argc] = NULL;
// //
// Enable logging based on environment variable // Enable logging based on environment variable
// //
@ -276,7 +284,7 @@ int __cdecl wmain(int argc, wchar_t* wargv[])
// //
// trace for startup // trace for startup
// //
DrLogI("Vertex Host starting"); DrLogE("Vertex Host starting");
// //
// Get environment variable to know whether to break into debugger // Get environment variable to know whether to break into debugger
@ -295,6 +303,12 @@ int __cdecl wmain(int argc, wchar_t* wargv[])
char** argv; char** argv;
DrGetUtf8CommandArgs(argc, wargv, &argv); DrGetUtf8CommandArgs(argc, wargv, &argv);
for (int i=0; i<managedArgs->Length; ++i)
{
free(wargv[i]);
}
delete [] wargv;
// //
// Initialize the dryad communication layer with the command line arguments // Initialize the dryad communication layer with the command line arguments
// //
@ -346,7 +360,8 @@ int __cdecl wmain(int argc, wchar_t* wargv[])
DrLogA("Unhandled exception: %s", DrString(e->ToString()).GetChars()); DrLogA("Unhandled exception: %s", DrString(e->ToString()).GetChars());
return 1; return 1;
} }
} }
};
// //
// Simple data class which contains the byte array and its length. // Simple data class which contains the byte array and its length.

27
GraphManager/GraphManager.vcxproj
View File

@ -1,6 +1,9 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<ItemGroup Label="ProjectConfigurations"> <ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64"> <ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration> <Configuration>Debug</Configuration>
@ -25,13 +28,13 @@
<ConfigurationType>DynamicLibrary</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CLRSupport>Pure</CLRSupport> <CLRSupport>Pure</CLRSupport>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType> <ConfigurationType>DynamicLibrary</ConfigurationType>
<CharacterSet>Unicode</CharacterSet> <CharacterSet>Unicode</CharacterSet>
<CLRSupport>Pure</CLRSupport> <CLRSupport>Pure</CLRSupport>
<PlatformToolset>v110</PlatformToolset> <PlatformToolset>v120</PlatformToolset>
</PropertyGroup> </PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings"> <ImportGroup Label="ExtensionSettings">
@ -59,6 +62,7 @@
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" /> <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad</TargetName> <TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad</TargetName>
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad</TargetName> <TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad</TargetName>
<NuGetPackageImportStamp>e2c2ee05</NuGetPackageImportStamp>
</PropertyGroup> </PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<Midl> <Midl>
@ -223,13 +227,22 @@
</ItemGroup> </ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
</ImportGroup> </ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
</Target> </Target>
</Project> </Project>

2
GraphManager/filesystem/DrAzureBlobClient.h
View File

@ -20,7 +20,7 @@ limitations under the License.
#pragma once #pragma once
using namespace Microsoft::Research::Peloponnese::Storage; using namespace Microsoft::Research::Peloponnese::Azure;
DRCLASS(DrAzureInputStream) : public DrInputStream DRCLASS(DrAzureInputStream) : public DrInputStream
{ {

205
GraphManager/filesystem/DrHdfsClient.cpp
View File

@ -129,37 +129,168 @@ DrHdfsInputStream::DrHdfsInputStream()
m_hdfsInstance = DrNull; m_hdfsInstance = DrNull;
} }
HRESULT DrHdfsInputStream::Open(DrUniversePtr universe, DrNativeString streamUri) HRESULT DrHdfsInputStream::Open(DrUniversePtr universe, DrNativeString streamUri, DrNativeString recordType)
{ {
DrString uri = DrString(streamUri); DrString uri = DrString(streamUri);
DrString record = DrString(recordType);
DrLogI("Opening instance for %s", uri.GetChars()); DrLogI("Opening instance for %s record type %s", uri.GetChars(), record.GetChars());
return OpenInternal(universe, uri); return OpenInternal(universe, uri, record);
} }
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri) #ifdef _MANAGED
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType)
{ {
m_streamUri = streamUri; m_streamUri = streamUri;
HRESULT err = S_OK; HRESULT err = S_OK;
#ifdef _MANAGED
try try
{ {
#endif DrLogI("Opening instance for %s: %s", streamUri.GetChars(), recordType.GetChars());
m_hdfsInstance = GetHdfsServiceInstance(streamUri);
HdfsFileInfo^ stream = m_hdfsInstance->GetFileInfo(streamUri.GetString(), true);
m_fileNameArray = stream->fileNameArray;
UInt32 totalPartitionCount;
if (recordType.Compare("Microsoft.Research.DryadLinq.LineRecord") == 0)
{
DrLogI("Getting block-level file info for %s", streamUri.GetChars());
totalPartitionCount = static_cast<UInt32>(stream->blockArray->Length);
}
else
{
DrLogI("Getting file info for %s", streamUri.GetChars());
totalPartitionCount = m_fileNameArray->Length;
}
DrLogI("Partition count %d", totalPartitionCount);
/* Allocate these arrays even if they're size 0, to avoid
NullReferenceException later */
m_affinity = DrNew DrAffinityArray(totalPartitionCount);
m_partOffsets = DrNew DrUINT64Array(totalPartitionCount);
m_partFileIds = DrNew DrUINT32Array(totalPartitionCount);
if (recordType.Compare("Microsoft.Research.DryadLinq.LineRecord") == 0)
{
for (UINT32 i = 0; i < totalPartitionCount; ++i)
{
HdfsBlockInfo^ partition = stream->blockArray[i];
m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(partition->Size);
m_partOffsets[i] = partition->Offset;
m_partFileIds[i] = partition->fileIndex;
for (int j = 0; j < partition->Hosts->Length; ++j)
{
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
m_affinity[i]->AddLocality(location);
}
}
}
}
else
{
int fileBlockIndex = 0;
for (UINT32 i = 0; i < totalPartitionCount; ++i)
{
m_partOffsets[i] = 0;
m_partFileIds[i] = i;
HdfsBlockInfo^ partition = stream->blockArray[fileBlockIndex];
DrAssert(partition->fileIndex == i);
long long fileSize = partition->Size;
HashSet<DrResourceRef>^ locations = DrNew HashSet<DrResourceRef>();
for (int j = 0; j < partition->Hosts->Length; ++j)
{
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
locations->Add(location);
}
}
++fileBlockIndex;
while (fileBlockIndex < stream->blockArray->Length && stream->blockArray[fileBlockIndex]->fileIndex == i)
{
partition = stream->blockArray[fileBlockIndex];
fileSize += partition->Size;
if (locations->Count > 0)
{
HashSet<DrResourceRef>^ newLocations = DrNew HashSet<DrResourceRef>();
for (int j = 0; j < partition->Hosts->Length; ++j)
{
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull)
{
newLocations->Add(location);
}
}
locations->IntersectWith(newLocations);
}
}
m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(fileSize);
System::Text::StringBuilder^ locationText;
if (locations->Count > 0)
{
locationText = gcnew System::Text::StringBuilder("File " + m_fileNameArray[i] + " merged locations:");
}
else
{
locationText = gcnew System::Text::StringBuilder("File " + m_fileNameArray[i] + " no shared locations");
}
HashSet<DrResourceRef>::Enumerator^ enumerator = locations->GetEnumerator();
while (enumerator->MoveNext())
{
m_affinity[i]->AddLocality(enumerator->Current);
locationText->Append(" ");
locationText->Append(enumerator->Current->GetName().GetString());
}
DrString locationLog(locationText->ToString());
DrLogI("%s", locationLog.GetChars());
}
}
}
catch (System::Exception ^e)
{
m_error = e->ToString();
DrString msg(m_error);
DrLogE("Got HDFS exception %s", msg.GetChars());
err = System::Runtime::InteropServices::Marshal::GetHRForException(e);
}
finally
{
// TODO: How do we clean this up?
//hdfsInstance->Discard();
}
return err;
}
#else
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType)
{
m_streamUri = streamUri;
HRESULT err = S_OK;
DrLogI("Opening instance for %s", streamUri.GetChars()); DrLogI("Opening instance for %s", streamUri.GetChars());
m_hdfsInstance = GetHdfsServiceInstance(streamUri); m_hdfsInstance = GetHdfsServiceInstance(streamUri);
#ifdef _MANAGED
DrLogI("Getting file info for %s", streamUri.GetChars());
HdfsFileInfo^ stream = m_hdfsInstance->GetFileInfo(streamUri.GetString(), true);
m_fileNameArray = stream->fileNameArray;
UInt32 totalPartitionCount = static_cast<UInt32>(stream->blockArray->Length);
#else
bool ret = HdfsBridgeNative::Initialize(); bool ret = HdfsBridgeNative::Initialize();
if (!ret) if (!ret)
{ {
@ -195,7 +326,6 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
totalPartitionCount = fs.GetNumberOfBlocks(); totalPartitionCount = fs.GetNumberOfBlocks();
m_fileNameArray = (const char **)fs.GetFileNameArray(); m_fileNameArray = (const char **)fs.GetFileNameArray();
#endif
DrLogI("Partition count %d", totalPartitionCount); DrLogI("Partition count %d", totalPartitionCount);
@ -207,21 +337,13 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
for (UINT32 i=0; i<totalPartitionCount; ++i) for (UINT32 i=0; i<totalPartitionCount; ++i)
{ {
#ifdef _MANAGED
HdfsBlockInfo^ partition = stream->blockArray[i];
#else
HdfsBridgeNative::HdfsBlockLocInfo* partition = fs.GetBlockInfo(i); HdfsBridgeNative::HdfsBlockLocInfo* partition = fs.GetBlockInfo(i);
#endif
m_affinity[i] = DrNew DrAffinity(); m_affinity[i] = DrNew DrAffinity();
m_affinity[i]->SetWeight(partition->Size); m_affinity[i]->SetWeight(partition->Size);
m_partOffsets[i] = partition->Offset; m_partOffsets[i] = partition->Offset;
m_partFileIds[i] = partition->fileIndex; m_partFileIds[i] = partition->fileIndex;
#ifdef _MANAGED
for (int j = 0; j < partition->Hosts->Length; ++j)
#else
for (int j = 0; j < partition->numberOfHosts; ++j) for (int j = 0; j < partition->numberOfHosts; ++j)
#endif
{ {
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]); DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
if (location != DrNull) if (location != DrNull)
@ -229,28 +351,12 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
m_affinity[i]->AddLocality(location); m_affinity[i]->AddLocality(location);
} }
} }
#ifndef _MANAGED
delete partition; delete partition;
#endif
} }
#ifdef _MANAGED
}
catch (System::Exception ^e)
{
m_error = e->ToString();
DrString msg(m_error);
DrLogE("Got HDFS exception %s", msg.GetChars());
err = System::Runtime::InteropServices::Marshal::GetHRForException(e);
}
finally
{
// TODO: How do we clean this up?
//hdfsInstance->Discard();
}
#endif
return err; return err;
} }
#endif
DrNativeString DrHdfsInputStream::GetError() DrNativeString DrHdfsInputStream::GetError()
{ {
@ -468,6 +574,25 @@ HRESULT DrHdfsOutputStream::FinalizeSuccessfulParts(DrOutputPartitionArrayRef pa
errorText.SetF("%s", DrString(m_error).GetChars()); errorText.SetF("%s", DrString(m_error).GetChars());
return E_FAIL; return E_FAIL;
} }
String^ userName = Environment::GetEnvironmentVariable("USER");
if (userName == nullptr)
{
userName = Environment::UserName;
}
try
{
m_hdfsInstance->SetOwnerAndPermission(m_baseUri, userName, nullptr, Convert::ToInt16("0755", 8));
}
catch (Exception^ e)
{
DrString drDst(m_baseUri);
DrString err(e->ToString());
DrLogE("Can't set %s permissions finalizing HDFS output: %s", drDst.GetChars(), err.GetChars());
m_error = "Can't set " + m_baseUri + " permissions finalizing HDFS output: " + e->ToString();
errorText.SetF("%s", DrString(m_error).GetChars());
return E_FAIL;
}
} }
else else
{ {

4
GraphManager/filesystem/DrHdfsClient.h
View File

@ -30,8 +30,8 @@ DRCLASS(DrHdfsInputStream) : public DrInputStream
{ {
public: public:
DrHdfsInputStream(); DrHdfsInputStream();
HRESULT Open(DrUniversePtr universe, DrNativeString streamUri); HRESULT Open(DrUniversePtr universe, DrNativeString streamUri, DrNativeString recordType);
HRESULT OpenInternal(DrUniversePtr universe, DrString streamUri); HRESULT OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType);
DrNativeString GetError(); DrNativeString GetError();
virtual DrString GetStreamName() DROVERRIDE; virtual DrString GetStreamName() DROVERRIDE;

24
GraphManager/filesystem/DrPartitionFile.cpp
View File

@ -79,6 +79,7 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
DrStringR remoteName, DrStringR remoteName,
DrPartitionInputStream::OverridePtr over, DrPartitionInputStream::OverridePtr over,
bool mustOverride, bool mustOverride,
bool pathIsRooted,
DrString line, DrString line,
DrUniversePtr universe) DrUniversePtr universe)
{ {
@ -112,9 +113,17 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
sep = lineCopy.IndexOfChar(','); sep = lineCopy.IndexOfChar(',');
if (sep == DrStr_InvalidIndex) if (sep == DrStr_InvalidIndex)
{ {
DrLogW("Malformed line %s: no list of machines", line.GetChars()); int n = sscanf_s(lineCopy.GetChars(), "%I64u", &parsedSize);
if (n != 1)
{
DrLogW("Malformed line %s: can't parse size", line.GetChars());
return false; return false;
} }
affinity->SetWeight(parsedSize);
lineCopy = DrString("");
}
else else
{ {
DrString partitionSizeString; DrString partitionSizeString;
@ -134,11 +143,17 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
} }
if (lineCopy.GetCharsLength() == 0) if (lineCopy.GetCharsLength() == 0)
{
if (!pathIsRooted || mustOverride)
{ {
DrLogW("Malformed line %s: no partition machines", line.GetChars()); DrLogW("Malformed line %s: no partition machines", line.GetChars());
return false; return false;
} }
remoteName.Set(" %Invalid% ");
return true;
}
int numberOfReplicas = 0; int numberOfReplicas = 0;
while (lineCopy.GetCharsLength() > 0) while (lineCopy.GetCharsLength() > 0)
{ {
@ -231,6 +246,12 @@ HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString st
mustOverride = true; mustOverride = true;
} }
bool pathIsRooted = false;
if (m_pathNameOnComputer.IndexOfChar(':') != DrStr_InvalidIndex)
{
pathIsRooted = true;
}
int numberOfParts; int numberOfParts;
int n = sscanf_s(partitionSizeLine.GetChars(), "%d", &numberOfParts); int n = sscanf_s(partitionSizeLine.GetChars(), "%d", &numberOfParts);
if (n != 1) if (n != 1)
@ -280,6 +301,7 @@ HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString st
remoteName, remoteName,
m_override[i], m_override[i],
mustOverride, mustOverride,
pathIsRooted,
partitionLine, partitionLine,
universe) == false) universe) == false)
{ {

19
GraphManager/jobmanager/version.cpp
View File

@ -19,5 +19,22 @@ limitations under the License.
*/ */
#ifdef _MANAGED #ifdef _MANAGED
[assembly:System::Runtime::InteropServices::ComVisible(false)]; #pragma managed
using namespace System::Reflection;
using namespace System::Runtime::InteropServices;
[assembly: AssemblyTitle("Microsoft.Research.Dryad")];
[assembly: AssemblyDescription("")];
[assembly: AssemblyConfiguration("")];
[assembly: AssemblyCompany("Microsoft Corporation")];
[assembly: AssemblyProduct("Microsoft.Research.Dryad")];
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
[assembly: AssemblyTrademark("")];
[assembly: AssemblyCulture("")];
[assembly: ComVisible(false)];
[assembly: AssemblyVersion("1.0.0.0")];
[assembly: AssemblyFileVersion("1.0.0.0")];
#endif #endif

5
GraphManager/packages.config
View File

@ -1,4 +1,7 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" /> <package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.0-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.0-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.0-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.0-beta" targetFramework="Native" />
</packages> </packages>

5
GraphManager/reporting/DrCalypsoReporting.cpp
View File

@ -24,7 +24,8 @@ limitations under the License.
#include <msclr\lock.h> #include <msclr\lock.h>
using namespace System::IO; using namespace System::IO;
using namespace Microsoft::Research::Peloponnese::Storage; using namespace Microsoft::Research::Peloponnese::Azure;
using namespace Microsoft::Research::Peloponnese::Hdfs;
using namespace msclr; using namespace msclr;
// //
@ -126,7 +127,7 @@ public:
DrCalypsoReporter::DrCalypsoReporter(DrNativeString uriString) DrCalypsoReporter::DrCalypsoReporter(DrNativeString uriString)
{ {
System::Uri^ uri = DrNew System::Uri(uriString); System::Uri^ uri = DrNew System::Uri(uriString);
if (uri->Scheme == AzureUtils::BlobScheme) if (uri->Scheme == Utils::BlobScheme)
{ {
m_logStream = DrNew AzureLogAppendStream(uri, 0x20, false, false, gcnew PeloponneseLogger()); m_logStream = DrNew AzureLogAppendStream(uri, 0x20, false, false, gcnew PeloponneseLogger());
m_flushInterval = 1000; m_flushInterval = 1000;

2
GraphManager/vertex/DrVertex.cpp
View File

@ -1743,7 +1743,7 @@ void DrTeeVertex::ReactToFailedVertex(DrVertexOutputGeneratorPtr failedGenerator
DrAssert(m_inputEdges->GetNumberOfEdges() == 1); DrAssert(m_inputEdges->GetNumberOfEdges() == 1);
DrEdge e = m_inputEdges->GetEdge(0); DrEdge e = m_inputEdges->GetEdge(0);
DrLogI("Tee vertex %d.%d: calling ReactToFailedVertex on remote edge", this->m_id, GetVersion()); DrLogI("Tee vertex %d.%d: calling ReactToFailedVertex on remote edge", this->m_id, failedGenerator->GetVersion());
e.m_remoteVertex->ReactToFailedVertex(failedGenerator, DrNull, DrNull, DrNull, originalReason); e.m_remoteVertex->ReactToFailedVertex(failedGenerator, DrNull, DrNull, DrNull, originalReason);
/* fill in a new generator if it's already there, e.g. if the upstream vertex is a DrStorageVertex */ /* fill in a new generator if it's already there, e.g. if the upstream vertex is a DrStorageVertex */

56
JobBrowser/Calypso/Properties/AssemblyInfo.cs Normal file
View File

@ -0,0 +1,56 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("DryadAnalysis")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("MSIT")]
[assembly: AssemblyProduct("DryadAnalysis")]
[assembly: AssemblyCopyright("Copyright © MSIT 2008")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// The following GUID is for the ID of the typelib if this project is exposed to COM
[assembly: Guid("286845bc-d060-49ff-9de8-4381471a0dc6")]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

1280
JobBrowser/JOM/ClusterConfiguration.cs
View File

File diff suppressed because it is too large Load Diff

375
JobBrowser/JOM/ClusterStatus.cs
View File

@ -19,12 +19,22 @@ limitations under the License.
*/ */
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using Microsoft.Research.Peloponnese.Storage; using System.Web;
using System.Xml;
using System.Xml.Linq;
using Microsoft.Research.Peloponnese.Hdfs;
using Microsoft.Research.Peloponnese.Shared;
using Microsoft.Research.Peloponnese.Yarn;
using Microsoft.Research.Tools; using Microsoft.Research.Tools;
using System.Text.RegularExpressions;
using System.Text;
using JobStatus = Microsoft.Research.Peloponnese.ClusterUtils.JobStatus;
namespace Microsoft.Research.JobObjectModel namespace Microsoft.Research.JobObjectModel
{ {
@ -68,12 +78,16 @@ namespace Microsoft.Research.JobObjectModel
/// <summary> /// <summary>
/// See if a status is already cached. /// See if a status is already cached.
/// </summary> /// </summary>
/// <param name="clusterName">Name of cluster.</param> /// <param name="config">Cluster configuration.</param>
/// <returns>The cached status.</returns> /// <returns>The cached status.</returns>
public static ClusterStatus LookupStatus(string clusterName) public static ClusterStatus LookupStatus(ClusterConfiguration config)
{ {
if (ClusterStatuses.ContainsKey(clusterName)) if (ClusterStatuses.ContainsKey(config.Name))
return ClusterStatuses[clusterName]; {
var retval = ClusterStatuses[config.Name];
if (retval.Config.Equals(config))
return retval;
}
return null; return null;
} }
@ -172,8 +186,8 @@ namespace Microsoft.Research.JobObjectModel
/// <param name="manager">Communication manager.</param> /// <param name="manager">Communication manager.</param>
public virtual void RefreshStatus(DryadLinqJobSummary summary, CommManager manager) public virtual void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
{ {
// refresh the whole list // refresh the whole list: too expensive
this.RecomputeClusterJobList(summary.VirtualCluster, manager); // this.RecomputeClusterJobList(summary.VirtualCluster, manager);
ClusterJobInformation info = this.DiscoverClusterJob(summary, manager); ClusterJobInformation info = this.DiscoverClusterJob(summary, manager);
if (info == null) if (info == null)
{ {
@ -193,6 +207,7 @@ namespace Microsoft.Research.JobObjectModel
/// <summary> /// <summary>
/// A fake cluster keeps some information about past jobs on a local filesystem, to allow post-mortem debugging. /// A fake cluster keeps some information about past jobs on a local filesystem, to allow post-mortem debugging.
/// </summary> /// </summary>
@ -442,20 +457,17 @@ namespace Microsoft.Research.JobObjectModel
/// <summary> /// <summary>
/// Status of an Azure DFS cluster. /// Status of an Azure DFS cluster.
/// </summary> /// </summary>
public class AzureDfsClusterStatus : ClusterStatus public abstract class DfsClusterStatus : ClusterStatus
{ {
private AzureDfsClusterConfiguration config;
/// <summary> /// <summary>
/// Create a cluster containing just the local machine. /// Create a cluster containing just the local machine.
/// </summary> /// </summary>
/// <param name="config">Configuration for the local machine.</param> /// <param name="config">Configuration for the local machine.</param>
public AzureDfsClusterStatus(ClusterConfiguration config) protected DfsClusterStatus(ClusterConfiguration config)
: base(config) : base(config)
{ {
if (!(config is AzureDfsClusterConfiguration)) if (!(config is DfsClusterConfiguration))
throw new ArgumentException("Expected a AzureYarnClusterConfiguration, got a " + config.GetType()); throw new ArgumentException("Expected a DfsClusterConfiguration, got a " + config.GetType());
this.config = config as AzureDfsClusterConfiguration;
} }
/// <summary> /// <summary>
@ -468,31 +480,6 @@ namespace Microsoft.Research.JobObjectModel
return result; return result;
} }
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var jobs = this.config.AzureClient.EnumerateDirectory("").ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(job);
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(job, info);
}
manager.Progress(100*done++/jobs.Count);
}
manager.Progress(100);
}
/// <summary> /// <summary>
/// Discover the (unique) dryadlinq job corresponding to a cluster job. /// Discover the (unique) dryadlinq job corresponding to a cluster job.
/// </summary> /// </summary>
@ -503,7 +490,7 @@ namespace Microsoft.Research.JobObjectModel
{ {
DryadLinqJobSummary result = new DryadLinqJobSummary( DryadLinqJobSummary result = new DryadLinqJobSummary(
clusterJob.Cluster, clusterJob.Cluster,
this.config.TypeOfCluster, this.Config.TypeOfCluster,
"", // virtual cluster "", // virtual cluster
"", // machine "", // machine
clusterJob.ClusterJobID, // jobId clusterJob.ClusterJobID, // jobId
@ -529,6 +516,78 @@ namespace Microsoft.Research.JobObjectModel
throw new InvalidOperationException(); throw new InvalidOperationException();
} }
/// <summary>
/// Cancel the specified job.
/// </summary>
/// <param name="job">Job whose execution is cancelled.</param>
/// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job)
{
return false;
}
}
/// <summary>
/// Status of an Azure DFS cluster.
/// </summary>
public class AzureDfsClusterStatus : DfsClusterStatus
{
private AzureDfsClusterConfiguration config;
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="config">Configuration for the local machine.</param>
public AzureDfsClusterStatus(ClusterConfiguration config)
: base(config)
{
if (!(config is AzureDfsClusterConfiguration))
throw new ArgumentException("Expected a AzureDfsClusterConfiguration, got a " + config.GetType());
this.config = config as AzureDfsClusterConfiguration;
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var jobs = this.config.AzureClient.ExpandFileOrDirectory(AzureDfsFile.UriFromPath(this.config, "")).ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
string jobRootFolder = AzureDfsFile.PathFromUri(this.config, job);
ClusterJobInformation info = this.GetJobInfo(jobRootFolder);
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(job.AbsolutePath, info);
}
manager.Progress(100*done++/jobs.Count);
}
manager.Progress(100);
}
/// <summary>
/// Extract blob name from a path.
/// </summary>
/// <param name="container">Container name.</param>
/// <param name="path">Path.</param>
/// <returns>The blob part of path.</returns>
public static string GetBlobName(string container, string path)
{
if (path.StartsWith("/" + container + "/"))
path = path.Substring(container.Length + 2);
int q = path.IndexOf('?');
if (q > 0)
path = path.Substring(0, q);
return path;
}
/// <summary> /// <summary>
/// Extract the job information from a folder with logs on the local machine. /// Extract the job information from a folder with logs on the local machine.
/// </summary> /// </summary>
@ -540,14 +599,19 @@ namespace Microsoft.Research.JobObjectModel
DateTime lastHeartBeat = DateTime.MinValue; DateTime lastHeartBeat = DateTime.MinValue;
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown; ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
bool found = false; bool found = false;
Uri uri = AzureDfsFile.UriFromPath(this.config, jobRootFolder);
var jobsFolders = this.config.AzureClient.ExpandFileOrDirectory(uri).ToList();
jobRootFolder = GetBlobName(this.config.Container, jobRootFolder);
string jobName = jobRootFolder; string jobName = jobRootFolder;
var jobsFolders = this.config.AzureClient.EnumerateDirectory(jobRootFolder).ToList();
foreach (var file in jobsFolders) foreach (var file in jobsFolders)
{ {
if (file.EndsWith("heartbeat")) if (file.AbsolutePath.EndsWith("heartbeat"))
{ {
var blob = this.config.AzureClient.Container.GetPageBlobReference(file); string blobName = GetBlobName(this.config.Container, file.AbsolutePath);
var blob = this.config.AzureClient.Container.GetPageBlobReference(blobName);
blob.FetchAttributes(); blob.FetchAttributes();
var props = blob.Metadata; var props = blob.Metadata;
if (props.ContainsKey("status")) if (props.ContainsKey("status"))
@ -597,11 +661,11 @@ namespace Microsoft.Research.JobObjectModel
found = true; found = true;
} }
else if (file.Contains("DryadLinqProgram__") && else if (file.AbsolutePath.Contains("DryadLinqProgram__") &&
// newer heartbeats contain the date // newer heartbeats contain the date
date != DateTime.MinValue) date != DateTime.MinValue)
{ {
var blob = this.config.AzureClient.Container.GetBlockBlobReference(file); var blob = this.config.AzureClient.Container.GetBlockBlobReference(AzureDfsFile.PathFromUri(this.config, file));
blob.FetchAttributes(); blob.FetchAttributes();
var props = blob.Properties; var props = blob.Properties;
if (props.LastModified.HasValue) if (props.LastModified.HasValue)
@ -628,7 +692,6 @@ namespace Microsoft.Research.JobObjectModel
/// <param name="manager">Communication manager.</param> /// <param name="manager">Communication manager.</param>
public override void RefreshStatus(DryadLinqJobSummary summary, CommManager manager) public override void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
{ {
// refresh the whole list
ClusterJobInformation info = this.GetJobInfo(summary.JobID); ClusterJobInformation info = this.GetJobInfo(summary.JobID);
if (info == null) if (info == null)
{ {
@ -645,8 +708,224 @@ namespace Microsoft.Research.JobObjectModel
/// <returns>True if the cancellation succeeded.</returns> /// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job) public override bool CancelJob(DryadLinqJobSummary job)
{ {
AzureUtils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId); Microsoft.Research.Peloponnese.Azure.Utils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId);
return true; return false;
}
}
/// <summary>
/// Cluster status of a WebHdfs cluster.
/// </summary>
public class WebHdfsClusterStatus : DfsClusterStatus
{
private WebHdfsClusterConfiguration config;
/// <summary>
/// Yarn client to access job status.
/// </summary>
private NativeYarnClient yarnClient;
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public WebHdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is WebHdfsClusterConfiguration))
throw new ArgumentException("Expected a WebHdfsClusterConfiguration, got a " + conf.GetType());
this.config = conf as WebHdfsClusterConfiguration;
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
private ClusterJobInformation GetJobInfo(string jobRootFolder)
{
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
long time;
long size;
this.config.DfsClient.GetFileStatus(uri, out time, out size);
DateTime date = DfsFile.TimeFromLong(time);
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
string jobName = Path.GetFileName(jobRootFolder);
string errorMsg = "";
try
{
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
var jobstatus = jobinfo.GetStatus();
errorMsg = jobinfo.ErrorMsg;
switch (jobstatus)
{
case JobStatus.NotSubmitted:
case JobStatus.Waiting:
status = ClusterJobInformation.ClusterJobStatus.Unknown;
break;
case JobStatus.Running:
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case JobStatus.Success:
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
break;
case JobStatus.Cancelled:
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
case JobStatus.Failure:
status = ClusterJobInformation.ClusterJobStatus.Failed;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception)
{
}
TimeSpan running = TimeSpan.Zero;
var info = new ClusterJobInformation(this.config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
return info;
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable once UnusedParameter.Global
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
var jobsEnum = this.config.DfsClient.EnumerateSubdirectories(uri);
List<Uri> jobs = jobsEnum != null ? jobsEnum.ToList() : new List<Uri>();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(DfsFile.PathFromUri(this.config.JobsFolderUri, job));
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(info.ClusterJobID, info);
}
manager.Progress(100 * done++ / jobs.Count);
}
manager.Progress(100);
}
}
/// <summary>
/// Cluster status of a WebHdfs cluster.
/// </summary>
public class HdfsClusterStatus : DfsClusterStatus
{
private HdfsClusterConfiguration config;
/// <summary>
/// Yarn client to access job status.
/// </summary>
private NativeYarnClient yarnClient;
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public HdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is HdfsClusterConfiguration))
throw new ArgumentException("Expected an HdfsClusterConfiguration, got a " + conf.GetType());
this.config = conf as HdfsClusterConfiguration;
// make a fake call to initialize the cluster on the foreground thread
// HDFS does not work if initialized on the background thread.
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
this.config.DfsClient.IsFileExists(uri); // ignore result
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Extract the job information from a folder with logs on the local machine.
/// </summary>
/// <param name="jobRootFolder">Folder with logs for the specified job.</param>
/// <returns>The job information, or null if not found.</returns>
private ClusterJobInformation GetJobInfo(string jobRootFolder)
{
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, jobRootFolder);
long time;
long size;
this.config.DfsClient.GetFileStatus(uri, out time, out size);
DateTime date = DfsFile.TimeFromLong(time);
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
string jobName = Path.GetFileName(jobRootFolder);
string errorMsg = "";
try
{
var jobinfo = this.yarnClient.QueryJob(jobName, uri);
var jobstatus = jobinfo.GetStatus();
errorMsg = jobinfo.ErrorMsg;
switch (jobstatus)
{
case JobStatus.NotSubmitted:
case JobStatus.Waiting:
status = ClusterJobInformation.ClusterJobStatus.Unknown;
break;
case JobStatus.Running:
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case JobStatus.Success:
status = ClusterJobInformation.ClusterJobStatus.Succeeded;
break;
case JobStatus.Cancelled:
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
case JobStatus.Failure:
status = ClusterJobInformation.ClusterJobStatus.Failed;
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (Exception)
{
}
TimeSpan running = TimeSpan.Zero;
var info = new ClusterJobInformation(config.Name, "", jobName, jobName, Environment.UserName, date, running, status);
return info;
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable once UnusedParameter.Global
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
var jobs = this.config.DfsClient.EnumerateSubdirectories(uri).ToList();
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(DfsFile.PathFromUri(this.config.JobsFolderUri, job));
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(info.ClusterJobID, info);
}
manager.Progress(100 * done++ / jobs.Count);
}
manager.Progress(100);
} }
} }
} }

266
JobBrowser/JOM/JobObjectModel.csproj
View File

@ -1,6 +1,12 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.6-alpha042\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.6-alpha042\build\Microsoft.Research.Peloponnese.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -9,39 +15,17 @@
<ProjectGuid>{77739535-7FAC-4487-887F-FEBA197E7572}</ProjectGuid> <ProjectGuid>{77739535-7FAC-4487-887F-FEBA197E7572}</ProjectGuid>
<OutputType>Library</OutputType> <OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>DistributedDataCollection</RootNamespace> <RootNamespace>JobObjectModel</RootNamespace>
<AssemblyName>DistributedDataCollection</AssemblyName> <AssemblyName>JobObjectModel</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<StartupObject> <NuGetPackageImportStamp>71873e18</NuGetPackageImportStamp>
</StartupObject>
<FileUpgradeFlags>
</FileUpgradeFlags>
<UpgradeBackupLocation>
</UpgradeBackupLocation>
<OldToolsVersion>3.5</OldToolsVersion>
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath> <OutputPath>bin\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants> <DefineConstants>DEBUG;TRACE</DefineConstants>
<DocumentationFile>bin\Debug\DistributedDataCollection.XML</DocumentationFile> <DocumentationFile>bin\Debug\JobObjectModel.XML</DocumentationFile>
<DebugType>full</DebugType> <DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget> <PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport> <ErrorReport>prompt</ErrorReport>
@ -56,108 +40,13 @@
<ErrorReport>prompt</ErrorReport> <ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet> <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup> </PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.Xml.Linq">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.DataSetExtensions">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup> <ItemGroup>
<Compile Include="ClusterAccess.cs" /> <Compile Include="ClusterAccess.cs" />
<Compile Include="ClusterConfiguration.cs" /> <Compile Include="ClusterConfiguration.cs" />
<Compile Include="ClusterStatus.cs" /> <Compile Include="ClusterStatus.cs" />
<Compile Include="dryadlog.cs" /> <Compile Include="dryadlog.cs" />
<Compile Include="jobinfo.cs" /> <Compile Include="jobinfo.cs" />
<Compile Include="JobSummary.cs" />
<Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Storage.cs" /> <Compile Include="Storage.cs" />
</ItemGroup> </ItemGroup>
@ -168,40 +57,121 @@
</ProjectReference> </ProjectReference>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5"> <None Include="app.config" />
<Visible>False</Visible> <None Include="packages.config">
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName> <SubType>Designer</SubType>
<Install>false</Install> </None>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<None Include="app.config" /> <Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<None Include="packages.config" /> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.3.1.6, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.1.6\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.HDInsight.Net.Http.Formatting, Version=1.3.1.6, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.1.6\lib\net40\Microsoft.HDInsight.Net.Http.Formatting.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.3.1.6, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.3.1.6\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.3.1.6, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.1.6\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.3.1.6, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.3.1.6\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.3.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System">
<HintPath>..\..\..\..\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5\System.dll</HintPath>
</Reference>
<Reference Include="System.Data" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" />
<Reference Include="System.XML" />
<Reference Include="System.Xml.Linq" />
</ItemGroup> </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.targets'))" />
</Target> </Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild"> <Target Name="BeforeBuild">

488
JobBrowser/JOM/JobSummary.cs Normal file
View File

@ -0,0 +1,488 @@
/*
Copyright (c) Microsoft Corporation
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License
at http://www.apache.org/licenses/LICENSE-2.0
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
See the Apache Version 2.0 License for specific language governing permissions and
limitations under the License.
*/
using System;
using System.IO;
using System.Text;
using Microsoft.Research.Tools;
namespace Microsoft.Research.JobObjectModel
{
/// <summary>
/// Error during conversation with cluster.
/// </summary>
public sealed class ClusterException : Exception
{
/// <summary>
/// Create an exception about handling a cluster.
/// </summary>
/// <param name="message">Exception message.</param>
public ClusterException(string message) : base(message) { }
}
/// <summary>
/// Identifier for a Dryad process; for now we are using globally-unique process GUID.
/// A Dryad Job identifier is always the same as the job manager process guid.
/// </summary>
[Serializable]
public class DryadProcessIdentifier : IEquatable<DryadProcessIdentifier>
{
/// <summary>
/// Process identifier.
/// </summary>
private string processIdentifier;
/// <summary>
/// Used just for XML serialization.
/// </summary>
public DryadProcessIdentifier()
{ }
/// <summary>
/// create an indentifier struct.
/// Be sure to initialize all fields before use;
/// </summary>
/// <param name="pid">The id of the process (platform-dependent).</param>
public DryadProcessIdentifier(string pid)
{
this.processIdentifier = pid;
}
/// <summary>
/// Process identifier; public for serialization only.
/// </summary>
public string ProcessIdentifier
{
get { return this.processIdentifier; }
set { this.processIdentifier = value; }
}
/// <summary>
/// If true the process identifier is not known.
/// </summary>
public bool IsUnknown { get { return this.ProcessIdentifier == null; } }
/// <summary>
/// Human-readable description of the process identifier.
/// </summary>
/// <returns>An empty string if the pid is not set.</returns>
public override string ToString()
{
if (this.ProcessIdentifier != null) return this.ProcessIdentifier;
return "";
}
/// <summary>
/// Equality test.
/// </summary>
/// <param name="obj">Object to compare to.</param>
/// <returns>True if both objects represent the same process id.</returns>
public override bool Equals(object obj)
{
if (!(obj is DryadProcessIdentifier))
return false;
return this.Equals((DryadProcessIdentifier)obj);
}
#region IEquatable<DryadProcessIdentifier> Members
/// <summary>
/// Equality test.
/// </summary>
/// <param name="other">Process id to compare to.</param>
/// <returns>True if the id's represent the same process.</returns>
public bool Equals(DryadProcessIdentifier other)
{
if (this.IsUnknown)
return other.IsUnknown;
if (other.IsUnknown)
return false;
return this.ProcessIdentifier.Equals(other.ProcessIdentifier);
}
/// <summary>
/// Overriden implementation of getHashCode.
/// </summary>
/// <returns>The hashcode of the process id.</returns>
public override int GetHashCode()
{
// ReSharper disable once BaseObjectGetHashCodeCallInGetHashCode
return base.GetHashCode();
}
#endregion
}
/// <summary>
/// Brief summary of an executed DryadLINQ job.
/// </summary>
[Serializable]
public sealed class DryadLinqJobSummary : IEquatable<DryadLinqJobSummary>
{
/// <summary>
/// Empty constructor for XML serialization.
/// </summary>
public DryadLinqJobSummary()
{ }
/// <summary>
/// Initialize a job summary.
/// </summary>
/// <param name="cluster">Cluster where the job ran.</param>
/// <param name="clusterType">A string corresponding to the type of ClusterConfiguration.</param>
/// <param name="machine">Machine where job manager ran.</param>
/// <param name="jobId">Id of job.</param>
/// <param name="jmProcessGuid">Guid of job manager process.</param>
/// <param name="clusterJobId">Id of job on the cluster.</param>
/// <param name="friendlyname">Friendly name used.</param>
/// <param name="username">Who ran the job.</param>
/// <param name="date">Start date (not completion date).</param>
/// <param name="status">Job status.</param>
/// <param name="endTime">Estimated end running time.</param>
/// <param name="virtualcluster">Virtual cluster where job ran.</param>
public DryadLinqJobSummary(
string cluster,
ClusterConfiguration.ClusterType clusterType,
string virtualcluster,
string machine,
string jobId,
string clusterJobId,
DryadProcessIdentifier jmProcessGuid,
string friendlyname,
string username,
DateTime date,
DateTime endTime,
ClusterJobInformation.ClusterJobStatus status)
{
this.VirtualCluster = virtualcluster;
this.Cluster = cluster;
this.ClusterType = clusterType;
this.Machine = machine;
this.Name = friendlyname;
this.User = username;
this.Date = date;
this.EndTime = endTime;
this.Status = status;
this.ManagerProcessGuid = jmProcessGuid;
this.JobID = jobId;
this.ClusterJobId = clusterJobId;
}
/// <summary>
/// Cluster where the job ran.
/// </summary>
public string Cluster { get; /*private*/ set; }
/// <summary>
/// Id of cluster job that originated this DryadLinq job (can be used to find the cluster job from the dryadlinq job).
/// </summary>
public string ClusterJobId { get; /*private*/ set; }
/// <summary>
/// Cluster where the job ran.
/// </summary>
public DateTime EndTime { get; /*private*/ set; }
/// <summary>
/// String describing cluster type.
/// </summary>
public ClusterConfiguration.ClusterType ClusterType { get; /*private*/ set; }
/// <summary>
/// Virtual cluster where job ran.
/// </summary>
public string VirtualCluster { get; /*private*/ set; }
/// <summary>
/// (Friendly) name of the job.
/// </summary>
public string Name { get; /*private*/ set; }
/// <summary>
/// User who submitted job.
/// </summary>
public string User { get; /*private*/ set; }
/// <summary>
/// ID of job on the cluster.
/// </summary>
public string JobID { get; /*private*/ set; }
/// <summary>
/// The Guid of the job manager process.
/// </summary>
public DryadProcessIdentifier ManagerProcessGuid { set; get; }
/// <summary>
/// User who submitted job.
/// </summary>
public string GetAlias()
{
int pos = User.IndexOf(@"\");
return User.Substring(pos + 1);
}
/// <summary>
/// Date when job was submitted.
/// </summary>
public DateTime Date { get; /*private*/ set; }
/// <summary>
/// Did the job fail?
/// </summary>
public ClusterJobInformation.ClusterJobStatus Status { get; /*internal*/ set; }
/// <summary>
/// Machine where the job manager ran.
/// </summary>
public string Machine { get; /*private*/ set; }
/// <summary>
/// Get a short name for this job summary.
/// </summary>
/// <returns>Short name of job summary.</returns>
public string ShortName()
{
// we use the starting time to uniquify the job name
return this.Date.ToString("s") + "-" + this.Name;
}
/// <summary>
/// True if these two summaries are the same. The status and end time do not matter, since the job may still be running.
/// </summary>
/// <param name="other">Summary to compare against.</param>
/// <returns>True if they are equal.</returns>
public bool Equals(DryadLinqJobSummary other)
{
return this.Cluster == other.Cluster &&
this.ClusterJobId == other.ClusterJobId &&
this.Date == other.Date &&
this.Machine == other.Machine &&
this.Name == other.Name &&
this.User == other.User;
}
/// <summary>
/// Hashcode proper for the equality test.
/// </summary>
/// <returns>The object hashcode.</returns>
public override int GetHashCode()
{
return this.ClusterJobId.GetHashCode() ^ this.ClusterJobId.GetHashCode() ^ this.Date.GetHashCode() ^ this.Machine.GetHashCode() ^ this.Name.GetHashCode() ^ this.User.GetHashCode();
}
/// <summary>
/// A string describing the unique identifying part of the summary.
/// Two different summaries may represent the same job at different times.
/// </summary>
/// <returns>The part common to all jobs.</returns>
public string AsIdentifyingString()
{
StringBuilder builder = new StringBuilder();
builder.AppendFormat("Cluster={0} ClusterJobID={1} Date={2} Machine={3} Name={4} User={5}",
this.Cluster, this.ClusterJobId, this.Date, this.Machine, this.Name, this.User);
return builder.ToString();
}
}
/// <summary>
/// This class is an abstraction of a cluster-level job, as opposed to a DryadLINQ job.
/// In Cosmos that's called a task, in HPC that's called a Job.
/// (In cosmos a task is a recurring job. In DryadLINQ running on top of cosmos, a task always contains exactly one job.)
/// </summary>
public class ClusterJobInformation : IEquatable<ClusterJobInformation>
{
/// <summary>
/// Status of a cluster job.
/// </summary>
public enum ClusterJobStatus
{
/// <summary>
/// Job is still running.
/// </summary>
Running,
/// <summary>
/// Job has finished successfully.
/// </summary>
Succeeded,
/// <summary>
/// Job has finished and has failed.
/// </summary>
Failed,
/// <summary>
/// Job has been cancelled. Not precise on cosmos clusters.
/// </summary>
Cancelled,
/// <summary>
/// Could not determine job status.
/// </summary>
Unknown,
};
/// <summary>
/// True if job is finished, false if not, or unknown.
/// </summary>
/// <param name="status">Job status.</param>
/// <returns>True if the job is no longer running.</returns>
public static bool JobIsFinished(ClusterJobStatus status)
{
switch (status)
{
case ClusterJobInformation.ClusterJobStatus.Failed:
case ClusterJobInformation.ClusterJobStatus.Succeeded:
case ClusterJobInformation.ClusterJobStatus.Cancelled:
return true;
case ClusterJobInformation.ClusterJobStatus.Running:
case ClusterJobInformation.ClusterJobStatus.Unknown:
return false;
default:
throw new InvalidDataException("Invalid job status " + status);
}
}
/// <summary>
/// Create a cluster job structure from a bunch of information.
/// </summary>
/// <param name="cluster">Cluster where the job is running.</param>
/// <param name="clusterJobGuid">Cluster job guid.</param>
/// <param name="jobName">Name of the cluster job.</param>
/// <param name="username">User who submitted cluster job.</param>
/// <param name="date">Last execution of cluster job.</param>
/// <param name="status">Execution status.</param>
/// <param name="runningTime">Time the job ran.</param>
/// <param name="virtualCluster">Cluster where the job has run.</param>
public ClusterJobInformation(
string cluster,
string virtualCluster,
string clusterJobGuid,
string jobName,
string username,
DateTime date,
TimeSpan runningTime,
ClusterJobStatus status)
{
this.VirtualCluster = virtualCluster;
this.Cluster = cluster;
this.ClusterJobID = clusterJobGuid;
this.Name = jobName;
this.User = username;
this.Date = date;
this.EstimatedRunningTime = runningTime;
this.Status = status;
this.JobSummary = null;
}
/// <summary>
/// Name of cluster job.
/// </summary>
public string Name { get; set; }
/// <summary>
/// Uset who submitted cluster job.
/// </summary>
public string User { get; set; }
/// <summary>
/// Date when job was submitted.
/// </summary>
public DateTime Date { get; set; }
/// <summary>
/// ID of Job on cluster.
/// </summary>
public string ClusterJobID { get; set; }
/// <summary>
/// Status of the execution.
/// </summary>
public ClusterJobStatus Status { get; set; }
/// <summary>
/// Cluster where the job ran.
/// </summary>
public string Cluster { get; set; }
/// <summary>
/// In some installations a cluster is composed of multiple virtual clusters.
/// </summary>
public string VirtualCluster { get; set; }
/// <summary>
/// Is the cluster job information still available on the cluster?
/// </summary>
public bool IsUnavailable { get; set; }
/// <summary>
/// Cache here the associated job, if available. Null if not cached.
/// </summary>
private DryadLinqJobSummary JobSummary { get; set; }
/// <summary>
/// Estimated time the job ran.
/// </summary>
public TimeSpan EstimatedRunningTime { get; set; }
/// <summary>
/// If known, set the associated job summary.
/// </summary>
/// <param name="summary">Job summary for this cluster job.</param>
public void SetAssociatedSummary(DryadLinqJobSummary summary)
{
this.JobSummary = summary;
}
/// <summary>
/// Discover the dryadlinq job associated with a cluster job.
/// </summary>
/// <param name="status">Cluster configuration.</param>
/// <returns>The job, if any</returns>
/// <param name="reporter">Delegate used to report errors.</param>
public DryadLinqJobSummary DiscoverDryadLinqJob(ClusterStatus status, StatusReporter reporter)
{
if (this.IsUnavailable)
return null;
if (this.JobSummary != null)
return this.JobSummary;
DryadLinqJobSummary j = status.DiscoverDryadLinqJobFromClusterJob(this, reporter);
if (j == null)
{
this.IsUnavailable = true;
}
return this.JobSummary = j;
}
/// <summary>
/// Copy the content of a cluster job.
/// </summary>
/// <param name="refresh">The value to copy from.</param>
internal void Copy(ClusterJobInformation refresh)
{
this.Name = refresh.Name;
this.Status = refresh.Status;
this.User = refresh.User;
this.JobSummary = refresh.JobSummary;
this.ClusterJobID = refresh.ClusterJobID;
this.Date = refresh.Date;
this.IsUnavailable = refresh.IsUnavailable;
this.VirtualCluster = refresh.VirtualCluster;
this.EstimatedRunningTime = refresh.EstimatedRunningTime;
}
/// <summary>
/// True if these two records represent the same job.
/// </summary>
/// <param name="other"></param>
/// <returns></returns>
public bool Equals(ClusterJobInformation other)
{
return
this.Cluster == other.Cluster &&
this.VirtualCluster == other.VirtualCluster &&
this.Name == other.Name &&
this.User == other.User &&
this.ClusterJobID == other.ClusterJobID &&
this.Date == other.Date;
}
}
}

8
JobBrowser/JOM/app.config
View File

@ -4,15 +4,15 @@
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1"> <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
@ -20,7 +20,7 @@
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" /> <bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly> </dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>

372
JobBrowser/JOM/clusteraccess.cs
View File

@ -19,14 +19,20 @@ limitations under the License.
*/ */
using System.Linq;
using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Diagnostics; using System.Diagnostics;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Azure;
using Microsoft.Research.Tools; using Microsoft.Research.Tools;
using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.Research.Peloponnese.Shared;
using Microsoft.Research.Peloponnese.WebHdfs;
using Microsoft.Research.Peloponnese.Hdfs;
namespace Microsoft.Research.JobObjectModel namespace Microsoft.Research.JobObjectModel
{ {
@ -43,7 +49,8 @@ namespace Microsoft.Research.JobObjectModel
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder. /// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary> /// </summary>
/// <returns>A stream that can be used to access the object contents.</returns> /// <returns>A stream that can be used to access the object contents.</returns>
ISharedStreamReader GetStream(); /// <param name="keepNewline">If true the streamReader will not strip the newlines.</param>
ISharedStreamReader GetStream(bool keepNewline);
/// <summary> /// <summary>
/// If the current object is a folder, it returns the contained objects. /// If the current object is a folder, it returns the contained objects.
/// </summary> /// </summary>
@ -203,12 +210,13 @@ namespace Microsoft.Research.JobObjectModel
/// A stream to the local cache, or null if the file is not cached. /// A stream to the local cache, or null if the file is not cached.
/// </summary> /// </summary>
/// <returns>A stream to access the file.</returns> /// <returns>A stream to access the file.</returns>
public virtual ISharedStreamReader GetStream() /// <param name="keepNewline">If true keep newlines.</param>
public virtual ISharedStreamReader GetStream(bool keepNewline)
{ {
if (this.LocalCachePath != null && File.Exists(this.LocalCachePath)) if (this.LocalCachePath != null && File.Exists(this.LocalCachePath))
{ {
CachedClusterResidentObject.Record(this); CachedClusterResidentObject.Record(this);
return new FileSharedStreamReader(this.LocalCachePath); return new FileSharedStreamReader(this.LocalCachePath, keepNewline);
} }
return null; return null;
} }
@ -355,14 +363,15 @@ namespace Microsoft.Research.JobObjectModel
/// The stream with the file contents. /// The stream with the file contents.
/// </summary> /// </summary>
/// <returns>A stream reder.</returns> /// <returns>A stream reder.</returns>
public override ISharedStreamReader GetStream() /// <param name="keepNewlines">If true keep the newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewlines)
{ {
try try
{ {
if (!this.RepresentsAFolder) if (!this.RepresentsAFolder)
{ {
//this.LocalCachePath = this.CachePath(this.Pathname); //this.LocalCachePath = this.CachePath(this.Pathname);
ISharedStreamReader baseStream = base.GetStream(); ISharedStreamReader baseStream = base.GetStream(keepNewlines);
if (baseStream != null) if (baseStream != null)
{ {
// file is cached // file is cached
@ -378,12 +387,12 @@ namespace Microsoft.Research.JobObjectModel
throw new ClusterException("Cannot cache folders"); throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream(); StreamWriter writer = this.CreateTempStream();
return new FileSharedStreamReader(this.Pathname.ToString(), writer, this.OnClose); return new FileSharedStreamReader(this.Pathname.ToString(), writer, keepNewlines, this.OnClose);
} }
else else
{ {
// dont cache it // dont cache it
return new FileSharedStreamReader(this.Pathname.ToString()); return new FileSharedStreamReader(this.Pathname.ToString(), keepNewlines);
} }
} }
catch (Exception ex) catch (Exception ex)
@ -650,9 +659,10 @@ namespace Microsoft.Research.JobObjectModel
/// The contents of the folder. /// The contents of the folder.
/// </summary> /// </summary>
/// <returns>The contents of the folder.</returns> /// <returns>The contents of the folder.</returns>
public override ISharedStreamReader GetStream() /// <param name="keepNewline">If true keep newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewline)
{ {
return this.OriginalFolder.GetStream(); return this.OriginalFolder.GetStream(keepNewline);
} }
/// <summary> /// <summary>
@ -720,9 +730,10 @@ namespace Microsoft.Research.JobObjectModel
/// A stream returning the contents. /// A stream returning the contents.
/// </summary> /// </summary>
/// <returns>The contents of this object.</returns> /// <returns>The contents of this object.</returns>
public ISharedStreamReader GetStream() /// <param name="keepNewlines">If true keep newlines.</param>
public ISharedStreamReader GetStream(bool keepNewlines)
{ {
return new StringIteratorStreamReader(this.contents); return new StringIteratorStreamReader(this.contents, keepNewlines);
} }
/// <summary> /// <summary>
@ -835,9 +846,10 @@ namespace Microsoft.Research.JobObjectModel
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder. /// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary> /// </summary>
/// <returns>A stream that can be used to access the object contents.</returns> /// <returns>A stream that can be used to access the object contents.</returns>
public ISharedStreamReader GetStream() /// <param name="keepNewlines">If true keep the newlines.</param>
public ISharedStreamReader GetStream(bool keepNewlines)
{ {
return new FileSharedStreamReader(this.path); return new FileSharedStreamReader(this.path, keepNewlines);
} }
/// <summary> /// <summary>
@ -969,8 +981,8 @@ namespace Microsoft.Research.JobObjectModel
this.RepresentsAFolder = isFolder; this.RepresentsAFolder = isFolder;
this.size = -1; this.size = -1;
if (!string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory)) if (!String.IsNullOrEmpty(CacheDirectory))
this.LocalCachePath = Path.Combine(CachedClusterResidentObject.CacheDirectory, this.path); this.LocalCachePath = Path.Combine(CacheDirectory, this.path);
} }
/// <summary> /// <summary>
@ -982,14 +994,46 @@ namespace Microsoft.Research.JobObjectModel
protected set; protected set;
} }
/// <summary>
/// Create a File URI from a file path.
/// </summary>
/// <param name="path">Path to file.</param>
/// <returns>The File uri.</returns>
/// <param name="config">Azure cluster configuration.</param>
public static Uri UriFromPath(AzureDfsClusterConfiguration config, string path)
{
if (path.StartsWith(config.Container))
throw new InvalidOperationException("Path contains container name");
var retval = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(config.AccountName, config.Container, path, null, config.AccountKey);
//Console.WriteLine("Uri {0}", retval);
return retval;
}
/// <summary>
/// Create a path from a URI.
/// </summary>
/// <param name="uri">URI of an HDFS file.</param>
/// <returns>The path to the file.</returns>
/// <param name="config">Azure cluster configuration.</param>
public static string PathFromUri(AzureDfsClusterConfiguration config, Uri uri)
{
string path = uri.LocalPath;
path = path.Trim('/');
if (path.StartsWith(config.Container))
path = path.Substring(config.Container.Length);
path = path.Trim('/');
return path;
}
/// <summary> /// <summary>
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder. /// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary> /// </summary>
/// <returns>A stream that can be used to access the object contents.</returns> /// <returns>A stream that can be used to access the object contents.</returns>
public override ISharedStreamReader GetStream() /// <param name="keepNewlines">If true keep the newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewlines)
{ {
ISharedStreamReader baseStream = base.GetStream(); ISharedStreamReader baseStream = base.GetStream(keepNewlines);
if (baseStream != null) if (baseStream != null)
{ {
// file is cached // file is cached
@ -1000,29 +1044,30 @@ namespace Microsoft.Research.JobObjectModel
Stream stream; Stream stream;
if (this.IsDfsStream) if (this.IsDfsStream)
{ {
var dfsFileStream = this.client.GetDfsFileStream(this.path); Uri uri = UriFromPath(this.Config as AzureDfsClusterConfiguration, this.path);
stream = dfsFileStream.Stream; stream = this.client.GetDfsStreamReader(uri);
} }
else else
{ {
string p = AzureDfsClusterStatus.GetBlobName(this.client.ContainerName,this.path);
stream = new AzureLogReaderStream( stream = new AzureLogReaderStream(
this.client.AccountName, this.client.AccountName,
this.client.AccountKey, this.client.AccountKey,
this.client.ContainerName, this.client.ContainerName,
this.path); p);
} }
long size = this.Size; long sz = this.Size;
int bufferSize = 1024*1024; int bufferSize = 1024*1024;
if (size >= 0) if (sz >= 0)
{ {
bufferSize = (int)(size/10); bufferSize = (int)(sz/10);
if (bufferSize < 1024*1024) if (bufferSize < 1024*1024)
bufferSize = 1024*1024; bufferSize = 1024*1024;
if (bufferSize > 20*1024*1024) if (bufferSize > 20*1024*1024)
bufferSize = 20*1024*1024; bufferSize = 20*1024*1024;
} }
StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, false, bufferSize); SimpleStreamReader reader = new SimpleStreamReader(stream, true, Encoding.UTF8, false, bufferSize);
if (this.ShouldCacheLocally && this.LocalCachePath != null) if (this.ShouldCacheLocally && this.LocalCachePath != null)
{ {
@ -1030,12 +1075,12 @@ namespace Microsoft.Research.JobObjectModel
if (this.RepresentsAFolder) if (this.RepresentsAFolder)
throw new ClusterException("Cannot cache folders"); throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream(); StreamWriter writer = this.CreateTempStream();
return new SharedStreamReader(reader, writer, this.OnClose); return new SharedStreamReader(reader, writer, keepNewlines, this.OnClose);
} }
else else
{ {
// dont cache it // dont cache it
return new SharedStreamReader(reader); return new SharedStreamReader(reader, keepNewlines);
} }
} }
@ -1060,7 +1105,7 @@ namespace Microsoft.Research.JobObjectModel
if (item is CloudBlockBlob) if (item is CloudBlockBlob)
{ {
CloudBlockBlob blob = (CloudBlockBlob)item; CloudBlockBlob blob = (CloudBlockBlob)item;
blocks.Add(blob.Name, blob.Properties.Length); this.blocks.Add(blob.Name, blob.Properties.Length);
} }
else if (item is CloudPageBlob) else if (item is CloudPageBlob)
{ {
@ -1072,8 +1117,8 @@ namespace Microsoft.Research.JobObjectModel
if (metadata.ContainsKey("writePosition")) if (metadata.ContainsKey("writePosition"))
{ {
long sz; long sz;
if (long.TryParse(metadata["writePosition"], out sz)) if (Int64.TryParse(metadata["writePosition"], out sz))
pages.Add(pageBlob.Name, sz); this.pages.Add(pageBlob.Name, sz);
} }
} }
else if (item is CloudBlobDirectory) else if (item is CloudBlobDirectory)
@ -1094,29 +1139,28 @@ namespace Microsoft.Research.JobObjectModel
this.PopulateCache(); this.PopulateCache();
long length = -1; long length = -1;
foreach (var child in this.client.EnumerateDirectory(this.path)) Uri uri = UriFromPath(this.Config as AzureDfsClusterConfiguration, this.path);
Console.WriteLine("AzureDfsFile.GetFileAndFolders({0}) -> {1}", this.path, uri);
foreach (var child in this.client.ExpandFileOrDirectory(uri))
{ {
Regex re = Utilities.RegexFromSearchPattern(match); Regex re = Utilities.RegexFromSearchPattern(match);
if (!re.IsMatch(child)) continue; if (!re.IsMatch(child.AbsolutePath)) continue;
bool isFolder = false; bool isFolder = false;
bool isDfsStream = false; bool isDfsStream = false;
if (blocks.ContainsKey(child)) if (this.blocks.ContainsKey(child.AbsolutePath))
{ {
isDfsStream = true; isDfsStream = true;
length = blocks[child]; length = this.blocks[child.AbsolutePath];
} }
else if (pages.ContainsKey(child)) else if (this.pages.ContainsKey(child.AbsolutePath))
{ {
isDfsStream = false; isDfsStream = false;
length = pages[child]; length = this.pages[child.AbsolutePath];
} }
else if (this.client != null)
// otherwise this information may be incorrect
isFolder = true;
var file = new AzureDfsFile(this.Config, this.Job, this.client, child, this.ShouldCacheLocally, isFolder); var file = new AzureDfsFile(this.Config, this.Job, this.client, PathFromUri(this.Config as AzureDfsClusterConfiguration, child), this.ShouldCacheLocally, isFolder);
file.IsDfsStream = isDfsStream; file.IsDfsStream = isDfsStream;
file.size = length; file.size = length;
@ -1158,30 +1202,24 @@ namespace Microsoft.Research.JobObjectModel
{ {
this.PopulateCache(); this.PopulateCache();
string filepath; string combined = Path.Combine(this.path, filename);
if (this.client != null) Uri filepath = UriFromPath(this.Config as AzureDfsClusterConfiguration, combined);
filepath = this.client.Combine(this.path, filename);
else
filepath = Path.Combine(this.path, filename);
bool isFolder = false; bool isFolder = false;
bool isDfsStream = false; bool isDfsStream = false;
long sz = -1; long sz = -1;
if (blocks.ContainsKey(filepath)) if (this.blocks.ContainsKey(combined))
{ {
isDfsStream = true; isDfsStream = true;
sz = blocks[filepath]; sz = this.blocks[filepath.AbsolutePath];
} }
else if (pages.ContainsKey(filepath)) else if (this.pages.ContainsKey(combined))
{ {
isDfsStream = false; isDfsStream = false;
sz = pages[filepath]; sz = this.pages[filepath.AbsolutePath];
} }
else if (this.client != null)
// if the client is null the information may be incorrect
isFolder = true;
var file = new AzureDfsFile(this.Config, this.Job, this.client, filepath, this.ShouldCacheLocally, isFolder); var file = new AzureDfsFile(this.Config, this.Job, this.client, combined, this.ShouldCacheLocally, isFolder);
file.IsDfsStream = isDfsStream; file.IsDfsStream = isDfsStream;
file.size = sz; file.size = sz;
return file; return file;
@ -1213,4 +1251,232 @@ namespace Microsoft.Research.JobObjectModel
return this.path; return this.path;
} }
} }
/// <summary>
/// A file residing on HDFS (accessed either using HDFS or WebHdfs).
/// </summary>
public class DfsFile : CachedClusterResidentObject
{
private Uri baseUri;
private Uri uri;
private HdfsClientBase client;
/// <summary>
/// A file with the specified path.
/// </summary>
/// <param name="path">Path to the file.</param>
/// <param name="client">Azure client.</param>
/// <param name="config">Cluster configuration.</param>
/// <param name="job">Job accessing this file.</param>
/// <param name="jobFolderUri">Uri to base folder.</param>
/// <param name="isFolder">If true this must be a folder.</param>
/// <param name="canCache">True if the file can be cached (it is immutable for sure).</param>
public DfsFile(ClusterConfiguration config, Uri jobFolderUri, DryadLinqJobSummary job, HdfsClientBase client, string path, bool canCache, bool isFolder)
: base(config, job)
{
this.client = client;
this.Exception = null;
this.baseUri = jobFolderUri;
this.uri = UriFromPath(jobFolderUri, path);
this.ShouldCacheLocally = canCache;
this.RepresentsAFolder = isFolder;
this.size = -1;
Console.WriteLine("DfsFile Uri={0}", this.uri);
if (!string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
this.LocalCachePath = Path.Combine(CachedClusterResidentObject.CacheDirectory, PathFromUri(this.baseUri, this.uri));
}
/// <summary>
/// Create a File URI from a file path.
/// </summary>
/// <param name="baseUri">Base URI point to jobs folder.</param>
/// <param name="path">Path to file.</param>
/// <returns>The File uri.</returns>
public static Uri UriFromPath(Uri baseUri, string path)
{
UriBuilder builder = new UriBuilder(baseUri);
builder.Path = builder.Path.TrimEnd('/') + "/" + path.TrimStart('/');
return builder.Uri;
}
/// <summary>
/// Create a path from a URI.
/// </summary>
/// <param name="uri">URI of an HDFS file.</param>
/// <param name="baseUri">Base URI (should be a prefix of the other uri).</param>
/// <returns>The path to the file.</returns>
public static string PathFromUri(Uri baseUri, Uri uri)
{
string path = uri.PathAndQuery;
if (path.StartsWith(baseUri.PathAndQuery))
path = path.Substring(baseUri.PathAndQuery.Length);
path = path.Trim('/');
return path;
}
private DfsFile(ClusterConfiguration config, DryadLinqJobSummary job, Exception ex)
: base(config, job)
{
this.Exception = ex;
}
/// <summary>
/// True if the object is a folder.
/// </summary>
public override bool RepresentsAFolder
{
get;
protected set;
}
/// <summary>
/// Returns a stream that can be used to access the contents of the object, if the object is not a folder.
/// </summary>
/// <returns>A stream that can be used to access the object contents.</returns>
/// <param name="keepNewlines">If true keep the newlines.</param>
public override ISharedStreamReader GetStream(bool keepNewlines)
{
ISharedStreamReader baseStream = base.GetStream(keepNewlines);
if (baseStream != null)
{
// file is cached
Trace.TraceInformation("Reading from local cache {0}", baseStream);
return baseStream;
}
Stream stream = this.client.GetDfsStreamReader(this.uri);
long sz = this.Size;
int bufferSize = 1024 * 1024;
if (sz >= 0)
{
bufferSize = (int)(sz / 10);
if (bufferSize < 1024 * 1024)
bufferSize = 1024 * 1024;
if (bufferSize > 20 * 1024 * 1024)
bufferSize = 20 * 1024 * 1024;
}
SimpleStreamReader reader = new SimpleStreamReader(stream, true, Encoding.UTF8, false, bufferSize);
if (this.ShouldCacheLocally && this.LocalCachePath != null)
{
// cache it
if (this.RepresentsAFolder)
throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream();
return new SharedStreamReader(reader, writer, keepNewlines, this.OnClose);
}
else
{
// dont cache it
return new SharedStreamReader(reader, keepNewlines);
}
}
/// <summary>
/// If the current object is a folder, it returns the contained objects.
/// </summary>
/// <returns>An iterator over all contained objects that match the specified string.</returns>
/// <param name="match">A shell expression (similar to the argument of Directory.GetFiles()).</param>
public override IEnumerable<IClusterResidentObject> GetFilesAndFolders(string match)
{
HashSet<Uri> folders = new HashSet<Uri>();
foreach (var child in this.client.EnumerateSubdirectories(this.uri))
{
folders.Add(child);
}
Regex re = Utilities.RegexFromSearchPattern(match);
foreach (var child in this.client.ExpandFileOrDirectory(this.uri))
{
if (!re.IsMatch(PathFromUri(this.baseUri, child))) continue;
bool isFolder = folders.Contains(child);
var file = new DfsFile(this.Config, this.baseUri, this.Job, this.client, PathFromUri(this.baseUri, child), this.ShouldCacheLocally, isFolder);
long length;
long time;
this.client.GetFileStatus(child, out time, out length);
file.size = length;
file.CreationTime = TimeFromLong(time);
yield return file;
}
}
private long size;
/// <summary>
/// Size of the object in bytes (if not a folder). The size can be -1 when it is unknown.
/// </summary>
public override long Size
{
get { return this.size; }
}
/// <summary>
/// Short name of the object.
/// </summary>
public override string Name
{
get { return Path.GetFileName(PathFromUri(this.baseUri, this.uri)); }
}
private static DateTime origin = new DateTime(1970, 1, 1);
/// <summary>
/// Convert file status time into a DateTime.
/// </summary>
/// <param name="fileTime">File time obtained from client.</param>
/// <returns>A DateTime object.</returns>
public static DateTime TimeFromLong(long fileTime)
{
return origin + TimeSpan.FromMilliseconds(fileTime);
}
/// <summary>
/// Date when object was created.
/// </summary>
public override DateTime CreationTime
{
get { return DateTime.Now; }
}
/// <summary>
/// For a folder object, returns the contained file with the specified name.
/// </summary>
/// <param name="filename">File name within the folder.</param>
/// <returns>The file within the folder.</returns>
public override IClusterResidentObject GetFile(string filename)
{
var matchingFiles = this.GetFilesAndFolders(filename).ToList();
if (matchingFiles.Count == 1)
return matchingFiles[0];
return new DfsFile(this.Config, this.Job, new Exception("Ambiguous name " + filename + " in " + this));
}
/// <summary>
/// For a folder object, returns the contained folder with the specified name.
/// </summary>
/// <param name="foldername">Folder name within the folder.</param>
/// <returns>The subfolder within the folder.</returns>
public override IClusterResidentObject GetFolder(string foldername)
{
var file = this.GetFile(foldername);
if (!file.RepresentsAFolder) throw new InvalidOperationException(foldername + " is not a folder");
return file;
}
/// <summary>
/// Returns a string that represents the current object.
/// </summary>
/// <returns>
/// A string that represents the current object.
/// </returns>
/// <filterpriority>2</filterpriority>
public override string ToString()
{
return this.uri.ToString();
}
}
} }

45
JobBrowser/JOM/jobinfo.cs
View File

@ -19,8 +19,6 @@ limitations under the License.
*/ */
#undef USE_LINQ_TO_DRYAD
#undef USE_HPC
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
@ -660,6 +658,13 @@ namespace Microsoft.Research.JobObjectModel
return 0; return 0;
} }
/// <summary>
/// Sometimes the StreamReader can read a partial line only.
/// We cache here the previously read line and if it looks like the current line is just a fragment, then we concatentate it
/// to the previous line. The line is a fragment if it does not start with 'logtimelocal'.
/// </summary>
private string previousLine = "";
/// <summary> /// <summary>
/// New JM stdout parsing code, for YARN-based DryadLINQ. /// New JM stdout parsing code, for YARN-based DryadLINQ.
/// </summary> /// </summary>
@ -668,6 +673,17 @@ namespace Microsoft.Research.JobObjectModel
private bool ParseStdoutLineNew(string line) private bool ParseStdoutLineNew(string line)
{ {
if (string.IsNullOrWhiteSpace(line)) return true; if (string.IsNullOrWhiteSpace(line)) return true;
// The line should start with logtimelocal, otherwise it's probably a fragment of the previous line
this.previousLine += line;
if (!(line.EndsWith("\r") || line.EndsWith("\n")))
{
// line is incomplete, return now, parse later
return true;
}
line = this.previousLine.Trim();
this.previousLine = "";
Dictionary<string, string> kvp = Utilities.ParseCSVKVP(line); Dictionary<string, string> kvp = Utilities.ParseCSVKVP(line);
if (kvp == null) return false; if (kvp == null) return false;
@ -848,7 +864,7 @@ namespace Microsoft.Research.JobObjectModel
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version); ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
long totalRead = TryGetNumeric(kvp, "totalRead"); long totalRead = TryGetNumeric(kvp, "totalRead");
long tempRead = TryGetNumeric(kvp, "tempRead"); //long tempRead = TryGetNumeric(kvp, "tempRead");
long tempReadInRack = TryGetNumeric(kvp, "tempReadInRack"); long tempReadInRack = TryGetNumeric(kvp, "tempReadInRack");
long tempReadCrossRack = TryGetNumeric(kvp, "tempReadCrossRack"); long tempReadCrossRack = TryGetNumeric(kvp, "tempReadCrossRack");
long localRead = TryGetNumeric(kvp, "localRead"); long localRead = TryGetNumeric(kvp, "localRead");
@ -1456,7 +1472,7 @@ namespace Microsoft.Research.JobObjectModel
manager.Status(message, StatusKind.LongOp); manager.Status(message, StatusKind.LongOp);
if (this.cachedStdoutReader == null) if (this.cachedStdoutReader == null)
this.cachedStdoutReader = file.GetStream(); this.cachedStdoutReader = file.GetStream(true);
if (this.cachedStdoutReader.Exception != null) if (this.cachedStdoutReader.Exception != null)
{ {
manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error); manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
@ -1472,11 +1488,15 @@ namespace Microsoft.Research.JobObjectModel
while (true) while (true)
{ {
manager.Token.ThrowIfCancellationRequested(); manager.Token.ThrowIfCancellationRequested();
int startLine = currentLine;
bool completeLine = true; bool completeLine = true;
try try
{ {
if (this.ClusterConfiguration is DfsClusterConfiguration)
completeLine = this.ParseStdoutLineNew(line); completeLine = this.ParseStdoutLineNew(line);
else
{
this.ParseStdoutLine(line);
}
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -1486,9 +1506,8 @@ namespace Microsoft.Research.JobObjectModel
if (!completeLine) if (!completeLine)
{ {
if (this.cachedStdoutReader.EndOfStream) if (this.cachedStdoutReader.EndOfStream)
{ // no exception, the log may be truncated
throw new Exception("File ended while scanning for closing quote started on line " + startLine); break;
}
string extraline = this.cachedStdoutReader.ReadLine(); string extraline = this.cachedStdoutReader.ReadLine();
line += "\n" + extraline; line += "\n" + extraline;
@ -1599,7 +1618,7 @@ namespace Microsoft.Research.JobObjectModel
{ {
bool success = true; bool success = true;
ISharedStreamReader sr = logfile.GetStream(); ISharedStreamReader sr = logfile.GetStream(false);
if (sr.Exception != null) if (sr.Exception != null)
{ {
statusReporter("Exception while opening file " + logfile + ":" + sr.Exception.Message, StatusKind.Error); statusReporter("Exception while opening file " + logfile + ":" + sr.Exception.Message, StatusKind.Error);
@ -2042,7 +2061,7 @@ namespace Microsoft.Research.JobObjectModel
{ {
DryadJobStaticPlan retval; DryadJobStaticPlan retval;
{ {
retval = new DryadLinqJobStaticPlan(config, file.GetStream()); retval = new DryadLinqJobStaticPlan(config, file.GetStream(false));
} }
retval.ParseQueryPlan(manager); retval.ParseQueryPlan(manager);
return retval; return retval;
@ -2723,7 +2742,7 @@ namespace Microsoft.Research.JobObjectModel
{ {
uri = uri.Substring(0, option); uri = uri.Substring(0, option);
} }
this.LocalPath = uri.Substring(sepindex + 3); this.LocalPath = uri.Substring(sepindex + 3).Trim();
if (uripathprefix != null) { if (uripathprefix != null) {
// Unfortunately the uri is absolute, although it should be relative sometimes. We fix this here. // Unfortunately the uri is absolute, although it should be relative sometimes. We fix this here.
@ -3293,7 +3312,7 @@ namespace Microsoft.Research.JobObjectModel
if (this.InputChannels != null) if (this.InputChannels != null)
// skip discovery // skip discovery
inputs = false; inputs = false;
ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream(); ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream(false);
var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, manager); var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, manager);
if (channels == null) if (channels == null)
{ {
@ -3336,7 +3355,7 @@ namespace Microsoft.Research.JobObjectModel
manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error); manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error);
return false; return false;
} }
ISharedStreamReader sr = files.First().GetStream(); ISharedStreamReader sr = files.First().GetStream(false);
if (sr.Exception != null) if (sr.Exception != null)
{ {
manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error); manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);

38
JobBrowser/JOM/packages.config
View File

@ -1,21 +1,25 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" /> <package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" /> <package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" /> <package id="Microsoft.Bcl.Build" version="1.0.21" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" /> <package id="Microsoft.Hadoop.Client" version="1.3.1.6" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" /> <package id="Microsoft.Net.Http" version="2.2.28" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Client" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.WebHdfs" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.3.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.1.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.3.1.6" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Management.Storage" version="3.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" /> <package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" /> <package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" /> <package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
</packages> </packages>

2
JobBrowser/JOM/storage.cs
View File

@ -19,8 +19,6 @@ limitations under the License.
*/ */
#undef USE_DSC
#undef USE_TIDYFS
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;

34
JobBrowser/JobBrowser/ClusterBrowser.cs
View File

@ -19,6 +19,7 @@ limitations under the License.
*/ */
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
@ -142,6 +143,8 @@ namespace Microsoft.Research.DryadAnalysis
ClusterConfiguration config = ClusterConfiguration.KnownClusterByName(clusterName); ClusterConfiguration config = ClusterConfiguration.KnownClusterByName(clusterName);
this.Configuration = config; this.Configuration = config;
this.diagnoseToolStripMenuItem.Visible = config.SupportsDiagnosis;
this.diagnoseToolStripMenuItem1.Visible = config.SupportsDiagnosis;
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -602,12 +605,8 @@ namespace Microsoft.Research.DryadAnalysis
ToolStripItem strip = item.OwnerItem; ToolStripItem strip = item.OwnerItem;
string clus = strip.Text; string clus = strip.Text;
var config = this.EditCluster(clus); var conf = this.EditCluster(clus);
if (config != null) this.ConfigurationChanged(conf);
{
ClusterConfiguration.AddKnownCluster(config);
this.AddClusterNameToMenu(config.Name);
}
} }
void selItem_Click(object sender, EventArgs e) void selItem_Click(object sender, EventArgs e)
@ -620,9 +619,8 @@ namespace Microsoft.Research.DryadAnalysis
this.ClusterSelected(clus); this.ClusterSelected(clus);
} }
void AddNewCluster(object sender, EventArgs e) private void ConfigurationChanged(ClusterConfiguration conf)
{ {
ClusterConfiguration conf = this.EditCluster(null);
if (conf == null) return; if (conf == null) return;
// you cannot have two cache clusters at the same time // you cannot have two cache clusters at the same time
@ -650,6 +648,13 @@ namespace Microsoft.Research.DryadAnalysis
ClusterConfiguration.AddKnownCluster(conf); ClusterConfiguration.AddKnownCluster(conf);
this.AddClusterNameToMenu(conf.Name); this.AddClusterNameToMenu(conf.Name);
this.Status("Added cluster " + conf.Name, StatusKind.OK);
}
void AddNewCluster(object sender, EventArgs e)
{
ClusterConfiguration conf = this.EditCluster(null);
this.ConfigurationChanged(conf);
} }
/// <summary> /// <summary>
@ -661,11 +666,24 @@ namespace Microsoft.Research.DryadAnalysis
{ {
ClusterConfigEditor editor = new ClusterConfigEditor(); ClusterConfigEditor editor = new ClusterConfigEditor();
try
{
if (clusterName != null) if (clusterName != null)
{ {
var config = ClusterConfiguration.KnownClusterByName(clusterName); var config = ClusterConfiguration.KnownClusterByName(clusterName);
editor.SetConfigToEdit(config); editor.SetConfigToEdit(config);
} }
else
{
editor.SetConfigToEdit(null);
}
}
catch (Exception)
{
// This can happen when the cluster serialization has changed
// and we can no longer read the saved properties
editor.SetConfigToEdit(null);
}
DialogResult res = editor.ShowDialog(); DialogResult res = editor.ShowDialog();
if (res == System.Windows.Forms.DialogResult.OK) if (res == System.Windows.Forms.DialogResult.OK)
{ {

133
JobBrowser/JobBrowser/ClusterConfigEditor.Designer.cs generated
View File

@ -49,13 +49,14 @@ namespace Microsoft.Research.JobObjectModel
private void InitializeComponent() private void InitializeComponent()
{ {
this.tableLayoutPanel_buttons = new System.Windows.Forms.TableLayoutPanel(); this.tableLayoutPanel_buttons = new System.Windows.Forms.TableLayoutPanel();
this.buttonOK = new System.Windows.Forms.Button();
this.buttonCancel = new System.Windows.Forms.Button(); this.buttonCancel = new System.Windows.Forms.Button();
this.buttonOK = new System.Windows.Forms.Button();
this.tableLayoutPanel_properties = new System.Windows.Forms.TableLayoutPanel(); this.tableLayoutPanel_properties = new System.Windows.Forms.TableLayoutPanel();
this.label_name = new System.Windows.Forms.Label();
this.textBox_name = new System.Windows.Forms.TextBox(); this.textBox_name = new System.Windows.Forms.TextBox();
this.label1 = new System.Windows.Forms.Label(); this.label_type = new System.Windows.Forms.Label();
this.label2 = new System.Windows.Forms.Label();
this.comboBox_clusterType = new System.Windows.Forms.ComboBox(); this.comboBox_clusterType = new System.Windows.Forms.ComboBox();
this.label_description = new System.Windows.Forms.Label();
this.panel1 = new System.Windows.Forms.Panel(); this.panel1 = new System.Windows.Forms.Panel();
this.tableLayoutPanel_buttons.SuspendLayout(); this.tableLayoutPanel_buttons.SuspendLayout();
this.tableLayoutPanel_properties.SuspendLayout(); this.tableLayoutPanel_properties.SuspendLayout();
@ -69,14 +70,26 @@ namespace Microsoft.Research.JobObjectModel
this.tableLayoutPanel_buttons.Controls.Add(this.buttonCancel, 1, 0); this.tableLayoutPanel_buttons.Controls.Add(this.buttonCancel, 1, 0);
this.tableLayoutPanel_buttons.Controls.Add(this.buttonOK, 0, 0); this.tableLayoutPanel_buttons.Controls.Add(this.buttonOK, 0, 0);
this.tableLayoutPanel_buttons.Dock = System.Windows.Forms.DockStyle.Bottom; this.tableLayoutPanel_buttons.Dock = System.Windows.Forms.DockStyle.Bottom;
this.tableLayoutPanel_buttons.Location = new System.Drawing.Point(0, 143); this.tableLayoutPanel_buttons.Location = new System.Drawing.Point(0, 216);
this.tableLayoutPanel_buttons.Name = "tableLayoutPanel_buttons"; this.tableLayoutPanel_buttons.Name = "tableLayoutPanel_buttons";
this.tableLayoutPanel_buttons.RowCount = 1; this.tableLayoutPanel_buttons.RowCount = 1;
this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100F)); this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100F));
this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 20F)); this.tableLayoutPanel_buttons.RowStyles.Add(new System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 33F));
this.tableLayoutPanel_buttons.Size = new System.Drawing.Size(381, 33); this.tableLayoutPanel_buttons.Size = new System.Drawing.Size(389, 33);
this.tableLayoutPanel_buttons.TabIndex = 0; this.tableLayoutPanel_buttons.TabIndex = 0;
// //
// buttonCancel
//
this.buttonCancel.DialogResult = System.Windows.Forms.DialogResult.Cancel;
this.buttonCancel.Dock = System.Windows.Forms.DockStyle.Fill;
this.buttonCancel.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.buttonCancel.Location = new System.Drawing.Point(197, 3);
this.buttonCancel.Name = "buttonCancel";
this.buttonCancel.Size = new System.Drawing.Size(189, 27);
this.buttonCancel.TabIndex = 1;
this.buttonCancel.Text = "Cancel";
this.buttonCancel.UseVisualStyleBackColor = true;
//
// buttonOK // buttonOK
// //
this.buttonOK.DialogResult = System.Windows.Forms.DialogResult.OK; this.buttonOK.DialogResult = System.Windows.Forms.DialogResult.OK;
@ -84,101 +97,104 @@ namespace Microsoft.Research.JobObjectModel
this.buttonOK.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); this.buttonOK.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.buttonOK.Location = new System.Drawing.Point(3, 3); this.buttonOK.Location = new System.Drawing.Point(3, 3);
this.buttonOK.Name = "buttonOK"; this.buttonOK.Name = "buttonOK";
this.buttonOK.Size = new System.Drawing.Size(184, 27); this.buttonOK.Size = new System.Drawing.Size(188, 27);
this.buttonOK.TabIndex = 0; this.buttonOK.TabIndex = 0;
this.buttonOK.Text = "OK"; this.buttonOK.Text = "OK";
this.buttonOK.UseVisualStyleBackColor = true; this.buttonOK.UseVisualStyleBackColor = true;
// //
// buttonCancel
//
this.buttonCancel.DialogResult = System.Windows.Forms.DialogResult.Cancel;
this.buttonCancel.Dock = System.Windows.Forms.DockStyle.Fill;
this.buttonCancel.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.buttonCancel.Location = new System.Drawing.Point(193, 3);
this.buttonCancel.Name = "buttonCancel";
this.buttonCancel.Size = new System.Drawing.Size(185, 27);
this.buttonCancel.TabIndex = 1;
this.buttonCancel.Text = "Cancel";
this.buttonCancel.UseVisualStyleBackColor = true;
//
// tableLayoutPanel_properties // tableLayoutPanel_properties
// //
this.tableLayoutPanel_properties.AutoSize = true; this.tableLayoutPanel_properties.AutoSize = true;
this.tableLayoutPanel_properties.ColumnCount = 2; this.tableLayoutPanel_properties.ColumnCount = 2;
this.tableLayoutPanel_properties.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle()); this.tableLayoutPanel_properties.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle());
this.tableLayoutPanel_properties.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100F)); this.tableLayoutPanel_properties.ColumnStyles.Add(new System.Windows.Forms.ColumnStyle(System.Windows.Forms.SizeType.Percent, 100F));
this.tableLayoutPanel_properties.Controls.Add(this.label2, 0, 0); this.tableLayoutPanel_properties.Controls.Add(this.comboBox_clusterType, 1, 2);
this.tableLayoutPanel_properties.Controls.Add(this.textBox_name, 1, 0); this.tableLayoutPanel_properties.Controls.Add(this.textBox_name, 1, 1);
this.tableLayoutPanel_properties.Controls.Add(this.label1, 0, 1); this.tableLayoutPanel_properties.Controls.Add(this.label_name, 0, 1);
this.tableLayoutPanel_properties.Controls.Add(this.comboBox_clusterType, 1, 1); this.tableLayoutPanel_properties.Controls.Add(this.label_description, 0, 0);
this.tableLayoutPanel_properties.Controls.Add(this.label_type, 0, 2);
this.tableLayoutPanel_properties.Dock = System.Windows.Forms.DockStyle.Top; this.tableLayoutPanel_properties.Dock = System.Windows.Forms.DockStyle.Top;
this.tableLayoutPanel_properties.Location = new System.Drawing.Point(0, 0); this.tableLayoutPanel_properties.Location = new System.Drawing.Point(0, 0);
this.tableLayoutPanel_properties.Name = "tableLayoutPanel_properties"; this.tableLayoutPanel_properties.Name = "tableLayoutPanel_properties";
this.tableLayoutPanel_properties.RowCount = 2; this.tableLayoutPanel_properties.RowCount = 3;
this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle()); this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle());
this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle()); this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle());
this.tableLayoutPanel_properties.Size = new System.Drawing.Size(381, 54); this.tableLayoutPanel_properties.RowStyles.Add(new System.Windows.Forms.RowStyle());
this.tableLayoutPanel_properties.Size = new System.Drawing.Size(389, 66);
this.tableLayoutPanel_properties.TabIndex = 1; this.tableLayoutPanel_properties.TabIndex = 1;
// //
// label_name
//
this.label_name.AutoSize = true;
this.label_name.Dock = System.Windows.Forms.DockStyle.Fill;
this.label_name.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.label_name.Location = new System.Drawing.Point(3, 19);
this.label_name.Name = "label_name";
this.label_name.Size = new System.Drawing.Size(80, 27);
this.label_name.TabIndex = 5;
this.label_name.Text = "Cluster name";
this.label_name.TextAlign = System.Drawing.ContentAlignment.MiddleRight;
//
// textBox_name // textBox_name
// //
this.textBox_name.Dock = System.Windows.Forms.DockStyle.Fill; this.textBox_name.Dock = System.Windows.Forms.DockStyle.Fill;
this.textBox_name.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); this.textBox_name.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.textBox_name.Location = new System.Drawing.Point(89, 3); this.textBox_name.Location = new System.Drawing.Point(89, 22);
this.textBox_name.Name = "textBox_name"; this.textBox_name.Name = "textBox_name";
this.textBox_name.Size = new System.Drawing.Size(289, 21); this.textBox_name.Size = new System.Drawing.Size(297, 21);
this.textBox_name.TabIndex = 3; this.textBox_name.TabIndex = 3;
// //
// label1 // label_type
// //
this.label1.AutoSize = true; this.label_type.AutoSize = true;
this.label1.Dock = System.Windows.Forms.DockStyle.Fill; this.label_type.Dock = System.Windows.Forms.DockStyle.Fill;
this.label1.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); this.label_type.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.label1.Location = new System.Drawing.Point(3, 27); this.label_type.Location = new System.Drawing.Point(3, 46);
this.label1.Name = "label1"; this.label_type.Name = "label_type";
this.label1.Size = new System.Drawing.Size(80, 27); this.label_type.Size = new System.Drawing.Size(80, 20);
this.label1.TabIndex = 4; this.label_type.TabIndex = 4;
this.label1.Text = "Cluster type"; this.label_type.Text = "Cluster type";
this.label1.TextAlign = System.Drawing.ContentAlignment.MiddleRight; this.label_type.TextAlign = System.Drawing.ContentAlignment.MiddleRight;
//
// label2
//
this.label2.AutoSize = true;
this.label2.Dock = System.Windows.Forms.DockStyle.Fill;
this.label2.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.label2.Location = new System.Drawing.Point(3, 0);
this.label2.Name = "label2";
this.label2.Size = new System.Drawing.Size(80, 27);
this.label2.TabIndex = 5;
this.label2.Text = "Cluster name";
this.label2.TextAlign = System.Drawing.ContentAlignment.MiddleRight;
// //
// comboBox_clusterType // comboBox_clusterType
// //
this.comboBox_clusterType.Dock = System.Windows.Forms.DockStyle.Fill; this.comboBox_clusterType.Dock = System.Windows.Forms.DockStyle.Fill;
this.comboBox_clusterType.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0))); this.comboBox_clusterType.Font = new System.Drawing.Font("Microsoft Sans Serif", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.comboBox_clusterType.FormattingEnabled = true; this.comboBox_clusterType.IntegralHeight = false;
this.comboBox_clusterType.Location = new System.Drawing.Point(89, 30); this.comboBox_clusterType.Location = new System.Drawing.Point(89, 49);
this.comboBox_clusterType.Name = "comboBox_clusterType"; this.comboBox_clusterType.Name = "comboBox_clusterType";
this.comboBox_clusterType.Size = new System.Drawing.Size(289, 23); this.comboBox_clusterType.Size = new System.Drawing.Size(297, 23);
this.comboBox_clusterType.TabIndex = 6; this.comboBox_clusterType.TabIndex = 6;
this.comboBox_clusterType.SelectedIndexChanged += new System.EventHandler(this.comboBox_clusterType_SelectedIndexChanged); this.comboBox_clusterType.SelectedIndexChanged += new System.EventHandler(this.comboBox_clusterType_SelectedIndexChanged);
// //
// label_description
//
this.label_description.AutoSize = true;
this.tableLayoutPanel_properties.SetColumnSpan(this.label_description, 2);
this.label_description.Dock = System.Windows.Forms.DockStyle.Fill;
this.label_description.Location = new System.Drawing.Point(3, 3);
this.label_description.Margin = new System.Windows.Forms.Padding(3);
this.label_description.Name = "label_description";
this.label_description.Size = new System.Drawing.Size(383, 13);
this.label_description.TabIndex = 7;
this.label_description.Text = "Description";
this.label_description.TextAlign = System.Drawing.ContentAlignment.MiddleCenter;
//
// panel1 // panel1
// //
this.panel1.Dock = System.Windows.Forms.DockStyle.Fill; this.panel1.Dock = System.Windows.Forms.DockStyle.Fill;
this.panel1.Location = new System.Drawing.Point(0, 54); this.panel1.Location = new System.Drawing.Point(0, 0);
this.panel1.Name = "panel1"; this.panel1.Name = "panel1";
this.panel1.Size = new System.Drawing.Size(381, 89); this.panel1.Size = new System.Drawing.Size(389, 216);
this.panel1.TabIndex = 2; this.panel1.TabIndex = 2;
// //
// ClusterConfigEditor // ClusterConfigEditor
// //
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(381, 176); this.ClientSize = new System.Drawing.Size(389, 249);
this.Controls.Add(this.panel1);
this.Controls.Add(this.tableLayoutPanel_properties); this.Controls.Add(this.tableLayoutPanel_properties);
this.Controls.Add(this.panel1);
this.Controls.Add(this.tableLayoutPanel_buttons); this.Controls.Add(this.tableLayoutPanel_buttons);
this.Name = "ClusterConfigEditor"; this.Name = "ClusterConfigEditor";
this.Text = "Cluster Editor"; this.Text = "Cluster Editor";
@ -196,11 +212,12 @@ namespace Microsoft.Research.JobObjectModel
private System.Windows.Forms.Button buttonCancel; private System.Windows.Forms.Button buttonCancel;
private System.Windows.Forms.Button buttonOK; private System.Windows.Forms.Button buttonOK;
private System.Windows.Forms.TableLayoutPanel tableLayoutPanel_properties; private System.Windows.Forms.TableLayoutPanel tableLayoutPanel_properties;
private System.Windows.Forms.Label label2; private System.Windows.Forms.Label label_name;
private System.Windows.Forms.TextBox textBox_name; private System.Windows.Forms.TextBox textBox_name;
private System.Windows.Forms.Label label1; private System.Windows.Forms.Label label_type;
private System.Windows.Forms.ComboBox comboBox_clusterType; private System.Windows.Forms.ComboBox comboBox_clusterType;
private System.Windows.Forms.Panel panel1; private System.Windows.Forms.Panel panel1;
private System.Windows.Forms.Label label_description;
} }
} }

18
JobBrowser/JobBrowser/ClusterConfigEditor.cs
View File

@ -23,7 +23,6 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Drawing; using System.Drawing;
using System.Windows.Forms; using System.Windows.Forms;
using Microsoft.Research.Tools;
namespace Microsoft.Research.JobObjectModel namespace Microsoft.Research.JobObjectModel
{ {
@ -43,9 +42,9 @@ namespace Microsoft.Research.JobObjectModel
this.addedControls = new List<Control>(); this.addedControls = new List<Control>();
this.propEditor = new Dictionary<string, TextBox>(); this.propEditor = new Dictionary<string, TextBox>();
for (var i = ClusterConfiguration.ClusterType.Unknown+1; i < ClusterConfiguration.ClusterType.MaxUnused; i++) foreach (var type in ClusterConfiguration.Available)
{ {
this.comboBox_clusterType.Items.Add(i.ToString()); this.comboBox_clusterType.Items.Add(type.ToString());
} }
} }
@ -72,6 +71,7 @@ namespace Microsoft.Research.JobObjectModel
this.textBox_name.Text = props.Name; this.textBox_name.Text = props.Name;
this.comboBox_clusterType.SelectedItem = props.Type.ToString(); this.comboBox_clusterType.SelectedItem = props.Type.ToString();
this.propEditor.Clear();
this.AddPropertiesToEdit(props.Properties); this.AddPropertiesToEdit(props.Properties);
} }
@ -84,6 +84,8 @@ namespace Microsoft.Research.JobObjectModel
/// <param name="props">Properties to add controls for.</param> /// <param name="props">Properties to add controls for.</param>
private void AddPropertiesToEdit(List<PropertySetting> props) private void AddPropertiesToEdit(List<PropertySetting> props)
{ {
this.label_description.Text = this.config.Description;
foreach (var prop in props) foreach (var prop in props)
{ {
Label labl = new Label(); Label labl = new Label();
@ -125,7 +127,7 @@ namespace Microsoft.Research.JobObjectModel
{ {
var type = (ClusterConfiguration.ClusterType)Enum.Parse(typeof(ClusterConfiguration.ClusterType), this.comboBox_clusterType.Text); var type = (ClusterConfiguration.ClusterType)Enum.Parse(typeof(ClusterConfiguration.ClusterType), this.comboBox_clusterType.Text);
ClusterConfigurationSerialization ser = new ClusterConfigurationSerialization() ClusterConfigurationSerialization ser = new ClusterConfigurationSerialization
{ {
Name = this.textBox_name.Text, Name = this.textBox_name.Text,
Type = type, Type = type,
@ -150,13 +152,15 @@ namespace Microsoft.Research.JobObjectModel
private void comboBox_clusterType_SelectedIndexChanged(object sender, EventArgs e) private void comboBox_clusterType_SelectedIndexChanged(object sender, EventArgs e)
{ {
// initial setting for an existing configuration if (!this.canChangeType)
if (this.config != null) return; // we are just editing the cluster properties
return;
this.RemoveAddedProperties(); this.RemoveAddedProperties();
var type = (ClusterConfiguration.ClusterType)Enum.Parse(typeof(ClusterConfiguration.ClusterType), this.comboBox_clusterType.Text); var type = (ClusterConfiguration.ClusterType)Enum.Parse(typeof(ClusterConfiguration.ClusterType), this.comboBox_clusterType.Text);
this.config = ClusterConfiguration.CreateConfiguration(type); this.config = ClusterConfiguration.CreateConfiguration(type);
this.AddPropertiesToEdit(this.config.ExtractData().Properties); var serialization = this.config.ExtractData();
this.AddPropertiesToEdit(serialization.Properties);
} }
} }
} }

17
JobBrowser/JobBrowser/Diagnosis.cs
View File

@ -19,6 +19,7 @@ limitations under the License.
*/ */
using System; using System;
using System.IO; using System.IO;
using System.Collections.Generic; using System.Collections.Generic;
@ -413,6 +414,9 @@ namespace Microsoft.Research.DryadAnalysis
bool success = int.TryParse(m.Groups[3].Value, out channelNo); bool success = int.TryParse(m.Groups[3].Value, out channelNo);
if (!success) if (!success)
return null; return null;
try
{
this.Vertex.DiscoverChannels(true, false, true, manager); this.Vertex.DiscoverChannels(true, false, true, manager);
var channels = this.Vertex.InputChannels; var channels = this.Vertex.InputChannels;
if (channels == null) if (channels == null)
@ -424,6 +428,11 @@ namespace Microsoft.Research.DryadAnalysis
} }
return channels[channelNo]; return channels[channelNo];
} }
catch (Exception)
{
return null;
}
}
/// <summary> /// <summary>
/// Detect whether vertex terminates with a stack overflow. /// Detect whether vertex terminates with a stack overflow.
@ -434,7 +443,7 @@ namespace Microsoft.Research.DryadAnalysis
IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary); IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
if (stdout.Exception != null) if (stdout.Exception != null)
return Decision.Dontknow; return Decision.Dontknow;
ISharedStreamReader sr = stdout.GetStream(); ISharedStreamReader sr = stdout.GetStream(false);
while (!sr.EndOfStream) while (!sr.EndOfStream)
{ {
string line = sr.ReadLine(); string line = sr.ReadLine();
@ -458,7 +467,7 @@ namespace Microsoft.Research.DryadAnalysis
IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary); IClusterResidentObject stdout = this.Job.ClusterConfiguration.ProcessStdoutFile(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
if (stdout.Exception != null) if (stdout.Exception != null)
return false; return false;
ISharedStreamReader sr = stdout.GetStream(); ISharedStreamReader sr = stdout.GetStream(false);
// only look for the error in the first 10 lines // only look for the error in the first 10 lines
for (int i = 0; i < 10; i++) for (int i = 0; i < 10; i++)
{ {
@ -496,7 +505,7 @@ namespace Microsoft.Research.DryadAnalysis
{ {
IClusterResidentObject logdir = this.Job.ClusterConfiguration.ProcessWorkDirectory(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary); IClusterResidentObject logdir = this.Job.ClusterConfiguration.ProcessWorkDirectory(this.Vertex.ProcessIdentifier, this.Vertex.VertexIsCompleted, this.Vertex.Machine, this.Job.Summary);
IClusterResidentObject stackTrace = logdir.GetFile(this.stackTraceFile); IClusterResidentObject stackTrace = logdir.GetFile(this.stackTraceFile);
ISharedStreamReader sr = stackTrace.GetStream(); ISharedStreamReader sr = stackTrace.GetStream(false);
if (sr.Exception == null) if (sr.Exception == null)
{ {
@ -676,7 +685,7 @@ namespace Microsoft.Research.DryadAnalysis
return Decision.Dontknow; return Decision.Dontknow;
} }
ISharedStreamReader sr = jmstdout.GetStream(); ISharedStreamReader sr = jmstdout.GetStream(false);
if (sr.Exception != null) if (sr.Exception != null)
{ {
this.Log(DiagnosisMessage.Importance.Tracing, "Could not read job manager standard output", sr.Exception.Message); this.Log(DiagnosisMessage.Importance.Tracing, "Could not read job manager standard output", sr.Exception.Message);

14
JobBrowser/JobBrowser/JobBrowser.cs
View File

@ -393,6 +393,10 @@ namespace Microsoft.Research.DryadAnalysis
this.graphViewer.InsertingEdge = false; this.graphViewer.InsertingEdge = false;
this.staticGraphZoomLevel = 0; this.staticGraphZoomLevel = 0;
this.diagnoseToolStripMenuItem.Visible = job.ClusterConfiguration.SupportsDiagnosis;
this.diagnoseToolStripMenuItem1.Visible = job.ClusterConfiguration.SupportsDiagnosis;
this.diagnoseToolStripMenuItem2.Visible = job.ClusterConfiguration.SupportsDiagnosis;
this.planDrawSurface = new DrawingSurface2D(this.panel_jobSchedule); this.planDrawSurface = new DrawingSurface2D(this.panel_jobSchedule);
this.planDrawSurface.SetMargins(4, 4, 4, 4); this.planDrawSurface.SetMargins(4, 4, 4, 4);
this.panel_jobSchedule.MouseDoubleClick += this.panel_jobSchedule_MouseDoubleClick; this.panel_jobSchedule.MouseDoubleClick += this.panel_jobSchedule_MouseDoubleClick;
@ -1671,7 +1675,7 @@ namespace Microsoft.Research.DryadAnalysis
else else
{ {
manager.Status("Extracting contents of " + path, StatusKind.LongOp); manager.Status("Extracting contents of " + path, StatusKind.LongOp);
ISharedStreamReader sr = path.GetStream(); ISharedStreamReader sr = path.GetStream(false);
if (sr.Exception != null) if (sr.Exception != null)
{ {
error += " [Error " + sr.Exception.Message + "]"; error += " [Error " + sr.Exception.Message + "]";
@ -3010,7 +3014,7 @@ namespace Microsoft.Research.DryadAnalysis
long done = 0; long done = 0;
foreach (var file in files) foreach (var file in files)
{ {
ISharedStreamReader sr = file.GetStream(); ISharedStreamReader sr = file.GetStream(false);
if (sr.Exception != null) if (sr.Exception != null)
{ {
logViewer.Status("Error opening file: " + sr.Exception.Message, StatusKind.Error); logViewer.Status("Error opening file: " + sr.Exception.Message, StatusKind.Error);
@ -3075,7 +3079,7 @@ namespace Microsoft.Research.DryadAnalysis
return false; return false;
} }
ISharedStreamReader sr = stdout.GetStream(); ISharedStreamReader sr = stdout.GetStream(false);
if (sr.Exception != null) if (sr.Exception != null)
{ {
logViewer.Status("Error opening JM stdout: " + sr.Exception.Message, StatusKind.Error); logViewer.Status("Error opening JM stdout: " + sr.Exception.Message, StatusKind.Error);
@ -3310,7 +3314,7 @@ namespace Microsoft.Research.DryadAnalysis
} }
this.Status("Caching " + file.Name, StatusKind.LongOp); this.Status("Caching " + file.Name, StatusKind.LongOp);
ISharedStreamReader reader = file.GetStream(); ISharedStreamReader reader = file.GetStream(false);
// ReSharper disable UnusedVariable // ReSharper disable UnusedVariable
foreach (string line in reader.ReadAllLines()) foreach (string line in reader.ReadAllLines())
// ReSharper restore UnusedVariable // ReSharper restore UnusedVariable
@ -3433,7 +3437,7 @@ namespace Microsoft.Research.DryadAnalysis
continue; continue;
} }
ISharedStreamReader reader = file.GetStream(); ISharedStreamReader reader = file.GetStream(false);
// ReSharper disable once UnusedVariable // ReSharper disable once UnusedVariable
foreach (string line in reader.ReadAllLines()) foreach (string line in reader.ReadAllLines())
{ {

207
JobBrowser/JobBrowser/JobBrowser.csproj
View File

@ -1,6 +1,11 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" /> <Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -10,7 +15,7 @@
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>JobBrowser</RootNamespace> <RootNamespace>JobBrowser</RootNamespace>
<AssemblyName>DryadLinqBrowser</AssemblyName> <AssemblyName>JobBrowser</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<StartupObject> <StartupObject>
@ -18,26 +23,7 @@
<IsWebBootstrapper>false</IsWebBootstrapper> <IsWebBootstrapper>false</IsWebBootstrapper>
<ApplicationIcon>icons\DryadAnalysis.ico</ApplicationIcon> <ApplicationIcon>icons\DryadAnalysis.ico</ApplicationIcon>
<ApplicationManifest>app.manifest</ApplicationManifest> <ApplicationManifest>app.manifest</ApplicationManifest>
<FileUpgradeFlags> <NuGetPackageImportStamp>fc6ab3dd</NuGetPackageImportStamp>
</FileUpgradeFlags>
<UpgradeBackupLocation>
</UpgradeBackupLocation>
<OldToolsVersion>3.5</OldToolsVersion>
<TargetFrameworkProfile />
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
@ -62,130 +48,99 @@
<RunPostBuildEvent>OnBuildSuccess</RunPostBuildEvent> <RunPostBuildEvent>OnBuildSuccess</RunPostBuildEvent>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Reference Include="EnvDTE, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a"> <Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE.dll</HintPath> <HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference> </Reference>
<Reference Include="EnvDTE80, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a"> <Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE80.dll</HintPath> <HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference> </Reference>
<Reference Include="EnvDTE90, Version=9.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a"> <Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE90.dll</HintPath> <HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Hadoop.Client, Version=1.3.1.6, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath> <HintPath>..\packages\Microsoft.Hadoop.Client.1.3.1.6\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.HDInsight.Net.Http.Formatting, Version=1.3.1.6, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath> <HintPath>..\packages\Microsoft.Hadoop.Client.1.3.1.6\lib\net40\Microsoft.HDInsight.Net.Http.Formatting.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.Msagl">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hpc.Scheduler, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\Daphne\Dependences\Microsoft.Hpc.Scheduler.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Hpc.Scheduler.Properties, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\Daphne\Dependences\Microsoft.Hpc.Scheduler.Properties.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Msagl, Version=3.0.0.0, Culture=neutral, PublicKeyToken=7f84f738e1fc245c, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath> <HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Msagl.Drawing, Version=3.0.0.0, Culture=neutral, PublicKeyToken=8a3d7c21d5fa1306, processorArchitecture=MSIL"> <Reference Include="Microsoft.Msagl.Drawing">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath> <HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Msagl.GraphViewerGdi, Version=3.0.0.0, Culture=neutral, PublicKeyToken=fffc27ea4058b3a1, processorArchitecture=MSIL"> <Reference Include="Microsoft.Msagl.GraphViewerGdi">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath> <HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Threading.Tasks"> <Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath> <HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions"> <Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath> <HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop"> <Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath> <HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Common"> <Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath> <HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework"> <Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath> <HintPath>..\packages\Microsoft.WindowsAzure.Common.1.3.0\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath> <HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management"> <Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.3.1.6, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath> <HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.3.1.6\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.3.1.6, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath> <HintPath>..\packages\Microsoft.Hadoop.Client.1.3.1.6\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL"> <Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.3.1.6, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath> <HintPath>..\packages\Microsoft.Hadoop.Client.1.3.1.6\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference> </Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage"> <Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion> <SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath> <HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.3.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference> </Reference>
<Reference Include="System" /> <Reference Include="System" />
<Reference Include="System.Core"> <Reference Include="System.Core" />
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.Linq" /> <Reference Include="System.Data.Linq" />
<Reference Include="System.Net" /> <Reference Include="System.Net" />
<Reference Include="System.Net.Http" /> <Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions"> <Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath> <HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference> </Reference>
<Reference Include="System.Net.Http.Primitives"> <Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath> <HintPath>..\packages\Microsoft.Net.Http.2.2.28\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference> </Reference>
<Reference Include="System.Net.Http.WebRequest" /> <Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Runtime.Serialization" /> <Reference Include="System.Runtime.Serialization" />
<Reference Include="System.ServiceModel" /> <Reference Include="System.ServiceModel" />
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
</Reference>
<Reference Include="System.Web" /> <Reference Include="System.Web" />
<Reference Include="System.Windows.Forms" /> <Reference Include="System.Windows.Forms" />
<Reference Include="System.Xml.Linq"> <Reference Include="System.Xml.Linq" />
<RequiredTargetFramework>3.5</RequiredTargetFramework> <Reference Include="System.Data.DataSetExtensions" />
</Reference>
<Reference Include="System.Data.DataSetExtensions">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data" /> <Reference Include="System.Data" />
<Reference Include="System.Deployment" /> <Reference Include="System.Deployment" />
<Reference Include="System.Drawing" /> <Reference Include="System.Drawing" />
@ -281,52 +236,32 @@
<Content Include="icons\DryadAnalysis.ico" /> <Content Include="icons\DryadAnalysis.ico" />
<Content Include="icons\JobBrowser.ico" /> <Content Include="icons\JobBrowser.ico" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.2.0">
<Visible>False</Visible>
<ProductName>.NET Framework 2.0 %28x86%29</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.0">
<Visible>False</Visible>
<ProductName>.NET Framework 3.0 %28x86%29</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup> <PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup> </PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" /> <Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" /> <Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.targets'))" />
</Target> </Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" /> <Import Project="..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.21\build\Microsoft.Bcl.Build.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.0-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.0-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.0-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.0-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.WebHdfs.0.8.0-beta\build\Microsoft.Research.Peloponnese.WebHdfs.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Client.0.8.0-beta\build\Microsoft.Research.Peloponnese.Client.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild"> <Target Name="BeforeBuild">

1
JobBrowser/JobBrowser/LocalDebuggingAndProfiling.cs
View File

@ -19,6 +19,7 @@ limitations under the License.
*/ */
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;

2
JobBrowser/JobBrowser/LogViewer.cs
View File

@ -127,7 +127,7 @@ namespace Microsoft.Research.DryadAnalysis
this.Initialize(text, basefilename); this.Initialize(text, basefilename);
//ISharedStreamReader sr = new FileSharedStreamReader(filename); //ISharedStreamReader sr = new FileSharedStreamReader(filename);
ISharedStreamReader sr = file.GetStream(); ISharedStreamReader sr = file.GetStream(false);
long lineno = 0; long lineno = 0;
long bytes = 0; long bytes = 0;

10
JobBrowser/JobBrowser/Properties/AssemblyInfo.cs
View File

@ -25,12 +25,12 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following // General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information // set of attributes. Change these attribute values to modify the information
// associated with an assembly. // associated with an assembly.
[assembly: AssemblyTitle("DryadLinqBrowser")] [assembly: AssemblyTitle("JobBrowser")]
[assembly: AssemblyDescription("")] [assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")] [assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("MSIT")] [assembly: AssemblyCompany("MSIT")]
[assembly: AssemblyProduct("DryadLinqBrowser")] [assembly: AssemblyProduct("JobBrowser")]
[assembly: AssemblyCopyright("Copyright © Microsoft")] [assembly: AssemblyCopyright("Copyright © MSIT 2008")]
[assembly: AssemblyTrademark("")] [assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")] [assembly: AssemblyCulture("")]
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers // You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below: // by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")] // [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.1.0")] [assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.1.0")] [assembly: AssemblyFileVersion("1.0.0.0")]

8
JobBrowser/JobBrowser/app.config
View File

@ -15,15 +15,15 @@
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" /> <bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" /> <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
@ -31,7 +31,7 @@
</dependentAssembly> </dependentAssembly>
<dependentAssembly> <dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" /> <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" /> <bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
</dependentAssembly> </dependentAssembly>
</assemblyBinding> </assemblyBinding>
</runtime> </runtime>

38
JobBrowser/JobBrowser/packages.config
View File

@ -1,22 +1,26 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" /> <package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" /> <package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" /> <package id="Microsoft.Bcl.Build" version="1.0.21" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" /> <package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" /> <package id="Microsoft.Hadoop.Client" version="1.3.1.6" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" /> <package id="Microsoft.Net.Http" version="2.2.28" targetFramework="net45" />
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" /> <package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Client" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" /> <package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese.WebHdfs" version="0.8.0-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.3.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.1.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.3.1.6" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" /> <package id="Microsoft.WindowsAzure.Management.Storage" version="3.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" /> <package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" /> <package id="System.Spatial" version="5.6.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" /> <package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
</packages> </packages>

41
JobBrowser/Tools/Tools.csproj
View File

@ -8,36 +8,16 @@
<ProjectGuid>{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}</ProjectGuid> <ProjectGuid>{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}</ProjectGuid>
<OutputType>Library</OutputType> <OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder> <AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Artemis.Tools</RootNamespace> <RootNamespace>Microsoft.Research.Calypso.Tools</RootNamespace>
<AssemblyName>Microsoft.Research.Calypso.Tools</AssemblyName> <AssemblyName>Microsoft.Research.Calypso.Tools</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<FileUpgradeFlags>
</FileUpgradeFlags>
<OldToolsVersion>3.5</OldToolsVersion>
<UpgradeBackupLocation />
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath> <OutputPath>bin\x64\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants> <DefineConstants>DEBUG;TRACE</DefineConstants>
<DocumentationFile>bin\Debug\Microsoft.Hpc.Daphe.Tools.xml</DocumentationFile> <DocumentationFile>bin\Debug\Microsoft.Research.Calypso.Tools.xml</DocumentationFile>
<DebugType>full</DebugType> <DebugType>full</DebugType>
<PlatformTarget>x64</PlatformTarget> <PlatformTarget>x64</PlatformTarget>
<ErrorReport>prompt</ErrorReport> <ErrorReport>prompt</ErrorReport>
@ -71,23 +51,6 @@
<Compile Include="tools.cs" /> <Compile Include="tools.cs" />
<Compile Include="Properties\AssemblyInfo.cs" /> <Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.

627
JobBrowser/Tools/tools.cs
View File

@ -1241,7 +1241,7 @@ namespace Microsoft.Research.Tools
/// <param name="url">Url to reach.</param> /// <param name="url">Url to reach.</param>
/// <returns>A streamreader that returns the loaded web page.</returns> /// <returns>A streamreader that returns the loaded web page.</returns>
/// <param name="credentials">Credentials to use.</param> /// <param name="credentials">Credentials to use.</param>
public static StreamReader Navigate(string url, ICredentials credentials) public static Stream Navigate(string url, ICredentials credentials)
{ {
CookieContainer cookiejar = new CookieContainer(); CookieContainer cookiejar = new CookieContainer();
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url); HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
@ -1258,9 +1258,7 @@ namespace Microsoft.Research.Tools
System.Diagnostics.Trace.Assert(resp.StatusCode == HttpStatusCode.OK); System.Diagnostics.Trace.Assert(resp.StatusCode == HttpStatusCode.OK);
Trace.TraceInformation("Received response"); Trace.TraceInformation("Received response");
// ReSharper disable once AssignNullToNotNullAttribute return resp.GetResponseStream();
StreamReader respReader = new StreamReader(resp.GetResponseStream());
return respReader;
} }
/// <summary> /// <summary>
@ -2887,55 +2885,10 @@ namespace Microsoft.Research.Tools
} }
} }
/// <summary>
/// An abstract class for streaming through files piece by piece.
/// </summary>
public interface IFileStreamer
{
/// <summary>
/// Done reading file.
/// </summary>
void Close();
/// <summary>
/// Read one "line" from the file.
/// </summary>
/// <returns>The line parsed into pieces.</returns>
string[] ReadLine();
/// <summary>
/// Read the file header.
/// </summary>
/// <returns>The line parsed into pieces.</returns>
string[] ReadHeader();
/// <summary>
/// Write the file header.
/// </summary>
/// <param name="header">Header to write.</param>
void WriteHeader(IEnumerable<string> header);
/// <summary>
/// Write a line in the file.
/// </summary>
/// <param name="line">Line to write.</param>
void WriteLine(IEnumerable<string> line);
/// <summary>
/// Start the stream from the beginning.
/// </summary>
void Reset();
/// <summary>
/// The whole contents of the file.
/// </summary>
/// <returns>The complete file contents.</returns>
IEnumerable<string[]> ReadFile();
/// <summary>
/// Line being parsed.
/// </summary>
long CurrentLineNumber { get; }
}
/// <summary> /// <summary>
/// Base implementation of file streamer. /// Base implementation of file streamer.
/// </summary> /// </summary>
public abstract class BaseFileStreamer : IFileStreamer public abstract class BaseFileStreamer
{ {
/// <summary> /// <summary>
/// True if the file is expected to contain a header. /// True if the file is expected to contain a header.
@ -2984,7 +2937,8 @@ namespace Microsoft.Research.Tools
/// <param name="filename">File to access.</param> /// <param name="filename">File to access.</param>
/// <param name="mode">File mode.</param> /// <param name="mode">File mode.</param>
/// <param name="statusReporter">Delegate used to report errors.</param> /// <param name="statusReporter">Delegate used to report errors.</param>
protected BaseFileStreamer(string filename, FileMode mode, StatusReporter statusReporter) /// <param name="keepNewlines">If true keep newlines.</param>
protected BaseFileStreamer(string filename, FileMode mode, bool keepNewlines, StatusReporter statusReporter)
{ {
this.reader = null; this.reader = null;
this.writer = null; this.writer = null;
@ -2992,19 +2946,19 @@ namespace Microsoft.Research.Tools
this.mode = mode; this.mode = mode;
this.statusReporter = statusReporter; this.statusReporter = statusReporter;
// ReSharper disable once DoNotCallOverridableMethodsInConstructor // ReSharper disable once DoNotCallOverridableMethodsInConstructor
this.Reset(); this.Reset(keepNewlines);
} }
/// <summary> /// <summary>
/// Go to the beginning. /// Go to the beginning.
/// </summary> /// </summary>
public virtual void Reset() public virtual void Reset(bool keepNewlines)
{ {
this.Close(); this.Close();
switch (this.mode) switch (this.mode)
{ {
case FileMode.Open: case FileMode.Open:
this.reader = new FileSharedStreamReader(filename); this.reader = new FileSharedStreamReader(filename, keepNewlines);
if (this.reader.Exception != null) if (this.reader.Exception != null)
throw this.reader.Exception; throw this.reader.Exception;
this.writer = null; this.writer = null;
@ -3150,7 +3104,7 @@ namespace Microsoft.Research.Tools
/// <param name="mode">Mode of access.</param> /// <param name="mode">Mode of access.</param>
/// <param name="statusReporter">Delegate used to report errors.</param> /// <param name="statusReporter">Delegate used to report errors.</param>
public KVPFileStreamer(string filename, FileMode mode, StatusReporter statusReporter) public KVPFileStreamer(string filename, FileMode mode, StatusReporter statusReporter)
: base(filename, mode, statusReporter) : base(filename, mode, false, statusReporter)
{ {
} }
@ -3213,7 +3167,7 @@ namespace Microsoft.Research.Tools
/// <param name="hasHeader">True if the file contains a header.</param> /// <param name="hasHeader">True if the file contains a header.</param>
/// <param name="statusReporter">Delegate used to report errors.</param> /// <param name="statusReporter">Delegate used to report errors.</param>
public CSVFileStreamer(string filename, FileMode mode, bool hasHeader, StatusReporter statusReporter) public CSVFileStreamer(string filename, FileMode mode, bool hasHeader, StatusReporter statusReporter)
: base(filename, mode, statusReporter) : base(filename, mode, false, statusReporter)
{ {
this.HasHeader = hasHeader; this.HasHeader = hasHeader;
this.header = null; this.header = null;
@ -3580,6 +3534,10 @@ namespace Microsoft.Research.Tools
/// Delegate to call when stream is closed if caching. /// Delegate to call when stream is closed if caching.
/// </summary> /// </summary>
Action onClose; Action onClose;
/// <summary>
/// If true the reader keeps all newlines.
/// </summary>
private bool readerKeepsNewlines;
/// <summary> /// <summary>
/// A shared stream reader representing an exception. /// A shared stream reader representing an exception.
@ -3594,16 +3552,19 @@ namespace Microsoft.Research.Tools
/// <summary> /// <summary>
/// Actual stream where the data is being read from. /// Actual stream where the data is being read from.
/// </summary> /// </summary>
protected StreamReader actualReader; protected ISimpleStreamReader actualReader;
/// <summary> /// <summary>
/// Create a stream reader for the specified stream. /// Create a stream reader for the specified stream.
/// </summary> /// </summary>
/// <param name="reader">Stream to read.</param> /// <param name="reader">Stream to read.</param>
public SharedStreamReader(StreamReader reader) /// <param name="keepNewlines">Keep newlines in the input stream.</param>
public SharedStreamReader(ISimpleStreamReader reader, bool keepNewlines)
{ {
this.actualReader = reader; this.actualReader = reader;
this.cacheWriter = null; this.cacheWriter = null;
this.readerKeepsNewlines = keepNewlines;
this.readerKeepsNewlines = false;
} }
/// <summary> /// <summary>
@ -3612,8 +3573,10 @@ namespace Microsoft.Research.Tools
/// <param name="reader">Stream to read.</param> /// <param name="reader">Stream to read.</param>
/// <param name="cacheWriter">Use this stream to copy the file to a cache.</param> /// <param name="cacheWriter">Use this stream to copy the file to a cache.</param>
/// <param name="onClose">Delegate to call when stream is completely read.</param> /// <param name="onClose">Delegate to call when stream is completely read.</param>
public SharedStreamReader(StreamReader reader, StreamWriter cacheWriter, Action onClose) /// <param name="readerKeepsNewlines">If true the reader keeps all newlines.</param>
public SharedStreamReader(ISimpleStreamReader reader, StreamWriter cacheWriter, bool readerKeepsNewlines, Action onClose)
{ {
this.readerKeepsNewlines = readerKeepsNewlines;
this.actualReader = reader; this.actualReader = reader;
this.cacheWriter = cacheWriter; this.cacheWriter = cacheWriter;
this.onClose = onClose; this.onClose = onClose;
@ -3624,8 +3587,10 @@ namespace Microsoft.Research.Tools
/// </summary> /// </summary>
/// <param name="cw">Stream used to write to the cache.</param> /// <param name="cw">Stream used to write to the cache.</param>
/// <param name="onCl">Action to invoke on close.</param> /// <param name="onCl">Action to invoke on close.</param>
public void SetCacheWriter(StreamWriter cw, Action onCl) /// <param name="keepNewlines">If true keep newlines.</param>
public void SetCacheWriter(StreamWriter cw, bool keepNewlines, Action onCl)
{ {
this.readerKeepsNewlines = keepNewlines;
this.cacheWriter = cw; this.cacheWriter = cw;
this.onClose = onCl; this.onClose = onCl;
} }
@ -3635,8 +3600,9 @@ namespace Microsoft.Research.Tools
/// </summary> /// </summary>
public SharedStreamReader() public SharedStreamReader()
{ {
this.actualReader = StreamReader.Null; this.actualReader = new WrapperSimpleStreamReader(StreamReader.Null);
this.cacheWriter = null; this.cacheWriter = null;
this.readerKeepsNewlines = false;
} }
/// <summary> /// <summary>
@ -3653,12 +3619,17 @@ namespace Microsoft.Research.Tools
/// <summary> /// <summary>
/// Read one line from the stream. /// Read one line from the stream.
/// </summary> /// </summary>
/// <returns></returns> /// <returns>The read line.</returns>
public override string ReadLine() public override string ReadLine()
{ {
string line = this.actualReader.ReadLine(); string line = this.actualReader.ReadLine();
if (this.cacheWriter != null) if (this.cacheWriter != null)
{
if (this.readerKeepsNewlines)
this.cacheWriter.Write(line);
else
this.cacheWriter.WriteLine(line); this.cacheWriter.WriteLine(line);
}
return line; return line;
} }
@ -3667,7 +3638,6 @@ namespace Microsoft.Research.Tools
/// </summary> /// </summary>
public override void Close() public override void Close()
{ {
if (this.actualReader != StreamReader.Null)
this.actualReader.Close(); this.actualReader.Close();
if (this.cacheWriter != null) if (this.cacheWriter != null)
this.cacheWriter.Close(); this.cacheWriter.Close();
@ -3690,10 +3660,10 @@ namespace Microsoft.Research.Tools
/// <param name="token">Can be used to cancel the reading.</param> /// <param name="token">Can be used to cancel the reading.</param>
public override string ReadToEnd(CancellationToken token) public override string ReadToEnd(CancellationToken token)
{ {
string result = this.actualReader.ReadToEnd(); token.ThrowIfCancellationRequested();
string result = this.actualReader.ReadToEnd(token);
if (this.cacheWriter != null) if (this.cacheWriter != null)
{ {
token.ThrowIfCancellationRequested();
this.cacheWriter.Write(result); this.cacheWriter.Write(result);
this.cacheWriter.Close(); this.cacheWriter.Close();
if (this.onClose != null) if (this.onClose != null)
@ -3725,13 +3695,14 @@ namespace Microsoft.Research.Tools
/// Create a stream reader for the specified file. /// Create a stream reader for the specified file.
/// </summary> /// </summary>
/// <param name="file">File to read.</param> /// <param name="file">File to read.</param>
public FileSharedStreamReader(string file) /// <param name="keepNewline">If true keep newlines.</param>
public FileSharedStreamReader(string file, bool keepNewline)
{ {
try try
{ {
this.file = file; this.file = file;
Stream rd = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); Stream rd = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
this.actualReader = new StreamReader(rd); this.actualReader = new SimpleStreamReader(rd, keepNewline);
} }
catch (Exception ex) catch (Exception ex)
@ -3749,13 +3720,14 @@ namespace Microsoft.Research.Tools
/// <param name="file">File to read from.</param> /// <param name="file">File to read from.</param>
/// <param name="cache">Cache here the contents read from the file.</param> /// <param name="cache">Cache here the contents read from the file.</param>
/// <param name="onClose">Action to invoke when file is closed.</param> /// <param name="onClose">Action to invoke when file is closed.</param>
public FileSharedStreamReader(string file, StreamWriter cache, Action onClose) /// <param name="keepNewlines">If true keep newlines.</param>
public FileSharedStreamReader(string file, StreamWriter cache, bool keepNewlines, Action onClose)
{ {
try try
{ {
Stream rd = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); Stream rd = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
this.actualReader = new StreamReader(rd); this.actualReader = new SimpleStreamReader(rd, true);
this.SetCacheWriter(cache, onClose); this.SetCacheWriter(cache, keepNewlines, onClose);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -3788,6 +3760,7 @@ namespace Microsoft.Research.Tools
/// True if we have reached the end of the stream. /// True if we have reached the end of the stream.
/// </summary> /// </summary>
bool endOfStream; bool endOfStream;
private bool keepNewlines;
/// <summary> /// <summary>
/// End of the stream. /// End of the stream.
@ -3810,9 +3783,11 @@ namespace Microsoft.Research.Tools
/// <summary> /// <summary>
/// A string iterator reading from this data. /// A string iterator reading from this data.
/// </summary> /// </summary>
/// <param name="data"></param> /// <param name="data">Strings to read from; each is a "line".</param>
public StringIteratorStreamReader(IEnumerable<string> data) /// <param name="keepNewlines">If true keep the newlines.</param>
public StringIteratorStreamReader(IEnumerable<string> data, bool keepNewlines)
{ {
this.keepNewlines = keepNewlines;
this.contents = data.GetEnumerator(); this.contents = data.GetEnumerator();
this.endOfStream = !this.contents.MoveNext(); this.endOfStream = !this.contents.MoveNext();
} }
@ -3820,7 +3795,7 @@ namespace Microsoft.Research.Tools
/// <summary> /// <summary>
/// Read one line from the stream. /// Read one line from the stream.
/// </summary> /// </summary>
/// <returns></returns> /// <returns>One line from the set of strings.</returns>
public override string ReadLine() public override string ReadLine()
{ {
string line = this.contents.Current; string line = this.contents.Current;
@ -3844,4 +3819,510 @@ namespace Microsoft.Research.Tools
this.contents = null; this.contents = null;
} }
} }
/// <summary>
/// A completely stripped-down StreamReader; only 3 public methods.
/// </summary>
public interface ISimpleStreamReader
: IDisposable
{
/// <summary>
/// Read one line of text.
/// </summary>
/// <returns>The read line.</returns>
string ReadLine();
/// <summary>
/// True if we have reached the end of stream.
/// </summary>
bool EndOfStream { get; }
/// <summary>
/// Read all the data from the stream.
/// </summary>
/// <param name="token">Cancellation token.</param>
/// <returns>All the data in the stream.</returns>
string ReadToEnd(CancellationToken token);
/// <summary>
/// Close the stream.
/// </summary>
void Close();
}
/// <summary>
/// A simple stream reader which wraps a true StreamReader.
/// This class is only here for compatibility purposes, it should be unused.
/// </summary>
public class WrapperSimpleStreamReader : ISimpleStreamReader
{
private StreamReader reader;
/// <summary>
/// Create a Wrapper around a stream reader.
/// </summary>
/// <param name="reader">Stream reader to wrap.</param>
public WrapperSimpleStreamReader(StreamReader reader)
{
this.reader = reader;
}
/// <summary>
/// Read one line of text.
/// </summary>
/// <returns>The read line.</returns>
public string ReadLine()
{
return this.reader.ReadLine();
}
/// <summary>
/// True if we have reached the end of stream.
/// </summary>
public bool EndOfStream
{
get { return this.reader.EndOfStream; }
}
/// <summary>
/// Read all the data from the stream.
/// </summary>
/// <param name="token">Cancellation token.</param>
/// <returns>All the data in the stream.</returns>
public string ReadToEnd(CancellationToken token)
{
token.ThrowIfCancellationRequested();
return this.reader.ReadToEnd();
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
/// <filterpriority>2</filterpriority>
public void Dispose()
{
this.reader.Dispose();
}
/// <summary>
/// Close the stream.
/// </summary>
public void Close()
{
this.reader.Close();
}
}
/// <summary>
/// A simple stream reader. This is almost like a standard StreamReader, but
/// has the option not to strip the end-of-line characters in ReadLine.
/// </summary>
public class SimpleStreamReader : ISimpleStreamReader
{
/// <summary>
/// Stream we are reading from.
/// </summary>
private Stream stream;
/// <summary>
/// If true do not strip the End of line characters.
/// </summary>
private bool keepEndOfLine;
/// <summary>
/// FIFO buffer.
/// </summary>
/// <typeparam name="T">Type of data stored in buffer.</typeparam>
private class Buffer<T>
{
/// <summary>
/// Data in buffer.
/// </summary>
public T[] Data;
/// <summary>
/// Last index of data available in the buffer.
/// </summary>
public int EndOfData;
/// <summary>
/// Current position for reading from buffer.
/// </summary>
public int CurrentPosition;
private const int minBufferSize = 4096;
/// <summary>
/// Allocate a buffer.
/// </summary>
/// <param name="size">Buffer size.</param>
public Buffer(int size)
{
if (size < minBufferSize)
size = minBufferSize;
this.Data = new T[size];
this.EndOfData = 0;
this.CurrentPosition = 0;
}
/// <summary>
/// Actual size of allocated buffer.
/// </summary>
public int Size
{
get
{
return this.Data.Length;
}
}
/// <summary>
/// True if the buffer is empty.
/// </summary>
/// <returns>A boolean indicating whether there is any data in the buffer.</returns>
public bool IsEmpty()
{
return this.CurrentPosition == this.EndOfData;
}
/// <summary>
/// Items available in the buffer.
/// </summary>
public int ItemsAvailable
{
get { return this.EndOfData - this.CurrentPosition; }
}
/// <summary>
/// Value at specified index in buffer.
/// </summary>
/// <param name="index">Index of value in buffer.</param>
/// <returns>The value in the buffer.</returns>
public T this[int index]
{
get
{
if (index < 0 || index > this.EndOfData)
throw new ArgumentOutOfRangeException("index", "must be between 0 and " + this.EndOfData);
return this.Data[index];
}
}
/// <summary>
/// Delete first items from the buffer.
/// </summary>
/// <param name="items">Number of items to delete.</param>
public void DeletePreamble(int items)
{
if (this.EndOfData < items)
throw new ArgumentException("Cannot delete " + items + " from buffer since only " + this.EndOfData + " are present");
Buffer.BlockCopy(this.Data, items, this.Data, 0, this.EndOfData - items);
this.EndOfData -= items;
}
}
/// <summary>
/// Buffer for read data.
/// </summary>
private Buffer<byte> byteBuffer;
/// <summary>
/// Buffer for decoded data.
/// </summary>
private Buffer<char> charBuffer;
private bool checkPreamble;
/// <summary>
/// Files may have a preamble which indicates the data encoding.
/// </summary>
private byte[] preamble;
/// <summary>
/// If true encoding must be detected.
/// </summary>
private bool encodingUnknown;
// ReSharper disable once PrivateFieldCanBeConvertedToLocalVariable
private Encoding dataEncoding;
private Decoder dataDecoder;
/// <summary>
/// Create a SimpleStream reader to read from a stream.
/// </summary>
/// <param name="Stream">Stream to read from.</param>
/// <param name="keepEndOfLine">If true do not strip the end of line characters.</param>
public SimpleStreamReader(Stream Stream, bool keepEndOfLine = false)
: this(Stream, keepEndOfLine, Encoding.UTF8, true)
{
}
/// <summary>
/// Create a SimpleStream reader reading from a stream.
/// </summary>
/// <param name="stream">Stream to read from.</param>
/// <param name="keepEndOfLine">If true do not strip the end of line characters.</param>
/// <param name="bufferSize">Size of buffer to use when reading from the underlying stream.</param>
/// <param name="encoding">Character encoding to use.</param>
/// <param name="detectEncoding">If true detect the encoding.</param>
public SimpleStreamReader(Stream stream, bool keepEndOfLine, Encoding encoding, bool detectEncoding, int bufferSize = 4096)
{
if (stream == null)
throw new ArgumentNullException("stream");
if (!stream.CanRead)
throw new ArgumentException("StreamNotReadable");
if (bufferSize <= 0)
throw new ArgumentOutOfRangeException("bufferSize");
this.dataEncoding = encoding;
this.dataDecoder = this.dataEncoding.GetDecoder();
this.stream = stream;
this.keepEndOfLine = keepEndOfLine;
this.byteBuffer = new Buffer<byte>(bufferSize);
int charSize = encoding.GetMaxCharCount(this.byteBuffer.Size);
this.charBuffer = new Buffer<char>(charSize);
this.encodingUnknown = detectEncoding;
this.preamble = encoding.GetPreamble();
this.checkPreamble = (this.preamble.Length > 0);
}
/// <summary>
/// Read one line of text.
/// </summary>
/// <returns>The read line.</returns>
public string ReadLine()
{
if (this.stream == null)
throw new InvalidOperationException("Reader closed");
if (this.charBuffer.IsEmpty())
{
if (this.ReadAndConvert() == 0) return null;
}
StringBuilder sb = new StringBuilder();
do
{
int i = this.charBuffer.CurrentPosition;
do
{
char ch = this.charBuffer[i];
if (ch == '\r' || ch == '\n')
{
int offset = this.keepEndOfLine ? 1 : 0;
sb.Append(this.charBuffer.Data, this.charBuffer.CurrentPosition, i + offset - this.charBuffer.CurrentPosition);
this.charBuffer.CurrentPosition = i + 1;
if (ch == '\r')
{
// check for \r\n
if (this.charBuffer.CurrentPosition < this.charBuffer.EndOfData || // one more character available
this.ReadAndConvert() > 0) // read next batch
{
if (this.charBuffer[this.charBuffer.CurrentPosition] == '\n')
this.charBuffer.CurrentPosition++;
if (this.keepEndOfLine)
sb.Append('\n');
}
}
return sb.ToString();
}
i++;
} while (i < this.charBuffer.EndOfData);
sb.Append(this.charBuffer.Data, this.charBuffer.CurrentPosition, this.charBuffer.ItemsAvailable);
}
while (this.ReadAndConvert() > 0);
return sb.ToString();
}
/// <summary>
/// True if we have reached the end of stream.
/// </summary>
public bool EndOfStream
{
get
{
if (this.stream == null)
throw new InvalidOperationException("Reader closed");
if (!this.charBuffer.IsEmpty())
return false;
int numRead = this.ReadAndConvert();
return numRead == 0;
}
}
private bool CheckForPreamble()
{
if (this.checkPreamble)
{
int len = (this.byteBuffer.EndOfData >= (this.preamble.Length)) ?
(this.preamble.Length - this.byteBuffer.CurrentPosition) : this.byteBuffer.ItemsAvailable;
for (int i = 0; i < len; i++)
{
if (this.byteBuffer.Data[this.byteBuffer.CurrentPosition + i] != this.preamble[this.byteBuffer.CurrentPosition + i])
{
this.checkPreamble = false;
return false;
}
}
// preamble found
this.byteBuffer.DeletePreamble(this.preamble.Length);
this.byteBuffer.CurrentPosition = 0;
this.checkPreamble = false;
this.encodingUnknown = false;
}
return this.checkPreamble;
}
/// <summary>
/// Read one buffer of bytes, and transfer them to a byte of chars.
/// Return number of chars transferred.
/// </summary>
/// <returns>The number of chars read.</returns>
private int ReadAndConvert()
{
this.charBuffer.EndOfData = this.charBuffer.CurrentPosition = 0;
if (!this.checkPreamble)
this.byteBuffer.EndOfData = 0;
do
{
if (this.checkPreamble)
{
int len = stream.Read(this.byteBuffer.Data, this.byteBuffer.CurrentPosition, this.byteBuffer.Size - this.byteBuffer.CurrentPosition);
if (len == 0)
{
if (this.byteBuffer.EndOfData > 0)
{
this.charBuffer.EndOfData += this.dataDecoder.GetChars(this.byteBuffer.Data, 0, this.byteBuffer.EndOfData, this.charBuffer.Data, this.charBuffer.EndOfData);
this.byteBuffer.CurrentPosition = this.byteBuffer.EndOfData = 0;
}
return this.charBuffer.EndOfData;
}
this.byteBuffer.EndOfData += len;
}
else
{
this.byteBuffer.EndOfData = stream.Read(this.byteBuffer.Data, 0, this.byteBuffer.Size);
if (this.byteBuffer.EndOfData == 0) // EOF
return this.charBuffer.EndOfData;
}
if (this.CheckForPreamble())
continue;
if (this.encodingUnknown && this.byteBuffer.EndOfData >= 2)
this.DetectEncoding();
this.charBuffer.EndOfData += this.dataDecoder.GetChars(this.byteBuffer.Data, 0, this.byteBuffer.EndOfData, this.charBuffer.Data, this.charBuffer.EndOfData);
} while (this.charBuffer.EndOfData == 0);
return this.charBuffer.EndOfData;
}
private void DetectEncoding()
{
// first few bytes in the buffer, starting at 0
int len = this.byteBuffer.EndOfData;
if (len < 2)
return;
this.encodingUnknown = false;
bool changed = false;
if (this.byteBuffer[0] == 0xFE && this.byteBuffer[1] == 0xFF)
{
// Big Endian Unicode
this.dataEncoding = new UnicodeEncoding(true, true);
this.byteBuffer.DeletePreamble(2);
changed = true;
}
else if (this.byteBuffer[0] == 0xFF && this.byteBuffer[1] == 0xFE)
{
if (len < 4 || this.byteBuffer[2] != 0 || this.byteBuffer[3] != 0)
{
this.dataEncoding = new UnicodeEncoding(false, true);
this.byteBuffer.DeletePreamble(2);
changed = true;
}
else
{
this.dataEncoding = new UTF32Encoding(false, true);
this.byteBuffer.DeletePreamble(4);
changed = true;
}
}
else if (len >= 3 && this.byteBuffer[0] == 0xEF && this.byteBuffer[1] == 0xBB && this.byteBuffer[2] == 0xBF)
{
this.dataEncoding = Encoding.UTF8;
this.byteBuffer.DeletePreamble(3);
changed = true;
}
else if (len >= 4 && this.byteBuffer[0] == 0 && this.byteBuffer[1] == 0 &&
this.byteBuffer[2] == 0xFE && this.byteBuffer[3] == 0xFF)
{
this.dataEncoding = new UTF32Encoding(true, true);
this.byteBuffer.DeletePreamble(4);
changed = true;
}
else if (len == 2)
this.encodingUnknown = true;
if (changed)
{
this.dataDecoder = this.dataEncoding.GetDecoder();
int maxCharsPerBuffer = this.dataEncoding.GetMaxCharCount(byteBuffer.Size);
this.charBuffer = new Buffer<char>(maxCharsPerBuffer);
}
}
/// <summary>
/// Read all the data from the stream.
/// </summary>
/// <param name="token">Cancellation token.</param>
/// <returns>All the data in the stream.</returns>
public string ReadToEnd(CancellationToken token)
{
if (this.stream == null)
throw new InvalidOperationException("Stream closed");
StringBuilder builder = new StringBuilder();
do
{
token.ThrowIfCancellationRequested();
builder.Append(this.charBuffer.Data, this.charBuffer.CurrentPosition, this.charBuffer.ItemsAvailable);
this.charBuffer.CurrentPosition = this.charBuffer.EndOfData;
this.ReadAndConvert();
} while (this.charBuffer.EndOfData > 0);
return builder.ToString();
}
/// <summary>
/// Close the stream.
/// </summary>
public void Close()
{
this.Dispose(true);
}
/// <summary>
/// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
/// </summary>
/// <filterpriority>2</filterpriority>
public void Dispose()
{
this.Dispose(false);
}
/// <summary>
/// Internal dispose method.
/// </summary>
/// <param name="disposing"></param>
protected void Dispose(bool disposing)
{
this.stream.Dispose();
this.stream = null;
this.charBuffer = null;
this.byteBuffer = null;
}
}
} }

43
JobBrowser/UsefulForms/UsefulForms.csproj
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup> <PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -12,26 +12,6 @@
<AssemblyName>Microsoft.Research.Calypso.UsefulForms</AssemblyName> <AssemblyName>Microsoft.Research.Calypso.UsefulForms</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment> <FileAlignment>512</FileAlignment>
<FileUpgradeFlags>
</FileUpgradeFlags>
<OldToolsVersion>3.5</OldToolsVersion>
<UpgradeBackupLocation />
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
<UpdateEnabled>false</UpdateEnabled>
<UpdateMode>Foreground</UpdateMode>
<UpdateInterval>7</UpdateInterval>
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
<UpdatePeriodically>false</UpdatePeriodically>
<UpdateRequired>false</UpdateRequired>
<MapFileExtensions>true</MapFileExtensions>
<ApplicationRevision>0</ApplicationRevision>
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'"> <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols> <DebugSymbols>true</DebugSymbols>
@ -56,9 +36,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<Reference Include="System" /> <Reference Include="System" />
<Reference Include="System.Core"> <Reference Include="System.Core" />
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data" /> <Reference Include="System.Data" />
<Reference Include="System.Drawing" /> <Reference Include="System.Drawing" />
<Reference Include="System.Windows.Forms" /> <Reference Include="System.Windows.Forms" />
@ -109,23 +87,6 @@
<Name>Tools</Name> <Name>Tools</Name>
</ProjectReference> </ProjectReference>
</ItemGroup> </ItemGroup>
<ItemGroup>
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
<Install>false</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
<Visible>False</Visible>
<ProductName>.NET Framework 3.5 SP1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
<Visible>False</Visible>
<ProductName>Windows Installer 3.1</ProductName>
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it. <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets. Other similar extension points exist, see Microsoft.Common.targets.

12
LinqToDryad/CodeGenHelper.cs
View File

@ -57,7 +57,8 @@ namespace Microsoft.Research.DryadLinq.Internal
DynamicMethod dm = new DynamicMethod("GetObjField", DynamicMethod dm = new DynamicMethod("GetObjField",
typeof(S), typeof(S),
new Type[] { typeT }, new Type[] { typeT },
typeT); typeT,
true);
ILGenerator ilgen = dm.GetILGenerator(); ILGenerator ilgen = dm.GetILGenerator();
ilgen.Emit(OpCodes.Ldarg_0); ilgen.Emit(OpCodes.Ldarg_0);
ilgen.Emit(OpCodes.Ldfld, finfo); ilgen.Emit(OpCodes.Ldfld, finfo);
@ -83,7 +84,8 @@ namespace Microsoft.Research.DryadLinq.Internal
DynamicMethod dm = new DynamicMethod("SetObjField", DynamicMethod dm = new DynamicMethod("SetObjField",
typeof(void), typeof(void),
new Type[] { typeT, typeof(S) }, new Type[] { typeT, typeof(S) },
typeT); typeT,
true);
ILGenerator ilgen = dm.GetILGenerator(); ILGenerator ilgen = dm.GetILGenerator();
ilgen.Emit(OpCodes.Ldarg_0); ilgen.Emit(OpCodes.Ldarg_0);
ilgen.Emit(OpCodes.Ldarg_1); ilgen.Emit(OpCodes.Ldarg_1);
@ -110,7 +112,8 @@ namespace Microsoft.Research.DryadLinq.Internal
DynamicMethod dm = new DynamicMethod("GetStructField", DynamicMethod dm = new DynamicMethod("GetStructField",
typeof(S), typeof(S),
new Type[] { typeT.MakeByRefType() }, new Type[] { typeT.MakeByRefType() },
typeT); typeT,
true);
ILGenerator ilgen = dm.GetILGenerator(); ILGenerator ilgen = dm.GetILGenerator();
ilgen.Emit(OpCodes.Ldarg_0); ilgen.Emit(OpCodes.Ldarg_0);
ilgen.Emit(OpCodes.Ldfld, finfo); ilgen.Emit(OpCodes.Ldfld, finfo);
@ -136,7 +139,8 @@ namespace Microsoft.Research.DryadLinq.Internal
DynamicMethod dm = new DynamicMethod("SetStructField", DynamicMethod dm = new DynamicMethod("SetStructField",
typeof(void), typeof(void),
new Type[] { typeT.MakeByRefType(), typeof(S) }, new Type[] { typeT.MakeByRefType(), typeof(S) },
typeT); typeT,
true);
ILGenerator ilgen = dm.GetILGenerator(); ILGenerator ilgen = dm.GetILGenerator();
ilgen.Emit(OpCodes.Ldarg_0); ilgen.Emit(OpCodes.Ldarg_0);
ilgen.Emit(OpCodes.Ldarg_1); ilgen.Emit(OpCodes.Ldarg_1);

53
LinqToDryad/Constants.cs
View File

@ -69,20 +69,47 @@ namespace Microsoft.Research.DryadLinq
public const string debugAzure = "DEBUG_AZURE"; public const string debugAzure = "DEBUG_AZURE";
// Recognized values are: OFF, CRITICAL, ERROR, WARN, INFO, VERBOSE // Recognized values are: OFF, CRITICAL, ERROR, WARN, INFO, VERBOSE
public const string TraceLevelEnvVar = "CCP_DRYADTRACELEVEL"; public const string LoggingLevelEnvVar = "DRYAD_LOGGING_LEVEL";
public const string TraceOff = "OFF"; public const string LoggingOff = "OFF";
public const string TraceCritical = "CRITICAL"; public const string LoggingCritical = "CRITICAL";
public const string TraceError = "ERROR"; public const string LoggingError = "ERROR";
public const string TraceWarning = "WARN"; public const string LoggingWarning = "WARN";
public const string TraceInfo = "INFO"; public const string LoggingInfo = "INFO";
public const string TraceVerbose = "VERBOSE"; public const string LoggingVerbose = "VERBOSE";
public const int TraceOffLevel = 0; public const int LoggingOffLevel = 0;
public const int TraceCriticalLevel = 1; public const int LoggingCriticalLevel = 1;
public const int TraceErrorLevel = 3; public const int LoggingErrorLevel = 3;
public const int TraceWarningLevel = 7; public const int LoggingWarningLevel = 7;
public const int TraceInfoLevel = 15; public const int LoggingInfoLevel = 15;
public const int TraceVerboseLevel = 31; public const int LoggingVerboseLevel = 31;
public static string LoggingStringFromLevel(int level)
{
switch (level)
{
case LoggingOffLevel:
return LoggingOff;
case LoggingCriticalLevel:
return LoggingCritical;
case LoggingErrorLevel:
return LoggingError;
case LoggingWarningLevel:
return LoggingWarning;
case LoggingInfoLevel:
return LoggingInfo;
case LoggingVerboseLevel:
return LoggingVerbose;
default:
throw new ApplicationException("Unknown logging level " + level);
}
}
// SchedulerHelper environment variables // SchedulerHelper environment variables
public const string clusterNameEnvVar = "CCP_CLUSTER_NAME"; public const string clusterNameEnvVar = "CCP_CLUSTER_NAME";

2
LinqToDryad/DataPath.cs
View File

@ -34,7 +34,7 @@ namespace Microsoft.Research.DryadLinq
{ {
internal static class DataPath internal static class DataPath
{ {
internal const string TEMPORARY_STREAM_NAME_PREFIX = "DryadLinqTemp/"; internal const string TEMPORARY_STREAM_NAME_PREFIX = "DryadLinqTemp";
internal const string DSC_URI_SCHEME = "hpcdsc"; internal const string DSC_URI_SCHEME = "hpcdsc";
internal const string HDFS_URI_SCHEME = "hdfs"; internal const string HDFS_URI_SCHEME = "hdfs";

152
LinqToDryad/DataProvider.cs
View File

@ -35,7 +35,7 @@ using System.Diagnostics;
using Microsoft.Research.DryadLinq.Internal; using Microsoft.Research.DryadLinq.Internal;
using System.IO.Compression; using System.IO.Compression;
using Microsoft.Research.Peloponnese.Storage; using Microsoft.Research.Peloponnese.Azure;
namespace Microsoft.Research.DryadLinq namespace Microsoft.Research.DryadLinq
{ {
@ -126,12 +126,14 @@ namespace Microsoft.Research.DryadLinq
/// <param name="metaData">The metadata for the collection.</param> /// <param name="metaData">The metadata for the collection.</param>
/// <param name="outputScheme">The compression scheme used to store the collection.</param> /// <param name="outputScheme">The compression scheme used to store the collection.</param>
/// <param name="isTemp">true to only store the collection temporarily with a time lease.</param> /// <param name="isTemp">true to only store the collection temporarily with a time lease.</param>
/// <param name="serializer">A stream-based serializer.</param>
public abstract void Ingress<T>(DryadLinqContext context, public abstract void Ingress<T>(DryadLinqContext context,
IEnumerable<T> source, IEnumerable<T> source,
Uri dataSetUri, Uri dataSetUri,
DryadLinqMetaData metaData, DryadLinqMetaData metaData,
CompressionScheme outputScheme, CompressionScheme outputScheme,
bool isTemp = false); bool isTemp,
Expression<Action<IEnumerable<T>, Stream>> serializer);
/// <summary> /// <summary>
/// Creates an instance of Stream for a dataset at a specified location. This is /// Creates an instance of Stream for a dataset at a specified location. This is
@ -197,28 +199,35 @@ namespace Microsoft.Research.DryadLinq
/// <typeparam name="T">The record type of the dataset.</typeparam> /// <typeparam name="T">The record type of the dataset.</typeparam>
/// <param name="context">An instance of <see cref="DryadLinqContext"/></param> /// <param name="context">An instance of <see cref="DryadLinqContext"/></param>
/// <param name="dataSetUri">The URI of the dataset</param> /// <param name="dataSetUri">The URI of the dataset</param>
/// <param name="deserializer">A stream-based deserializer</param>
/// <returns>A query object representing the specified dataset.</returns> /// <returns>A query object representing the specified dataset.</returns>
internal static DryadLinqQuery<T> GetPartitionedTable<T>(DryadLinqContext context, Uri dataSetUri) internal static DryadLinqQuery<T>
GetPartitionedTable<T>(DryadLinqContext context,
Uri dataSetUri,
Expression<Func<Stream, IEnumerable<T>>> deserializer)
{ {
string scheme = DataPath.GetScheme(dataSetUri); string scheme = DataPath.GetScheme(dataSetUri);
DataProvider dataProvider = DataProvider.GetDataProvider(scheme); DataProvider dataProvider = DataProvider.GetDataProvider(scheme);
dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri); dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri);
return new DryadLinqQuery<T>(context, dataProvider, dataSetUri); return new DryadLinqQuery<T>(context, dataProvider, dataSetUri, deserializer);
} }
/// <summary> /// <summary>
/// Reads a specified dataset. /// Reads the dataset specified by a URI.
/// </summary> /// </summary>
/// <typeparam name="T">The record type of the dataset.</typeparam> /// <typeparam name="T">The record type of the dataset</typeparam>
/// <param name="context">An instance of <see cref="DryadLinqContext"/></param> /// <param name="context">An instance of <see cref="DryadLinqContext"/></param>
/// <param name="dataSetUri">The URI of the dataset.</param> /// <param name="dataSetUri">The URI of the dataset</param>
/// <returns>A sequence of records as IEnumerable{T}.</returns> /// <param name="deserializer">A stream-based deserializer</param>
public static IEnumerable<T> ReadData<T>(DryadLinqContext context, Uri dataSetUri) /// <returns>A sequence of records as IEnumerable{T}</returns>
internal static IEnumerable<T> ReadData<T>(DryadLinqContext context,
Uri dataSetUri,
Expression<Func<Stream, IEnumerable<T>>> deserializer)
{ {
string scheme = DataPath.GetScheme(dataSetUri); string scheme = DataPath.GetScheme(dataSetUri);
DataProvider dataProvider = DataProvider.GetDataProvider(scheme); DataProvider dataProvider = DataProvider.GetDataProvider(scheme);
dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri); dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri);
return new DryadLinqQueryEnumerable<T>(context, dataProvider, dataSetUri); return new DryadLinqQueryEnumerable<T>(context, dataProvider, dataSetUri, deserializer);
} }
/// <summary> /// <summary>
@ -231,19 +240,23 @@ namespace Microsoft.Research.DryadLinq
/// <param name="metaData">The metadata of the data.</param> /// <param name="metaData">The metadata of the data.</param>
/// <param name="outputScheme">The compression scheme.</param> /// <param name="outputScheme">The compression scheme.</param>
/// <param name="isTemp">true if the data is only stored temporarily.</param> /// <param name="isTemp">true if the data is only stored temporarily.</param>
/// <param name="serializer">A stream-based serializer</param>
/// <param name="deserializer">A stream-based deserializer</param>
/// <returns>An instance of IQueryable{T} for the data.</returns> /// <returns>An instance of IQueryable{T} for the data.</returns>
internal static DryadLinqQuery<T> StoreData<T>(DryadLinqContext context, internal static DryadLinqQuery<T> StoreData<T>(DryadLinqContext context,
IEnumerable<T> source, IEnumerable<T> source,
Uri dataSetUri, Uri dataSetUri,
DryadLinqMetaData metaData, DryadLinqMetaData metaData,
CompressionScheme outputScheme, CompressionScheme outputScheme,
bool isTemp = false) bool isTemp,
Expression<Action<IEnumerable<T>, Stream>> serializer,
Expression<Func<Stream, IEnumerable<T>>> deserializer)
{ {
string scheme = DataPath.GetScheme(dataSetUri); string scheme = DataPath.GetScheme(dataSetUri);
DataProvider dataProvider = DataProvider.GetDataProvider(scheme); DataProvider dataProvider = DataProvider.GetDataProvider(scheme);
dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri); dataSetUri = dataProvider.RewriteUri<T>(context, dataSetUri);
dataProvider.Ingress(context, source, dataSetUri, metaData, outputScheme, isTemp); dataProvider.Ingress(context, source, dataSetUri, metaData, outputScheme, isTemp, serializer);
DryadLinqQuery<T> res = DataProvider.GetPartitionedTable<T>(context, dataSetUri); DryadLinqQuery<T> res = DataProvider.GetPartitionedTable<T>(context, dataSetUri, deserializer);
res.CheckAndInitialize(); // must initialize res.CheckAndInitialize(); // must initialize
return res; return res;
} }
@ -285,7 +298,7 @@ namespace Microsoft.Research.DryadLinq
public override Uri GetTemporaryStreamUri(DryadLinqContext context, string path) public override Uri GetTemporaryStreamUri(DryadLinqContext context, string path)
{ {
return context.Cluster.MakeDefaultUri(DataPath.TEMPORARY_STREAM_NAME_PREFIX + path); return context.Cluster.MakeInternalClusterUri("tmp", DataPath.TEMPORARY_STREAM_NAME_PREFIX, path);
} }
public override DryadLinqMetaData GetMetaData(DryadLinqContext context, Uri dataSetUri) public override DryadLinqMetaData GetMetaData(DryadLinqContext context, Uri dataSetUri)
@ -294,11 +307,28 @@ namespace Microsoft.Research.DryadLinq
return null; return null;
} }
public override Uri RewriteUri<T>(DryadLinqContext context, Uri dataSetUri, FileAccess access)
{
UriBuilder builder = new UriBuilder(dataSetUri);
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(builder.Query);
if (access != FileAccess.Write &&
typeof(T) == typeof(Microsoft.Research.DryadLinq.LineRecord))
{
query["seekBoundaries"] = "Microsoft.Research.DryadLinq.LineRecord";
}
builder.Query = query.ToString();
return builder.Uri;
}
public override DryadLinqStreamInfo GetStreamInfo(DryadLinqContext context, Uri dataSetUri) public override DryadLinqStreamInfo GetStreamInfo(DryadLinqContext context, Uri dataSetUri)
{ {
Int32 parCnt = 0; Int32 parCnt = 0;
Int64 size = -1; Int64 size = -1;
context.Cluster.DfsClient.GetContentSummary(dataSetUri.AbsolutePath, ref size, ref parCnt); NameValueCollection query = System.Web.HttpUtility.ParseQueryString(dataSetUri.Query);
bool expandBlocks = (query["seekboundaries"] == "Microsoft.Research.DryadLinq.LineRecord");
context.GetHdfsClient.GetDirectoryContentSummary(dataSetUri, expandBlocks, ref size, ref parCnt);
if (parCnt == 0) if (parCnt == 0)
{ {
throw new DryadLinqException("Got 0 partition count for " + dataSetUri.AbsoluteUri); throw new DryadLinqException("Got 0 partition count for " + dataSetUri.AbsoluteUri);
@ -311,26 +341,36 @@ namespace Microsoft.Research.DryadLinq
Uri dataSetUri, Uri dataSetUri,
DryadLinqMetaData metaData, DryadLinqMetaData metaData,
CompressionScheme outputScheme, CompressionScheme outputScheme,
bool isTemp = false) bool isTemp,
Expression<Action<IEnumerable<T>, Stream>> serializer)
{ {
throw new DryadLinqException("TBA"); DryadLinqFactory<T> factory = (DryadLinqFactory<T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
using (Stream stream = context.GetHdfsClient.GetDfsStreamWriter(dataSetUri))
{
DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream);
DryadLinqRecordWriter<T> writer = factory.MakeWriter(nativeStream);
foreach (T rec in source)
{
writer.WriteRecordSync(rec);
}
writer.Close();
}
} }
public override Stream Egress(DryadLinqContext context, Uri dataSetUri) public override Stream Egress(DryadLinqContext context, Uri dataSetUri)
{ {
throw new DryadLinqException("TBA"); return context.GetHdfsClient.GetDfsDirectoryStreamReader(dataSetUri);
} }
public override void CheckExistence(DryadLinqContext context, Uri dataSetUri, bool deleteIfExists) public override void CheckExistence(DryadLinqContext context, Uri dataSetUri, bool deleteIfExists)
{ {
WebHdfsClient client = new WebHdfsClient(dataSetUri.Host, 8033, 50070); if (context.GetHdfsClient.IsFileExists(dataSetUri))
if (client.IsFileExists(dataSetUri.AbsolutePath))
{ {
if (!deleteIfExists) if (!deleteIfExists)
{ {
throw new DryadLinqException("Can't output to existing HDFS collection " + dataSetUri.AbsoluteUri); throw new DryadLinqException("Can't output to existing HDFS collection " + dataSetUri.AbsoluteUri);
} }
client.DeleteDfsFile(dataSetUri.AbsolutePath); context.GetHdfsClient.DeleteDfsFile(dataSetUri, true);
} }
} }
} }
@ -380,25 +420,25 @@ namespace Microsoft.Research.DryadLinq
Uri dataSetUri, Uri dataSetUri,
DryadLinqMetaData metaData, DryadLinqMetaData metaData,
CompressionScheme compressionScheme, CompressionScheme compressionScheme,
bool isTemp = false) bool isTemp,
Expression<Action<IEnumerable<T>, Stream>> serializer)
{ {
// Write the partition: string fileName = dataSetUri.LocalPath;
string partDir = context.PartitionUncPath; if (!String.IsNullOrEmpty(dataSetUri.Host))
if (partDir == null)
{ {
partDir = Path.GetDirectoryName(dataSetUri.LocalPath); fileName = @"\\" + dataSetUri.Host + fileName;
} }
if (!Path.IsPathRooted(partDir)) // Write the partition:
{ string partDir = Path.GetDirectoryName(fileName);
partDir = Path.Combine("/", partDir);
}
partDir = Path.Combine(partDir, DryadLinqUtil.MakeUniqueName()); partDir = Path.Combine(partDir, DryadLinqUtil.MakeUniqueName());
Directory.CreateDirectory(partDir); Directory.CreateDirectory(partDir);
string partPath = Path.Combine(partDir, "Part"); string uncPath = Path.Combine(partDir, "Part");
string partFilePath = partPath + ".00000000"; string partitionPath = uncPath + ".00000000";
DryadLinqFactory<T> factory = (DryadLinqFactory<T>)DryadLinqCodeGen.GetFactory(context, typeof(T)); DryadLinqFactory<T> factory = (DryadLinqFactory<T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
using (FileStream fstream = new FileStream(partFilePath, FileMode.CreateNew, FileAccess.Write)) using (FileStream fstream = new FileStream(partitionPath, FileMode.CreateNew, FileAccess.Write))
{
if (serializer == null)
{ {
DryadLinqFileBlockStream nativeStream = new DryadLinqFileBlockStream(fstream, compressionScheme); DryadLinqFileBlockStream nativeStream = new DryadLinqFileBlockStream(fstream, compressionScheme);
DryadLinqRecordWriter<T> writer = factory.MakeWriter(nativeStream); DryadLinqRecordWriter<T> writer = factory.MakeWriter(nativeStream);
@ -408,20 +448,30 @@ namespace Microsoft.Research.DryadLinq
} }
writer.Close(); writer.Close();
} }
else
{
Action<IEnumerable<T>, Stream> serializerFunc = serializer.Compile();
serializerFunc(source, fstream);
}
}
// Write the partfile: // Write the partfile:
FileInfo finfo = new FileInfo(partFilePath); long partSize = new FileInfo(partitionPath).Length;
using (StreamWriter writer = File.CreateText(dataSetUri.LocalPath)) using (StreamWriter writer = File.CreateText(fileName))
{ {
writer.WriteLine(partPath); writer.WriteLine(uncPath);
writer.WriteLine("1"); writer.WriteLine("1");
writer.WriteLine("{0},{1},{2}", 0, finfo.Length, Environment.MachineName); writer.WriteLine("{0},{1}", 0, partSize);
} }
} }
public override Stream Egress(DryadLinqContext context, Uri dataSetUri) public override Stream Egress(DryadLinqContext context, Uri dataSetUri)
{ {
string fileName = dataSetUri.LocalPath; string fileName = dataSetUri.LocalPath;
if (!String.IsNullOrEmpty(dataSetUri.Host))
{
fileName = @"\\" + dataSetUri.Host + fileName;
}
var lines = File.ReadAllLines(fileName); var lines = File.ReadAllLines(fileName);
if (lines.Length < 3) if (lines.Length < 3)
{ {
@ -434,6 +484,10 @@ namespace Microsoft.Research.DryadLinq
public override void CheckExistence(DryadLinqContext context, Uri dataSetUri, bool deleteIfExists) public override void CheckExistence(DryadLinqContext context, Uri dataSetUri, bool deleteIfExists)
{ {
string fileName = dataSetUri.LocalPath; string fileName = dataSetUri.LocalPath;
if (!String.IsNullOrEmpty(dataSetUri.Host))
{
fileName = @"\\" + dataSetUri.Host + fileName;
}
if (File.Exists(fileName)) if (File.Exists(fileName))
{ {
if (!deleteIfExists) if (!deleteIfExists)
@ -460,18 +514,17 @@ namespace Microsoft.Research.DryadLinq
private string[] GetPartitionPaths(string[] lines) private string[] GetPartitionPaths(string[] lines)
{ {
bool isLocalPath = lines[0].Contains(':');
string[] filePathArray = new string[lines.Length - 2]; string[] filePathArray = new string[lines.Length - 2];
for (int i = 2; i < lines.Length; i++) for (int i = 2; i < lines.Length; i++)
{ {
int idx = i - 2; int idx = i - 2;
string[] fields = lines[i].Split(','); string[] fields = lines[i].Split(',');
if (fields[2].Contains(':')) if (fields.Length > 2 && fields[2].Contains(':'))
{ {
string[] parts = fields[2].Split(':'); string[] parts = fields[2].Split(':');
filePathArray[idx] = String.Format(@"\\{0}\{1}", parts[0], parts[1]); filePathArray[idx] = String.Format(@"\\{0}\{1}", parts[0], parts[1]);
} }
else if (isLocalPath) else if (Path.IsPathRooted(lines[0]))
{ {
filePathArray[idx] = String.Format("{0}.{1:X8}", lines[0], idx); filePathArray[idx] = String.Format("{0}.{1:X8}", lines[0], idx);
} }
@ -501,13 +554,13 @@ namespace Microsoft.Research.DryadLinq
public override Uri GetTemporaryStreamUri(DryadLinqContext context, string path) public override Uri GetTemporaryStreamUri(DryadLinqContext context, string path)
{ {
return context.Cluster.MakeDefaultUri(DataPath.TEMPORARY_STREAM_NAME_PREFIX + path); return context.Cluster.MakeInternalClusterUri(DataPath.TEMPORARY_STREAM_NAME_PREFIX, path);
} }
public override Uri RewriteUri<T>(DryadLinqContext context, Uri dataSetUri, FileAccess access) public override Uri RewriteUri<T>(DryadLinqContext context, Uri dataSetUri, FileAccess access)
{ {
string account, key, container, blob; string account, key, container, blob;
AzureUtils.FromAzureUri(dataSetUri, out account, out key, out container, out blob); Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dataSetUri, out account, out key, out container, out blob);
UriBuilder builder = new UriBuilder(dataSetUri); UriBuilder builder = new UriBuilder(dataSetUri);
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(builder.Query); NameValueCollection query = System.Web.HttpUtility.ParseQueryString(builder.Query);
@ -563,17 +616,20 @@ namespace Microsoft.Research.DryadLinq
Uri dataSetUri, Uri dataSetUri,
DryadLinqMetaData metaData, DryadLinqMetaData metaData,
CompressionScheme compressionScheme, CompressionScheme compressionScheme,
bool isTemp = false) bool isTemp,
Expression<Action<IEnumerable<T>, Stream>> serializer)
{ {
string account, key, container, blob; string account, key, container, blob;
AzureUtils.FromAzureUri(dataSetUri, out account, out key, out container, out blob); Utils.FromAzureUri(dataSetUri, out account, out key, out container, out blob);
if (compressionScheme != CompressionScheme.None) if (compressionScheme != CompressionScheme.None)
{ {
throw new DryadLinqException("Not implemented: writing to Azure temporary storage with compression enabled"); throw new DryadLinqException("Not implemented: writing to Azure temporary storage with compression enabled");
} }
AzureDfsClient client = new AzureDfsClient(account, key, container); AzureDfsClient client = new AzureDfsClient(account, key, container);
DryadLinqFactory<T> factory = (DryadLinqFactory<T>)DryadLinqCodeGen.GetFactory(context, typeof(T)); DryadLinqFactory<T> factory = (DryadLinqFactory<T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
using (Stream stream = client.GetFileStreamWriterAsync(blob).Result) using (Stream stream = client.GetDfsStreamWriterAsync(dataSetUri).Result)
{
if (serializer == null)
{ {
DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream); DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream);
DryadLinqRecordWriter<T> writer = factory.MakeWriter(nativeStream); DryadLinqRecordWriter<T> writer = factory.MakeWriter(nativeStream);
@ -583,6 +639,12 @@ namespace Microsoft.Research.DryadLinq
} }
writer.Close(); writer.Close();
} }
else
{
Action<IEnumerable<T>, Stream> serializerFunc = serializer.Compile();
serializerFunc(source, stream);
}
}
} }
public override Stream Egress(DryadLinqContext context, Uri dataSetUri) public override Stream Egress(DryadLinqContext context, Uri dataSetUri)

6
LinqToDryad/DryadLinqBinaryReader.cs
View File

@ -84,11 +84,11 @@ namespace Microsoft.Research.DryadLinq
/// </summary> /// </summary>
~DryadLinqBinaryReader() ~DryadLinqBinaryReader()
{ {
if (!this.m_isClosed) // Only release native resoure here
if (this.m_curDataBlockInfo.ItemHandle != IntPtr.Zero)
{ {
this.m_isClosed = true;
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle); this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
this.m_nativeStream.Close(); this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
} }
} }

10
LinqToDryad/DryadLinqBinaryWriter.cs
View File

@ -101,15 +101,11 @@ namespace Microsoft.Research.DryadLinq
/// </summary> /// </summary>
~DryadLinqBinaryWriter() ~DryadLinqBinaryWriter()
{ {
if (!this.m_isClosed) // Only release native resoure here
{ if (this.m_curDataBlockInfo.ItemHandle != IntPtr.Zero)
this.m_isClosed = true;
this.Flush();
if (this.m_curBlockSize > 0)
{ {
this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle); this.m_nativeStream.ReleaseDataBlock(this.m_curDataBlockInfo.ItemHandle);
} this.m_curDataBlockInfo.ItemHandle = IntPtr.Zero;
this.m_nativeStream.Close();
} }
} }

16
LinqToDryad/DryadLinqCodeGen.cs
View File

@ -138,6 +138,7 @@ namespace Microsoft.Research.DryadLinq.Internal
s_BuiltinTypeToSerializerName.Add(typeof(string), "StringDryadLinqSerializer"); s_BuiltinTypeToSerializerName.Add(typeof(string), "StringDryadLinqSerializer");
s_BuiltinTypeToSerializerName.Add(typeof(Guid), "GuidDryadLinqSerializer"); s_BuiltinTypeToSerializerName.Add(typeof(Guid), "GuidDryadLinqSerializer");
s_BuiltinTypeToSerializerName.Add(typeof(SqlDateTime), "SqlDateTimeDryadLinqSerializer"); s_BuiltinTypeToSerializerName.Add(typeof(SqlDateTime), "SqlDateTimeDryadLinqSerializer");
s_BuiltinTypeToSerializerName.Add(typeof(LineRecord), "LineRecordDryadLinqSerializer");
// Initialize the mapping from type to its factory // Initialize the mapping from type to its factory
s_TypeToFactory = new Dictionary<Type, object>(20); s_TypeToFactory = new Dictionary<Type, object>(20);
@ -1901,8 +1902,7 @@ namespace Microsoft.Research.DryadLinq.Internal
CodeExpression arg1 = new CodeArgumentReferenceExpression("args"); CodeExpression arg1 = new CodeArgumentReferenceExpression("args");
CodeExpression arg2 = new CodeArgumentReferenceExpression(VertexParamName); CodeExpression arg2 = new CodeArgumentReferenceExpression(VertexParamName);
CodeExpression CodeExpression denvInitExpr = new CodeObjectCreateExpression("VertexEnv", arg1, arg2);
denvInitExpr = new CodeObjectCreateExpression("VertexEnv", arg1, arg2);
return new CodeVariableDeclarationStatement("VertexEnv", VertexEnvName, denvInitExpr); return new CodeVariableDeclarationStatement("VertexEnv", VertexEnvName, denvInitExpr);
} }
@ -1940,7 +1940,6 @@ namespace Microsoft.Research.DryadLinq.Internal
CodeMethodInvokeExpression debuggerCheckExpr = new CodeMethodInvokeExpression( CodeMethodInvokeExpression debuggerCheckExpr = new CodeMethodInvokeExpression(
new CodeMethodReferenceExpression(new CodeTypeReferenceExpression(HelperClassName), new CodeMethodReferenceExpression(new CodeTypeReferenceExpression(HelperClassName),
DebugHelperMethodName)); DebugHelperMethodName));
vertexMethod.Statements.Add(new CodeExpressionStatement(debuggerCheckExpr)); vertexMethod.Statements.Add(new CodeExpressionStatement(debuggerCheckExpr));
} }
@ -1971,14 +1970,7 @@ namespace Microsoft.Research.DryadLinq.Internal
vertexMethod.Statements.Add(new CodeExpressionStatement(setParamsExpr)); vertexMethod.Statements.Add(new CodeExpressionStatement(setParamsExpr));
} }
// Push the parallel-code settings into DryadLinqVertex // Initialize vertex env:
bool multiThreading = this.m_context.EnableMultiThreadingInVertex;
vertexMethod.Statements.Add(SetVertexParamField("MultiThreading", multiThreading));
vertexMethod.Statements.Add(
new CodeAssignStatement(
new CodeFieldReferenceExpression(DLVTypeExpr, "s_multiThreading"),
new CodePrimitiveExpression(multiThreading)));
vertexMethod.Statements.Add(MakeVertexEnvDecl(node)); vertexMethod.Statements.Add(MakeVertexEnvDecl(node));
Type[] outputTypes = node.OutputTypes; Type[] outputTypes = node.OutputTypes;
@ -2122,7 +2114,7 @@ namespace Microsoft.Research.DryadLinq.Internal
pipeline[pipeline.Length - 1].OutputDataSetInfo.orderByInfo.IsOrdered); pipeline[pipeline.Length - 1].OutputDataSetInfo.orderByInfo.IsOrdered);
CodeExpression applyExpr; CodeExpression applyExpr;
if (this.m_context.EnableMultiThreadingInVertex) if (this.m_context.ThreadsPerWorker > 1)
{ {
applyExpr = new CodeMethodInvokeExpression( applyExpr = new CodeMethodInvokeExpression(
DryadLinqCodeGen.DLVTypeExpr, DryadLinqCodeGen.DLVTypeExpr,

Some files were not shown because too many files have changed in this diff Show More