berkus
/
Dryad
mirror of https://github.com/MicrosoftResearch/Dryad.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Initial v0.1.2 commit

This commit is contained in:
Michael Isard 2014-04-17 16:29:34 -07:00
parent cce4d976b2
commit 5ad103cba7
184 changed files with 8356 additions and 2759 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
ClusterInterface/ClusterInterface.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -51,15 +51,36 @@
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
@ -72,17 +93,29 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<Reference Include="Newtonsoft.Json">
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
@ -105,14 +138,19 @@
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

4
ClusterInterface/Properties/AssemblyInfo.cs
View File

@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.0.0")]
[assembly: AssemblyFileVersion("0.1.0.0")]
[assembly: AssemblyVersion("0.1.2.0")]
[assembly: AssemblyFileVersion("0.1.2.0")]

12
ClusterInterface/packages.config
View File

@ -1,12 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />

60
DryadLinqGraphManager/DryadLinqGraphManager.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -71,15 +71,39 @@
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
@ -92,6 +116,10 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
@ -108,6 +136,17 @@
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
@ -164,14 +203,19 @@
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

4
DryadLinqGraphManager/Properties/AssemblyInfo.cs
View File

@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.0.0")]
[assembly: AssemblyFileVersion("0.1.0.0")]
[assembly: AssemblyVersion("0.1.2.0")]
[assembly: AssemblyFileVersion("0.1.2.0")]

12
DryadLinqGraphManager/packages.config
View File

@ -1,12 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />

270
DryadLinqTests/ApplyAndForkTests.cs Normal file
View File

@ -0,0 +1,270 @@
using Microsoft.Research.DryadLinq;
using Microsoft.Research.Peloponnese.Storage;
using System;
using System.Collections.Generic;
using System.Linq;
namespace DryadLinqTests
{
public class ApplyAndForkTests
{
public static IEnumerable<int> NonHomomorphic_Unary_Func(IEnumerable<int> input)
{
return input;
}
// [DistributiveOverConcat]
public static IEnumerable<int> Homomorphic_Unary_Func(IEnumerable<int> input)
{
return input;
}
public static IEnumerable<int> NonHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
{
return left;
}
// [LeftDistributiveOverConcat]
public static IEnumerable<int> LeftHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
{
return left;
}
// Note: an apply function must only consume each enumerable once, and it must produce an enumerable
// So for a simple pass-through function that does a little work, we must enumerate only once.
// Else we get the error: "An HpcLinq channel can't be read more than once."
// [DistributiveOverConcat]
public static IEnumerable<int> FullHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
{
long cLeft = 0;
foreach (int x in left)
{
cLeft++;
yield return x;
}
long cRight = 0;
foreach (int x in right)
{
cRight++;
yield return x;
}
if (cLeft == 0)
throw new Exception("a node received empty left-data");
if (cRight == 0)
throw new Exception("a node received empty right-data");
}
public static bool NonHomomorphicUnaryApply()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/NonHomomorphicUnaryApply.out";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool HomomorphicUnaryApply()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/HomomorphicUnaryApply.out";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool NonHomomorphicBinaryApply()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/NonHomomorphicBinaryApply.out";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool LeftHomomorphicBinaryApply()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/LeftHomomorphicBinaryApply.out";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool FullHomomorphicBinaryApply_DifferentDataSets()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/FullHomomorphicBinaryApply_DifferentDataSets.out";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> pt2 = simple.Select(x => x.First());
var q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool FullHomomorphicBinaryApply_IdenticalDataSets()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/FullHomomorphicBinaryApply_2.out";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
var jobInfo = q1.ToStore<int>(outFile).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
[Associative(typeof(AssociativeRecursive1))]
public static string IntToStringCSVAggregator(string agg, int next)
{
return agg + "," + next.ToString();
}
public class AssociativeRecursive1 : IAssociative<string>
{
public string Seed()
{
return "";
}
public string RecursiveAccumulate(string first, string second)
{
return first + second;
}
}
public static bool Aggregate_WithCombiner()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
}
}

1087
DryadLinqTests/BasicAPITests.cs Normal file
View File

File diff suppressed because it is too large Load Diff

80
DryadLinqTests/DryadLinqTests.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -63,37 +63,78 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client">
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge">
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils">
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json">
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
@ -106,8 +147,14 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="ApplyAndForkTests.cs" />
<Compile Include="BasicAPITests.cs" />
<Compile Include="GroupByReduceTests.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="RangePartitionAPICoverageTests.cs" />
<Compile Include="Utils.cs" />
<Compile Include="Validate.cs" />
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
@ -137,14 +184,19 @@
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

1003
DryadLinqTests/GroupByReduceTests.cs Normal file
View File

File diff suppressed because it is too large Load Diff

81
DryadLinqTests/Program.cs
View File

@ -21,6 +21,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Threading.Tasks;
using Microsoft.Research.DryadLinq;
@ -32,64 +33,32 @@ namespace DryadLinqTests
{
public static void Main(string[] args)
{
//Test1(args);
//Test2(args);
//Test3(args);
//Test4(args);
//Test5(args);
}
//BasicAPITests.ToStoreThrowsForNonQuery();
//BasicAPITests.ToStoreGetEnumeratorThrows();
//BasicAPITests.GetEnumeratorNonToStoreTerminated();
//BasicAPITests.ToStoreSubmitGetEnumerator();
//BasicAPITests.SubmitNonToStoreTerminated();
//BasicAPITests.MaterializeToStoreTerminated();
//BasicAPITests.MaterializeNonToStoreTerminated();
//BasicAPITests.EnumeratePlainData();
public static void Test1(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var lines = input.Where(x => x.Line.Contains("white"));
var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
result.SubmitAndWait();
}
//BasicAPITests.RepeatSubmit();
//BasicAPITests.RepeatMaterialize();
//BasicAPITests.MaterializeMentionsSameQueryTwice();
//BasicAPITests.QueryOnDataBackedDLQ();
//BasicAPITests.Bug11781_CountandFirstOrDefault();
//BasicAPITests.Bug11782_Aggregate();
//BasicAPITests.Bug11782_LowLevelQueryableManipulation();
//BasicAPITests.Bug11638_LongWhere();
//BasicAPITests.AssumeRangePartition();
//BasicAPITests.Bug11638_LongMethods();
//BasicAPITests.ContextConfigIsReadOnly();
//BasicAPITests.ToggleSpeculativeDuplication();
//BasicAPITests.Bug15068_ConfigResourcesAPI();
//BasicAPITests.Bug_16341_SubmitThrowsForDifferentContexts();
//BasicAPITests.Bug14449_ContextShouldExposeVersionIDs();
//BasicAPITests.Bug_16341_VariousTestsForSubmit();
public static void Test2(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile:///d:/DryadLinqTemp/PartFiles");
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var q1 = input.Where(x => x.Line.Contains("white"));
var q2 = input.Where(x => x.Line.Contains("the"));
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
DryadLinqQueryable.SubmitAndWait(res1, res2);
}
public static void Test3(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile:///d:/DryadLinqTemp/PartFiles");
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
result.SubmitAndWait();
}
public static void Test4(string[] args)
{
DryadLinqContext context = new DryadLinqContext("svc-d2-01");
var input = context.FromStore<LineRecord>("hdfs://svc-d2-01:8033/user/misard/foo.txt");
var lines = input.Where(x => x.Line.Contains("white"));
var result = lines.ToStore("hdfs://svc-d2-01:8033/user/yuanbyu/foo.txt", true);
result.SubmitAndWait();
}
public static void Test5(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1);
Uri dataUri = AzureUtils.ToAzureUri("msrsvc", "I4JPlk0bZ6YWypg+RJamyq0us1b+kCcuoeKlPhfiHTcVW7P4xvuzURvlRShSo1O3UDhcL2LiY4kMaarD+p1lKg==", "test", "testwrite");
IEnumerable<LineRecord> lines = DataProvider.ReadData<LineRecord>(context, dataUri);
foreach (var x in lines)
{
Console.WriteLine(x);
}
}
}
}

4
DryadLinqTests/Properties/AssemblyInfo.cs
View File

@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.0.0")]
[assembly: AssemblyFileVersion("0.1.0.0")]
[assembly: AssemblyVersion("0.1.2.0")]
[assembly: AssemblyFileVersion("0.1.2.0")]

12
DryadLinqTests/RangePartitionAPICoverageTests.cs Normal file
View File

@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace DryadLinqTests
{
public class RangePartitionAPICoverageTests
{
}
}

157
DryadLinqTests/Utils.cs Normal file
View File

@ -0,0 +1,157 @@
using Microsoft.Research.DryadLinq;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace DryadLinqTests
{
public static class Config
{
public static string accountName = @"MyAccountName";
public static string storageKey = @"MyStorageKey";
public static string containerName = @"MyContainerName";
public static string cluster = "MyCcluster";
}
public class DataGenerator
{
public DataGenerator()
{
}
public static IEnumerable<IEnumerable<int>> CreateSimpleFileSets()
{
IEnumerable<IEnumerable<int>> data = new int[][]
{
new[] { 1, 2, 3, 4 },
new[] { 5, 6, 7, 8 },
new[] { 9, 10, 11, 12 },
};
return data;
}
public static IEnumerable<IEnumerable<int>> CreateGroupByReduceDataSet()
{
// we need quite a few elements to ensure the combiner will be activated in Stage#1 groupBy.
// 33 elements per partition should suffice, but 100 per partition is safer.
IEnumerable<IEnumerable<int>> data = new int[][]
{
Enumerable.Range(1,100).ToArray(),
Enumerable.Range(101,100).ToArray(),
};
return data;
}
}
public class Utils
{
public static bool DeleteFile(string accountName, string accountKey, string containerName, string fileName, bool delSubDirs)
{
try
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + accountName + ";AccountKey=" + accountKey);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
container.CreateIfNotExists();
BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
container.SetPermissions(containerPermissions);
if (false == delSubDirs)
{
CloudBlockBlob remoteFile = container.GetBlockBlobReference(fileName);
remoteFile.DeleteIfExists();
}
if (true == delSubDirs)
{
foreach (IListBlobItem item in container.ListBlobs(fileName, true))
{
CloudBlockBlob blob = (CloudBlockBlob)item;
blob.DeleteIfExists();
}
}
}
catch (Exception)
{
return false;
}
return true;
}
public static bool FileExists(string accountName, string accountKey, string containerName, string fileName)
{
try
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + accountName + ";AccountKey=" + accountKey);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
container.CreateIfNotExists();
BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
container.SetPermissions(containerPermissions);
CloudBlockBlob remoteFile = container.GetBlockBlobReference(fileName);
if (!remoteFile.Exists())
return false;
}
catch (Exception)
{
return false;
}
return true;
}
internal static DryadLinqContext MakeBasicConfig(string cluster) //???
{
var context = new DryadLinqContext(cluster);
try
{
context.JobFriendlyName = "DryadLinq_DevUnitTests";
context.CompileForVertexDebugging = true;
context.JobEnvironmentVariables.Add("DummyEnvVar", "hello"); //note: this is consumed by a unit-test.
if (File.Exists("Microsoft.Hpc.Linq.pdb")) // TODO: fix references
{
context.ResourcesToAdd.Add("Microsoft.Hpc.Linq.pdb");
}
if (File.Exists("Microsoft.Hpc.Dsc.Client.pdb")) // TODO: fix references
{
context.ResourcesToAdd.Add("Microsoft.Hpc.Dsc.Client.pdb");
}
// To prevent job from running forever, and blocking other test
context.JobRuntimeLimit = (int)TimeSpan.FromMinutes(30).TotalSeconds;
//config.AllowConcurrentUserDelegatesInSingleProcess = false;
// If we are on Azure, we have to set the nodeGroup to "NodeRole" so that the default of "ComputeNodes" is not used
// This fixes "FromEnumerableTests" on Azure which queries the active node-group.
// Note also, the headnode for an azure deployment defaults to "HPCCluster" (at least from James' script)
int onAzureInt = 0;
string onAzureString = Environment.GetEnvironmentVariable("CCP_SCHEDULERONAZURE");
if (onAzureString != null)
{
int.TryParse(onAzureString, out onAzureInt);
}
if (onAzureInt == 1)
{
context.NodeGroup = "NodeRole";
}
}
catch (DryadLinqException)
{
}
return context;
}
}
}

267
DryadLinqTests/Validate.cs Normal file
View File

@ -0,0 +1,267 @@
///-------------------------------------------------------------------------------------------------
// file: Validate.cs
//
// summary: Implements the validate class
///-------------------------------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
namespace BenchmarkFramework {
///-------------------------------------------------------------------------------------------------
/// <summary> Validation utils </summary>
///-------------------------------------------------------------------------------------------------
public class Validate {
public static void
Check<T>(
IEnumerable<T>[] ss,
IComparer<T> comparer = null,
bool sort = true,
bool verbose = false,
IComparer<T> sortcomparer = null
) {
if(ss.Length == 0) return;
if(comparer == null) {
comparer = Comparer<T>.Default;
if(comparer == null) {
throw new ArgumentNullException("Can't not be null.");
}
}
if(sortcomparer == null)
sortcomparer = comparer;
T[][] aa = new T[ss.Length][];
for(int i = 0; i < aa.Length; i++) {
aa[i] = ss[i].ToArray();
if(sort) Array.Sort(aa[i], sortcomparer);
}
int len = aa[0].Length;
for(int i = 1; i < aa.Length; i++) {
if(aa[i].Length != len) {
throw new Exception("Wrong number of elements.");
}
}
for(int i = 0; i < len; i++) {
T elem = aa[0][i];
for(int j = 1; j < aa.Length; j++) {
if(verbose) {
//TestOutput.WriteLine("Comparing {0} to {1}", elem.ToString(), aa[j][i].ToString());
}
if(comparer.Compare(elem, aa[j][i]) != 0) {
throw new Exception("Elements failed to match: " + elem + " != " + aa[j][i]);
}
}
}
}
//public static void
//GroupCheck<K, V>(
// IEnumerable<IGrouping<K, V>>[] ss,
// IComparer<K> kComparer = null,
// IComparer<V> vComparer = null
// ) {
// if(ss.Length == 0) return;
// if(kComparer == null) {
// kComparer = Comparer<K>.Default;
// if(kComparer == null) {
// throw new ArgumentNullException("Can't not be null.");
// }
// }
// if(vComparer == null) {
// vComparer = Comparer<V>.Default;
// if(vComparer == null) {
// throw new ArgumentNullException("Can't not be null.");
// }
// }
// IGrouping<K, V>[][] aa = new IGrouping<K, V>[ss.Length][];
// for(int i = 0; i < aa.Length; i++) {
// aa[i] = ss[i].ToArray();
// K[] keys = aa[i].Select(x => x.Key).ToArray();
// Array.Sort(keys, aa[i], kComparer);
// }
// int len = aa[0].Length;
// for(int i = 1; i < aa.Length; i++) {
// if(aa[i].Length != len) {
// throw new Exception("Wrong number of elements.");
// }
// }
// for(int i = 0; i < len; i++) {
// IEnumerable<V> elem = aa[0][i];
// for(int j = 1; j < aa.Length; j++) {
// Check(new IEnumerable<V>[] { elem, aa[j][i] }, vComparer);
// }
// }
//}
}
/*
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantDoubleComparer : IComparer<double> {
Double EPSILON;
public TolerantDoubleComparer(Double _epsilon = 0.000001f) {
EPSILON = _epsilon;
}
public int Compare(Double a, Double b) {
Double delta = a - b;
if(Math.Abs(delta) <= EPSILON)
return 0;
return delta < 0.0f ? -1 : 1;
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantFloatComparer : IComparer<float> {
float EPSILON;
public TolerantFloatComparer(float _epsilon = 0.000001f) {
EPSILON = _epsilon;
}
public int Compare(float a, float b) {
float delta = a - b;
if(Math.Abs(delta) <= EPSILON)
return 0;
return delta < 0.0f ? -1 : 1;
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantVectorComparer : IComparer<Vector> {
float EPSILON;
public TolerantVectorComparer(float _epsilon = 0.0001f) {
EPSILON = _epsilon;
}
public int Compare(Vector a, Vector b) {
for(int i = 0; i < a.m_elems.Length; i++) {
float delta = a.m_elems[i] - b.m_elems[i];
if(Math.Abs(delta) > EPSILON)
return delta < 0.0f ? -1 : 1;
}
return 0;
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Interface for epsilon comparable single. </summary>
///
/// <remarks> Crossbac, 1/16/2014. </remarks>
///
/// <typeparam name="T"> Generic type parameter. </typeparam>
///-------------------------------------------------------------------------------------------------
public interface IEpsilonComparableSingle<T> {
int EpsilonCompare(T a, T b, float epsilon);
}
///-------------------------------------------------------------------------------------------------
/// <summary> Interface for epsilon comparable double. </summary>
///
/// <remarks> Crossbac, 1/16/2014. </remarks>
///
/// <typeparam name="T"> Generic type parameter. </typeparam>
///-------------------------------------------------------------------------------------------------
public interface IEpsilonComparableDouble<T> {
int EpsilonCompare(T a, T b, double epsilon);
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class EpsilonComparer<T> : IComparer<T> where T : IEpsilonComparableSingle<T> {
float EPSILON;
public EpsilonComparer(float _epsilon = 0.0001f) {
EPSILON = _epsilon;
}
public int Compare(T a, T b) {
return a.EpsilonCompare(a, b, EPSILON);
}
}
///-------------------------------------------------------------------------------------------------
/// <summary> Tolerant float comparer for images. Floating point differences between
/// GPU and CPU cause the default comparer to fail sometimes even when
/// the result is correct. Use this comparer to introduce some tolerance
/// for this
/// </summary>
///
/// <remarks> Crossbac, 2/19/2013. </remarks>
///-------------------------------------------------------------------------------------------------
public class TolerantImageComparer : IComparer<Image> {
float EPSILON;
public TolerantImageComparer(float _epsilon = 0.0001f) {
EPSILON = _epsilon;
}
public int Compare(Image a, Image b) {
for(int i = 0; i < a.m_elems.Length; i++) {
float delta = a.m_elems[i] - b.m_elems[i];
if(Math.Abs(delta) > EPSILON)
return delta < 0.0f ? -1 : 1;
}
return 0;
}
}
/// <summary>
/// Compare two instances of Pair<int,int>.
/// Implement for concrete type as Pair<T1,T2> cannot straighforwardly implement IComparable.
/// </summary>
///
/// <remarks> jcurrey, 3/11/2013. </remarks>
public class PairIntIntComparer : IComparer<Pair<int, int>> {
public int Compare(Pair<int, int> a, Pair<int, int> b) {
int keyComparison = a.Key.CompareTo(b.Key);
if(keyComparison == 0) {
return a.Value.CompareTo(b.Value);
} else {
return keyComparison;
}
}
}
*/
}

12
DryadLinqTests/packages.config
View File

@ -1,12 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />

10
DryadVertex/VertexHost/system/channel/channel.vcxproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -149,13 +149,13 @@
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
</ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
</Project>

4
DryadVertex/VertexHost/system/channel/packages.config
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
</packages>

10
DryadVertex/VertexHost/system/dprocess/dprocess.vcxproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -123,13 +123,13 @@
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
</ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
</Project>

4
DryadVertex/VertexHost/system/dprocess/packages.config
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
</packages>

4
DryadVertex/VertexHost/system/managedchannel/Properties/AssemblyInfo.cs
View File

@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.0.0")]
[assembly: AssemblyFileVersion("0.1.0.0")]
[assembly: AssemblyVersion("0.1.2.0")]
[assembly: AssemblyFileVersion("0.1.2.0")]

60
DryadVertex/VertexHost/system/managedchannel/managedchannel.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -53,15 +53,39 @@
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
@ -74,6 +98,10 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
@ -85,6 +113,17 @@
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
@ -112,14 +151,19 @@
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

12
DryadVertex/VertexHost/system/managedchannel/packages.config
View File

@ -1,12 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />

10
DryadVertex/VertexHost/vertex/vertexHost/VertexHost.vcxproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -124,13 +124,13 @@
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
</ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
</Project>

4
DryadVertex/VertexHost/vertex/vertexHost/packages.config
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
</packages>

10
GraphManager/GraphManager.vcxproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
@ -223,13 +223,13 @@
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
</ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
</Project>

4
GraphManager/GraphManager.vcxproj.filters
View File

@ -5,10 +5,6 @@
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
</Filter>
<Filter Include="Resource Files">
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx</Extensions>
</Filter>
<Filter Include="Source Files">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>

2
GraphManager/packages.config
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
</packages>

89
JobBrowser/JOM/ClusterConfiguration.cs
View File

@ -19,30 +19,31 @@ limitations under the License.
*/
using Microsoft.Research.Calypso.Tools;
using System.Security.Cryptography.X509Certificates;
using System.Xml.Linq;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Net;
using System.Diagnostics;
using Microsoft.Research.Peloponnese.Storage;
using Microsoft.Research.Tools;
using Microsoft.WindowsAzure.Management.HDInsight;
namespace Microsoft.Research.Calypso.JobObjectModel
namespace Microsoft.Research.JobObjectModel
{
/// <summary>
/// Error during conversation with cluster.
/// </summary>
public sealed class CalypsoClusterException : Exception
public sealed class ClusterException : Exception
{
/// <summary>
/// Create an exception about handling a cluster.
/// </summary>
/// <param name="message">Exception message.</param>
public CalypsoClusterException(string message) : base(message) { }
public ClusterException(string message) : base(message) { }
}
/// <summary>
@ -606,13 +607,22 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// </summary>
public abstract string Initialize();
/// <summary>
/// Enumerate all clusters this user is subscribed to.
/// </summary>
/// <returns>A list of clusters.</returns>
public static IEnumerable<ClusterConfiguration> EnumerateSubscribedClusters()
{
return AzureDfsClusterConfiguration.EnumerateAzureDfsSubscribedClusters();
}
/// <summary>
/// Create serialization data structure for this configuration.
/// </summary>
/// <returns>The corresponding serialization.</returns>
public ClusterConfigurationSerialization ExtractData()
{
ClusterConfigurationSerialization result = new ClusterConfigurationSerialization()
ClusterConfigurationSerialization result = new ClusterConfigurationSerialization
{
Type = this.TypeOfCluster,
Name = this.Name,
@ -925,7 +935,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
IClusterResidentObject dir = this.ProcessWorkDirectory(job.ManagerProcessGuid, true, job.Machine, job); // immutable
var matchingfiles = dir.GetFilesAndFolders("DryadLinqProgram__*.xml").ToList();
if (matchingfiles.Count() != 1)
throw new CalypsoClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
throw new ClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
IClusterResidentObject result = matchingfiles.First();
result.ShouldCacheLocally = true; // immutable
return result;
@ -1306,7 +1316,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
//IClusterResidentObject dir = this.ProcessWorkDirectory(new DryadProcessIdentifier("Process.000.001"), true, job.Machine, job);
var matchingfiles = dir.GetFilesAndFolders("DryadLinqProgram__*.xml").ToList();
if (matchingfiles.Count() != 1)
throw new CalypsoClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
throw new ClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
IClusterResidentObject result = matchingfiles.First();
result.ShouldCacheLocally = true; // immutable
return result;
@ -1395,10 +1405,67 @@ namespace Microsoft.Research.Calypso.JobObjectModel
{
}
/// <summary>
/// Enumerate all the clusters this user is subscribed to.
/// </summary>
/// <returns>The list of clusters this user is subscribed to.</returns>
public static IEnumerable<ClusterConfiguration> EnumerateAzureDfsSubscribedClusters()
{
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var configDir = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
"Windows Azure Powershell");
var defaultFile = Path.Combine(configDir, "WindowsAzureProfile.xml");
if (File.Exists(defaultFile))
{
using (FileStream s = new FileStream(defaultFile, FileMode.Open, FileAccess.Read))
{
XDocument doc = XDocument.Load(s);
XNamespace ns = doc.Root.GetDefaultNamespace();
IEnumerable<XElement> subs = doc.Descendants(ns + "AzureSubscriptionData");
foreach (XElement sub in subs)
{
string thumbprint = sub.Descendants(ns + "ManagementCertificate").Single().Value;
string subId = sub.Descendants(ns + "SubscriptionId").Single().Value;
Guid subGuid = new Guid(subId);
X509Certificate2 cert = store.Certificates.Cast<X509Certificate2>().First(item => item.Thumbprint == thumbprint);
HDInsightCertificateCredential sCred = new HDInsightCertificateCredential(subGuid, cert);
IHDInsightClient sClient = HDInsightClient.Connect(sCred);
var clusters = sClient.ListClusters();
foreach (var cluster in clusters)
{
var account = cluster.DefaultStorageAccount;
var accountName = account.Name.Split('.').First();
Console.WriteLine("Cluster " + cluster.Name + " uses account " + accountName + " with key " + account.Key);
AzureDfsClusterConfiguration config = null;
try
{
config = new AzureDfsClusterConfiguration();
config.AzureClient = new AzureDfsClient(accountName, account.Key, "dryad-jobs");
config.Name = cluster.Name;
}
catch (Exception ex)
{
Console.WriteLine("Exception while reconstructing cluster " + cluster.Name + ": " + ex);
}
if (config != null)
yield return config;
}
}
}
}
}
/// <summary>
/// Azure account name.
/// </summary>
public string AccountName { get; set; }
public
string AccountName { get; set; }
/// <summary>
/// Azure account key.
/// </summary>
@ -1567,7 +1634,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
IClusterResidentObject dir = this.ProcessWorkDirectory(job.ManagerProcessGuid, true, job.Machine, job); // immutable
var matchingfiles = dir.GetFilesAndFolders("DryadLinqProgram__*.xml").ToList();
if (matchingfiles.Count() != 1)
throw new CalypsoClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
throw new ClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
IClusterResidentObject result = matchingfiles.First();
(result as AzureDfsFile).IsDfsStream = true;
result.ShouldCacheLocally = true; // immutable

134
JobBrowser/JOM/ClusterStatus.cs
View File

@ -23,11 +23,10 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Research.Calypso.Tools;
using Microsoft.Research.Peloponnese.Storage;
using Microsoft.Research.Tools;
namespace Microsoft.Research.Calypso.JobObjectModel
namespace Microsoft.Research.JobObjectModel
{
/// <summary>
/// Dynamic information of all the jobs and machines in a cluster.
@ -119,22 +118,20 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// The cached of tasks on the cluster.
/// </summary>
/// <param name="virtualCluster">Virtual cluster selected; defined only for Scope clusters.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="reportProgress">Used to report progress.</param>
public IEnumerable<ClusterJobInformation> GetClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
/// <param name="manager">Communication manager.</param>
public IEnumerable<ClusterJobInformation> GetClusterJobList(string virtualCluster, CommManager manager)
{
this.RecomputeClusterJobList(virtualCluster, reporter, reportProgress);
this.RecomputeClusterJobList(virtualCluster, manager);
return this.clusterJobs.Values.ToList();
}
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="reportProgress">Used to report progress.</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable once UnusedParameter.Global
protected abstract void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress);
protected abstract void RecomputeClusterJobList(string virtualCluster, CommManager manager);
/// <summary>
/// Discover the (unique) dryadlinq job corresponding to a cluster job.
@ -160,12 +157,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// </summary>
/// <param name="job">Job to discover.</param>
/// <returns>The cluster job, or null if not found.</returns>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="reportProgress">Used to report progress.</param>
public virtual ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, StatusReporter reporter, Action<int> reportProgress)
/// <param name="manager">Communication manager.</param>
public virtual ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, CommManager manager)
{
if (this.clusterJobs == null)
this.RecomputeClusterJobList(job.VirtualCluster, reporter, reportProgress);
this.RecomputeClusterJobList(job.VirtualCluster, manager);
return this.clusterJobs[job.ClusterJobId];
}
@ -173,13 +169,12 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// Refresh the job summary status.
/// </summary>
/// <param name="summary">Summary to refresh.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="reportProgress">Used to report progress.</param>
public virtual void RefreshStatus(DryadLinqJobSummary summary, StatusReporter reporter, Action<int> reportProgress)
/// <param name="manager">Communication manager.</param>
public virtual void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
{
// refresh the whole list
this.RecomputeClusterJobList(summary.VirtualCluster, reporter, reportProgress);
ClusterJobInformation info = this.DiscoverClusterJob(summary, reporter, reportProgress);
this.RecomputeClusterJobList(summary.VirtualCluster, manager);
ClusterJobInformation info = this.DiscoverClusterJob(summary, manager);
if (info == null)
{
summary.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
@ -226,10 +221,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <summary>
/// Recompute the list of jobs on the cluster and add them to the clusterJobs field.
/// </summary>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="virtualCluster">Unused.</param>
/// <param name="reportProgress">Used to report progress.</param>
protected override void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
if (string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
@ -242,32 +236,32 @@ namespace Microsoft.Research.Calypso.JobObjectModel
string[] files = Directory.GetFiles(joblist, "*.xml");
foreach (var file in files)
{
manager.Token.ThrowIfCancellationRequested();
DryadLinqJobSummary job = Utilities.LoadXml<DryadLinqJobSummary>(file);
string cjid = job.Cluster + "-" + job.ClusterJobId; // there may be two jobs with same id from different clusters
ClusterJobInformation ci = new ClusterJobInformation(this.Config.Name, job.Cluster, cjid, job.Name, job.User, job.Date, job.EndTime - job.Date, job.Status);
ci.SetAssociatedSummary(job);
if (this.clusterJobs.ContainsKey(cjid))
{
reporter("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error);
manager.Status("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error);
continue;
}
this.clusterJobs.Add(cjid, ci);
}
reportProgress(100);
manager.Progress(100);
}
/// <summary>
/// Refresh the job summary status.
/// </summary>
/// <param name="job">Summary to refresh.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="reportProgress">Used to report progres.</param>
public override void RefreshStatus(DryadLinqJobSummary job, StatusReporter reporter, Action<int> reportProgress)
/// <param name="manager">Communication manager.</param>
public override void RefreshStatus(DryadLinqJobSummary job, CommManager manager)
{
ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job);
ClusterStatus actualStatus = actual.CreateClusterStatus();
actualStatus.RefreshStatus(job, reporter, reportProgress);
ClusterJobInformation info = actualStatus.DiscoverClusterJob(job, reporter, reportProgress);
actualStatus.RefreshStatus(job, manager);
ClusterJobInformation info = actualStatus.DiscoverClusterJob(job, manager);
if (info == null)
{
job.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
@ -303,13 +297,12 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// </summary>
/// <param name="job">Cluster job.</param>
/// <returns>Throws an exception.</returns>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="reportProgress">Used to report progress.</param>
public override ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, StatusReporter reporter, Action<int> reportProgress)
/// <param name="manager">Communication manager.</param>
public override ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, CommManager manager)
{
ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job);
ClusterStatus actualStatus = actual.CreateClusterStatus();
return actualStatus.DiscoverClusterJob(job, reporter, reportProgress);
return actualStatus.DiscoverClusterJob(job, manager);
}
/// <summary>
@ -358,10 +351,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="reportProgress">Used to report progress.</param>
protected override void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
if (!Directory.Exists(this.config.JobsFolder))
@ -371,6 +363,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
int done = 0;
foreach (var job in subfolders)
{
manager.Token.ThrowIfCancellationRequested();
string jobId = Path.GetFileName(job);
ClusterJobInformation info = this.GetJobInfo(job, jobId);
if (info != null)
@ -378,9 +371,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(jobId, info);
}
reportProgress(done++ *100/subfolders.Length);
manager.Progress(done++ *100/subfolders.Length);
}
reportProgress(100);
manager.Progress(100);
}
/// <summary>
@ -478,10 +471,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <summary>
/// Force the recomputation of the cluster job list.
/// </summary>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
/// <param name="reportProgress">Used to report progress.</param>
protected override void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
/// <param name="manager">Communication manager.</param>
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
{
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
var jobs = this.config.AzureClient.EnumerateDirectory("").ToList();
@ -489,15 +481,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
int done = 0;
foreach (var job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
ClusterJobInformation info = this.GetJobInfo(job);
if (info != null)
{
// ReSharper disable once AssignNullToNotNullAttribute
this.clusterJobs.Add(job, info);
}
reportProgress(100*done++/jobs.Count);
manager.Progress(100*done++/jobs.Count);
}
reportProgress(100);
manager.Progress(100);
}
/// <summary>
@ -547,23 +540,12 @@ namespace Microsoft.Research.Calypso.JobObjectModel
DateTime lastHeartBeat = DateTime.MinValue;
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
bool found = false;
string jobName = jobRootFolder;
var jobsFolders = this.config.AzureClient.EnumerateDirectory(jobRootFolder).ToList();
foreach (var file in jobsFolders)
{
if (file.Contains("DryadLinqProgram__"))
{
var blob = this.config.AzureClient.Container.GetBlockBlobReference(file);
blob.FetchAttributes();
var props = blob.Properties;
if (props.LastModified.HasValue)
{
date = props.LastModified.Value.DateTime;
date = date.ToLocalTime();
}
found = true;
}
else if (file.EndsWith("heartbeat"))
if (file.EndsWith("heartbeat"))
{
var blob = this.config.AzureClient.Container.GetPageBlobReference(file);
blob.FetchAttributes();
@ -582,6 +564,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
case "running":
status = ClusterJobInformation.ClusterJobStatus.Running;
break;
case "killed":
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
break;
default:
Console.WriteLine("Unknown status " + st);
break;
@ -599,6 +584,31 @@ namespace Microsoft.Research.Calypso.JobObjectModel
status = ClusterJobInformation.ClusterJobStatus.Failed;
}
}
if (props.ContainsKey("jobname"))
{
jobName = props["jobname"];
}
if (props.ContainsKey("starttime"))
{
var t = props["starttime"];
if (DateTime.TryParse(t, out date))
date = date.ToLocalTime();
}
found = true;
}
else if (file.Contains("DryadLinqProgram__") &&
// newer heartbeats contain the date
date != DateTime.MinValue)
{
var blob = this.config.AzureClient.Container.GetBlockBlobReference(file);
blob.FetchAttributes();
var props = blob.Properties;
if (props.LastModified.HasValue)
{
date = props.LastModified.Value.DateTime;
date = date.ToLocalTime();
}
}
}
@ -607,7 +617,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
TimeSpan running = TimeSpan.Zero;
if (date != DateTime.MinValue && lastHeartBeat != DateTime.MinValue)
running = lastHeartBeat - date;
var info = new ClusterJobInformation(this.config.Name, "", jobRootFolder, jobRootFolder, Environment.UserName, date, running, status);
var info = new ClusterJobInformation(this.config.Name, "", jobRootFolder, jobName, Environment.UserName, date, running, status);
return info;
}
@ -615,9 +625,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// Refresh the job summary status.
/// </summary>
/// <param name="summary">Summary to refresh.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="reportProgress">Used to report progress.</param>
public override void RefreshStatus(DryadLinqJobSummary summary, StatusReporter reporter, Action<int> reportProgress)
/// <param name="manager">Communication manager.</param>
public override void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
{
// refresh the whole list
ClusterJobInformation info = this.GetJobInfo(summary.JobID);
@ -636,7 +645,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <returns>True if the cancellation succeeded.</returns>
public override bool CancelJob(DryadLinqJobSummary job)
{
throw new InvalidOperationException();
AzureUtils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId);
return true;
}
}
}

99
JobBrowser/JOM/JobObjectModel.csproj
View File

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -10,7 +10,7 @@
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>DistributedDataCollection</RootNamespace>
<AssemblyName>Microsoft.Research.DryadLinq.JobBrowser.DistributedDataCollection</AssemblyName>
<AssemblyName>DistributedDataCollection</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<StartupObject>
@ -37,28 +37,6 @@
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<DocumentationFile>bin\Debug\Microsoft.Research.DryadLinq.JobBrowser.DistributedDataCollection.xml</DocumentationFile>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath>
@ -91,43 +69,59 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client">
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hpc.Scheduler, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\Dependencies\Microsoft.Hpc.Scheduler.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hpc.Scheduler.Properties, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\Dependencies\Microsoft.Hpc.Scheduler.Properties.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<Reference Include="Newtonsoft.Json">
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
@ -135,6 +129,15 @@
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.Services.Client" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
@ -153,7 +156,7 @@
<Compile Include="ClusterAccess.cs" />
<Compile Include="ClusterConfiguration.cs" />
<Compile Include="ClusterStatus.cs" />
<Compile Include="Cosmos.cs" />
<Compile Include="dryadlog.cs" />
<Compile Include="jobinfo.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Storage.cs" />
@ -186,15 +189,19 @@
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.0-beta023\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.0-beta023\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

8
JobBrowser/JOM/Properties/AssemblyInfo.cs
View File

@ -28,9 +28,9 @@ using System.Runtime.InteropServices;
[assembly: AssemblyTitle("DistributedDataCollection")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("DistributedDataCollection")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyCopyright("Copyright © 2009")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.0.0")]
[assembly: AssemblyFileVersion("0.1.0.0")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

8
JobBrowser/JOM/app.config
View File

@ -2,10 +2,6 @@
<configuration>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
@ -22,6 +18,10 @@
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>

31
JobBrowser/JOM/clusteraccess.cs
View File

@ -20,15 +20,15 @@ limitations under the License.
*/
using System.Text.RegularExpressions;
using Microsoft.Research.Calypso.Tools;
using System;
using System.Collections.Generic;
using System.IO;
using System.Diagnostics;
using Microsoft.Research.Peloponnese.Storage;
using Microsoft.Research.Tools;
using Microsoft.WindowsAzure.Storage.Blob;
namespace Microsoft.Research.Calypso.JobObjectModel
namespace Microsoft.Research.JobObjectModel
{
/// <summary>
/// A cluster-resident object is a file or a folder.
@ -158,7 +158,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
if (file.Job == null)
return;
if (string.IsNullOrEmpty(file.LocalCachePath))
throw new CalypsoClusterException("Missing expected LocalCachePath");
throw new ClusterException("Missing expected LocalCachePath");
CachedClusterResidentObject.RecordCachedFile(file.Job, file.LocalCachePath);
}
@ -375,7 +375,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
{
// cache it
if (this.RepresentsAFolder)
throw new CalypsoClusterException("Cannot cache folders");
throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream();
return new FileSharedStreamReader(this.Pathname.ToString(), writer, this.OnClose);
@ -504,7 +504,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
get
{
if (this.RepresentsAFolder)
throw new CalypsoClusterException("Cannot get size of a folder");
throw new ClusterException("Cannot get size of a folder");
if (File.Exists(this.LocalCachePath))
{
FileInfo info = new FileInfo(this.LocalCachePath);
@ -732,7 +732,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <returns>Throws an exception.</returns>
public IEnumerable<IClusterResidentObject> GetFilesAndFolders(string match)
{
throw new CalypsoClusterException("Object is not a folder");
throw new ClusterException("Object is not a folder");
}
private long size;
@ -787,7 +787,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <returns>Throws an exception.</returns>
public IClusterResidentObject GetFile(string filename)
{
throw new CalypsoClusterException("Object is not a folder");
throw new ClusterException("Object is not a folder");
}
/// <summary>
@ -797,7 +797,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <returns>Throws an exception.</returns>
public IClusterResidentObject GetFolder(string foldername)
{
throw new CalypsoClusterException("Object is not a folder");
throw new ClusterException("Object is not a folder");
}
}
@ -1011,13 +1011,24 @@ namespace Microsoft.Research.Calypso.JobObjectModel
this.client.ContainerName,
this.path);
}
StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, false, 1024 * 1024);
long size = this.Size;
int bufferSize = 1024*1024;
if (size >= 0)
{
bufferSize = (int)(size/10);
if (bufferSize < 1024*1024)
bufferSize = 1024*1024;
if (bufferSize > 20*1024*1024)
bufferSize = 20*1024*1024;
}
StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, false, bufferSize);
if (this.ShouldCacheLocally && this.LocalCachePath != null)
{
// cache it
if (this.RepresentsAFolder)
throw new CalypsoClusterException("Cannot cache folders");
throw new ClusterException("Cannot cache folders");
StreamWriter writer = this.CreateTempStream();
return new SharedStreamReader(reader, writer, this.OnClose);
}

12
JobBrowser/JOM/cosmos.cs → JobBrowser/JOM/dryadlog.cs
View File

@ -19,17 +19,17 @@ limitations under the License.
*/
namespace Microsoft.Research.Calypso.JobObjectModel
namespace Microsoft.Research.JobObjectModel
{
using System.Text.RegularExpressions;
using System;
using System.Diagnostics;
/// <summary>
/// Information about a standard Cosmos log entry.
/// Information about a standard Dryad log entry.
/// </summary>
[Serializable]
public class CosmosLogEntry : IParse
public class DryadLogEntry : IParse
{
/// <summary>
/// Message severity.
@ -98,7 +98,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <summary>
/// Allocate an empty log entry
/// </summary>
public CosmosLogEntry()
public DryadLogEntry()
{
this.Malformed = true;
}
@ -107,7 +107,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// Allocate a log entry from a given string.
/// </summary>
/// <param name="line">String to initialize the log entry.</param>
public CosmosLogEntry(string line)
public DryadLogEntry(string line)
{
this.Malformed = true;
// ReSharper disable once DoNotCallOverridableMethodsInConstructor
@ -167,7 +167,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// An extended log entry is like a cosmos log entry, but it has a prefix: GUID,Machine
/// </summary>
[Serializable]
public class ExtendedLogEntry : CosmosLogEntry
public class ExtendedLogEntry : DryadLogEntry
{
static Regex loglineregex =
new Regex(@"([-0-9A-F]+), # 1 guid, inserted by reader

248
JobBrowser/JOM/jobinfo.cs
View File

@ -27,24 +27,23 @@ using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using Microsoft.Research.Calypso.Tools;
using System.Diagnostics;
using Microsoft.Research.Tools;
namespace Microsoft.Research.Calypso.JobObjectModel
namespace Microsoft.Research.JobObjectModel
{
/// <summary>
/// Exception throw by Calypso when it cannot understand the structure of a Dryad/DryadLINQ job.
/// Exception thrown when we cannot understand the structure of a Dryad/DryadLINQ job.
/// </summary>
public class CalypsoDryadException : Exception
public class DryadException : Exception
{
/// <summary>
/// Create a new CalypsoDryadException.
/// Create a new DryadException.
/// </summary>
/// <param name="message">Message conveyed by the exception.</param>
public CalypsoDryadException(string message) : base(message) { }
public DryadException(string message) : base(message) { }
}
/// <summary>
@ -195,7 +194,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
this.SuccessfulVertices++;
break;
default:
throw new CalypsoDryadException("Unexpected vertex state " + vertex.State);
throw new DryadException("Unexpected vertex state " + vertex.State);
}
}
this.TotalInitiatedVertices -= this.AbandonedVertices;
@ -423,22 +422,21 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <param name="cf">Cluster configuration.</param>
/// <param name="summary">Summary description of the job.</param>
/// <returns>The Dryad job description, or null.</returns>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="fill">If true, fill all the information, otherwise the user will have to call FillInformation on the result later.</param>
/// <param name="updateProgress">Delegate used to report progress.</param>
public static DryadLinqJobInfo CreateDryadLinqJobInfo(ClusterConfiguration cf, DryadLinqJobSummary summary, bool fill, StatusReporter reporter, Action<int> updateProgress)
/// <param name="manager">Communication manager.</param>
public static DryadLinqJobInfo CreateDryadLinqJobInfo(ClusterConfiguration cf, DryadLinqJobSummary summary, bool fill, CommManager manager)
{
try
{
DryadLinqJobInfo job = new DryadLinqJobInfo(cf, summary);
if (fill)
job.CollectEssentialInformation(reporter, updateProgress);
job.CollectEssentialInformation(manager);
return job;
}
catch (Exception e)
{
Trace.TraceInformation(e.ToString());
reporter("Could not collect job information for " + summary.Name + ": " + e.Message, StatusKind.Error);
manager.Status("Could not collect job information for " + summary.Name + ": " + e.Message, StatusKind.Error);
return null;
}
}
@ -506,7 +504,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
if (this.stdoutpath == null)
{
throw new CalypsoClusterException("Could not locate JM standard output file in folder " + jmdir);
throw new ClusterException("Could not locate JM standard output file in folder " + jmdir);
}
}
}
@ -515,9 +513,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <summary>
/// Refresh the job status.
/// </summary>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="updateProgress">Used to report progress.</param>
public void RefreshJobStatus(StatusReporter reporter, Action<int> updateProgress)
/// <param name="manager">Communication manager.</param>
public void RefreshJobStatus(CommManager manager)
{
// skip if job is finished
if (this.Summary.Status == ClusterJobInformation.ClusterJobStatus.Failed ||
@ -526,18 +523,17 @@ namespace Microsoft.Research.Calypso.JobObjectModel
return;
ClusterStatus status = this.ClusterConfiguration.CreateClusterStatus();
status.RefreshStatus(this.Summary, reporter, updateProgress);
status.RefreshStatus(this.Summary, manager);
}
/// <summary>
/// Fill the job info by parsing the stdout.txt.
/// <param name="statusReporter">Delegate used to report errors.</param>
/// <returns>True if it succeeds, false otherwise.</returns>
/// <param name="updateProgress">Delegate used to report progress.</param>
/// <returns>The updated job.</returns>
/// <param name="manager">Communication manager.</param>
/// </summary>
public bool CollectEssentialInformation(StatusReporter statusReporter, Action<int> updateProgress)
public bool CollectEssentialInformation(CommManager manager)
{
this.RefreshJobStatus(statusReporter, updateProgress);
this.RefreshJobStatus(manager);
if (this.ManagerVertex == null)
{
this.ManagerVertex = new ExecutedVertexInstance(this, -1, 0, "JobManager", "", this.Summary.Date);
@ -562,13 +558,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
if (this.stdoutpath == null)
return false;
bool success = this.ParseStdout(this.stdoutpath, statusReporter, updateProgress);
updateProgress(100);
bool success = this.ParseStdout(this.stdoutpath, manager);
manager.Progress(100);
if (!success)
return false;
this.JobInfoCannotBeCollected = false;
statusReporter("Stdout parsed", StatusKind.OK);
manager.Status("Stdout parsed", StatusKind.OK);
this.LastUpdatetime = DateTime.Now;
if (this.Summary.Status == ClusterJobInformation.ClusterJobStatus.Running)
@ -600,13 +596,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
{
Match m = numberre.Match(vertexlist);
if (!m.Success)
throw new CalypsoDryadException("Could not find vertex number in " + vertexlist);
throw new DryadException("Could not find vertex number in " + vertexlist);
string number = m.Groups[1].Value;
// now scan a balanced number of parantheses
string rest = m.Groups[2].Value;
if (rest[0] != '(')
throw new CalypsoDryadException("Expecting open parens after vertex number");
throw new DryadException("Expecting open parens after vertex number");
int opened = 0;
int i;
for (i = 0; i < rest.Length; i++)
@ -624,7 +620,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
}
}
if (opened != 0 || i <= 2)
throw new CalypsoDryadException("did not find matched parantheses in vertex name in " + vertexlist + ", can't parse");
throw new DryadException("did not find matched parantheses in vertex name in " + vertexlist + ", can't parse");
string name = rest.Substring(1, i - 2); // skip first and last paranthesis
yield return new Tuple<string, int>(name, int.Parse(number));
vertexlist = rest.Substring(i);
@ -761,7 +757,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
case "running":
{
string process;
kvp.TryGetValue("id", out process); // "process" is also good
kvp.TryGetValue("id", out process);
if (process == null)
kvp.TryGetValue("process", out process);
string machine = kvp["computer"];
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
this.jobVertices.Remap(vi, process);
@ -776,6 +774,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
vi.SetState(ExecutedVertexInstance.VertexState.Successful);
vi.End = timeStamp;
vi.ExitCode = "";
this.UsefulCPUTime += vi.RunningTime;
break;
}
case "failed":
@ -784,7 +783,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
if (vi.State != ExecutedVertexInstance.VertexState.Started)
vi.SetState(ExecutedVertexInstance.VertexState.Cancelled);
else
{
vi.SetState(ExecutedVertexInstance.VertexState.Failed);
if (vi.RunningTime > TimeSpan.Zero)
this.WastedCPUTime += vi.RunningTime;
}
if (kvp.ContainsKey("errorstring"))
vi.AddErrorString(kvp["errorstring"]);
string exitcode;
@ -797,33 +800,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
}
else if (kvp.ContainsKey("outputChannel"))
{
string chan = kvp["outputChannel"];
int channelNo = int.Parse(chan);
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
if (!kvp.ContainsKey("errorstatus"))
{
}
else
{
if (kvp.ContainsKey("errorstring"))
vi.AddErrorString(kvp["errorstring"]);
}
}
else if (kvp.ContainsKey("inputChannel"))
{
string chan = kvp["inputChannel"];
int channelNo = int.Parse(chan);
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
if (!kvp.ContainsKey("errorstatus"))
{
}
else
{
if (kvp.ContainsKey("errorstring"))
vi.AddErrorString(kvp["errorstring"]);
}
}
else if (kvp.ContainsKey("io"))
{
if (kvp["io"] == "starting")
@ -834,6 +820,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
if (vi.InputChannels == null)
vi.InputChannels = new Dictionary<int, ChannelEndpointDescription>();
for (int i = 0; i < numberOfInputs; i++)
{
string uri;
@ -870,6 +857,24 @@ namespace Microsoft.Research.Calypso.JobObjectModel
vi.DataRead = totalRead;
vi.DataWritten = totalWritten;
if (vi.InputChannels != null)
{
foreach (int ch in vi.InputChannels.Keys)
{
long bytes = TryGetNumeric(kvp, "rb." + ch);
vi.InputChannels[ch].Size = bytes;
}
}
if (vi.OutputChannels != null)
{
foreach (int ch in vi.OutputChannels.Keys)
{
long bytes = TryGetNumeric(kvp, "wb." + ch);
vi.OutputChannels[ch].Size = bytes;
}
}
this.TotalDataRead += totalRead;
this.LocalReadData += localRead;
this.CrossPodDataRead += tempReadCrossRack;
@ -879,17 +884,26 @@ namespace Microsoft.Research.Calypso.JobObjectModel
{
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
if (vi.InputChannels != null)
{
foreach (int ch in vi.InputChannels.Keys)
{
long bytes = TryGetNumeric(kvp, "rb." + ch);
vi.InputChannels[ch].Size = bytes;
bytes = TryGetNumeric(kvp, "tb." + ch);
vi.InputChannels[ch].TotalSize = bytes;
}
}
if (vi.InputChannels != null)
{
foreach (int ch in vi.OutputChannels.Keys)
{
long bytes = TryGetNumeric(kvp, "wb." + ch);
vi.OutputChannels[ch].Size = bytes;
}
}
long totalRead = TryGetNumeric(kvp, "totalRead");
long totalWritten = TryGetNumeric(kvp, "totalWritten");
@ -1062,7 +1076,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
else if (m.Groups[1].Value == "ices")
onevertex = false;
else
throw new CalypsoDryadException("Can't figure out if one or many vertices");
throw new DryadException("Can't figure out if one or many vertices");
IEnumerable<Tuple<string, int>> vertexList = DryadLinqJobInfo.ParseVertices(vertices);
@ -1084,7 +1098,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
}
if (vertexcount > 1 && onevertex)
throw new CalypsoDryadException("Expected one vertex, found " + vertexcount);
throw new DryadException("Expected one vertex, found " + vertexcount);
}
else
{
@ -1225,7 +1239,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
if (vi.State == ExecutedVertexInstance.VertexState.Started)
{
Console.WriteLine("Timing information while vertex is still running " + vi);
//throw new CalypsoClusterException("Timing information for vertex still running: " + vi);
//throw new ClusterException("Timing information for vertex still running: " + vi);
}
DateTime last = vi.SetTiming(createtime, m.Groups[5].Value, m.Groups[6].Value, m.Groups[7].Value, m.Groups[8].Value, m.Groups[9].Value);
if (last > this.lastTimestampSeen)
@ -1245,7 +1259,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
}
}
else
throw new CalypsoDryadException("Unmatched timing information line " + line);
throw new DryadException("Unmatched timing information line " + line);
}
else if (line.Contains("Process has failed"))
{
@ -1410,7 +1424,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
return retval;
}
private ISharedStreamReader cachedStdoutReader = null;
private ISharedStreamReader cachedStdoutReader;
/// <summary>
/// Remember how many lines were parsed, and skip them on a second invocation.
@ -1420,10 +1434,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// Parse the stdout.txt file from the job manager.
/// </summary>
/// <param name="file">File to parse.</param>
/// <param name="statusReporter">Delegate used to report errors.</param>
/// <param name="updateProgress">Delegate used to report progress.</param>
/// <param name="manager">Communication manager.</param>
/// <returns>True if the parsing succeeds.</returns>
private bool ParseStdout(IClusterResidentObject file, StatusReporter statusReporter, Action<int> updateProgress)
private bool ParseStdout(IClusterResidentObject file, CommManager manager)
{
int currentLine = 0;
if (this.stdoutLinesParsed == 0)
@ -1440,15 +1453,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
string message = "Scanning JM stdout " + file;
if (filesize >= 0)
message += string.Format("({0:N0} bytes)", filesize);
statusReporter(message, StatusKind.LongOp);
manager.Status(message, StatusKind.LongOp);
if (this.cachedStdoutReader == null)
this.cachedStdoutReader = file.GetStream();
if (this.cachedStdoutReader.Exception != null)
{
statusReporter("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
return false;
}
while (!this.cachedStdoutReader.EndOfStream)
{
string line = this.cachedStdoutReader.ReadLine();
@ -1457,6 +1471,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
{
while (true)
{
manager.Token.ThrowIfCancellationRequested();
int startLine = currentLine;
bool completeLine = true;
try
@ -1465,7 +1480,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
}
catch (Exception ex)
{
statusReporter(string.Format("Line {0}: Exception {1}", currentLine, ex.Message), StatusKind.Error);
manager.Status(string.Format("Line {0}: Exception {1}", currentLine, ex.Message), StatusKind.Error);
Console.WriteLine("Line {0}: Exception {1}", currentLine, ex);
}
if (!completeLine)
@ -1485,7 +1500,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
currentLine++;
if (currentLine % 100 == 0 && filesize > 0)
{
updateProgress(Math.Min(100, (int)(100 * readbytes / filesize)));
manager.Progress(Math.Min(100, (int)(100 * readbytes / filesize)));
}
}
@ -1500,13 +1515,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
// we are done with this stream
if (this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Failed ||
this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Successful)
{
this.cachedStdoutReader.Close();
this.cachedStdoutReader = null; // will force reopening if refreshed
}
}
return true;
}
catch (Exception e)
{
statusReporter("Exception while reading stdout " + e.Message, StatusKind.Error);
manager.Status("Exception while reading stdout " + e.Message, StatusKind.Error);
Trace.TraceInformation(e.ToString());
return false;
}
@ -1592,7 +1610,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
string line = sr.ReadLine();
if (!line.Contains("DryadProfiler")) continue;
CosmosLogEntry le = new CosmosLogEntry(line);
DryadLogEntry le = new DryadLogEntry(line);
if (le.Subsystem != "DryadProfiler") continue;
if (!le.Message.EndsWith("channel status")) continue;
@ -1905,7 +1923,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <summary>
/// Parse the query plan: cluster-specific.
/// </summary>
protected abstract void ParseQueryPlan();
/// <param name="manager">Communication manager.</param>
protected abstract void ParseQueryPlan(CommManager manager);
int fictitiousStages;
@ -2009,11 +2028,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// Factory: create the plan for a given job.
/// </summary>
/// <param name="dryadLinqJobInfo">Job to create plan for.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <returns>The plan or null.</returns>
public static DryadJobStaticPlan CreatePlan(DryadLinqJobInfo dryadLinqJobInfo, StatusReporter reporter)
/// <param name="manager">Communication manager.</param>
public static DryadJobStaticPlan CreatePlan(DryadLinqJobInfo dryadLinqJobInfo, CommManager manager)
{
reporter("Trying to build static plan", StatusKind.LongOp);
manager.Status("Trying to build static plan", StatusKind.LongOp);
ClusterConfiguration config = dryadLinqJobInfo.ClusterConfiguration;
IClusterResidentObject file = config.JobQueryPlan(dryadLinqJobInfo.Summary);
if (config is CacheClusterConfiguration)
@ -2025,12 +2044,12 @@ namespace Microsoft.Research.Calypso.JobObjectModel
{
retval = new DryadLinqJobStaticPlan(config, file.GetStream());
}
retval.ParseQueryPlan();
retval.ParseQueryPlan(manager);
return retval;
}
else
{
reporter("Exception while looking for plan " + file.Exception.Message, StatusKind.Error);
manager.Status("Exception while looking for plan " + file.Exception.Message, StatusKind.Error);
return null;
}
}
@ -2056,9 +2075,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <summary>
/// Parse an XML query plan and represent that information.
/// </summary>
protected override void ParseQueryPlan()
/// <param name="manager">Communicaton manager.</param>
protected override void ParseQueryPlan(CommManager manager)
{
string planString = this.planStream.ReadToEnd();
string planString = this.planStream.ReadToEnd(manager.Token);
XDocument plan = XDocument.Parse(planString);
// ReSharper disable PossibleNullReferenceException
@ -2104,7 +2124,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
info.Arity = Connection.ConnectionType.AllToAll;
break;
default:
throw new CalypsoDryadException("Don't know about connection of type " + connection);
throw new DryadException("Don't know about connection of type " + connection);
}
switch (cht)
{
@ -2118,7 +2138,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
info.ChannelKind = Connection.ChannelType.Fifo;
break;
default:
throw new CalypsoDryadException("Don't know about channel of type " + cht);
throw new DryadException("Don't know about channel of type " + cht);
}
this.perNodeConnectionInfo.Add(stage.Id, info);
}
@ -2247,14 +2267,15 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <summary>
/// Parse the Algebra file.
/// </summary>
private void ParseAlgebra()
/// <param name="manager">Communication manager.</param>
private void ParseAlgebra(CommManager manager)
{
// TODO: this parser is not really complete, as I don't understand the semantics of all xml elements.
Dictionary<string, string> outToStage = new Dictionary<string, string>(); // map an output to a stage name. Assume that ios have unique names.
Dictionary<string, List<string>> inputs = new Dictionary<string, List<string>>();
// <CsJobAlgebra> <graph> <process> ...
string planString = this.planStream.ReadToEnd();
string planString = this.planStream.ReadToEnd(manager.Token);
XDocument plan = XDocument.Parse(planString);
// ReSharper disable PossibleNullReferenceException
XElement graph = plan.Root.Element("graph"); // graph node, children are stages
@ -2416,13 +2437,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <summary>
/// Parse the vertex definition file.
/// </summary>
private void ParseVertexDef()
private void ParseVertexDef(CommManager manager)
{
if (this.vertexDef.Exception != null)
return;
// <ScopeVertices> <ScopeVertex> <operator> <input> </input> <output> </output>
string planString = this.vertexDef.ReadToEnd();
string planString = this.vertexDef.ReadToEnd(manager.Token);
XDocument vxDef = XDocument.Parse(planString);
XElement vertices = vxDef.Root;
@ -2469,10 +2490,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <summary>
/// Parse the query plan for a Scope job.
/// </summary>
protected override void ParseQueryPlan()
/// <param name="manager">Communication manager.</param>
protected override void ParseQueryPlan(CommManager manager)
{
this.ParseAlgebra();
this.ParseVertexDef();
this.ParseAlgebra(manager);
this.ParseVertexDef(manager);
}
}
@ -2653,9 +2675,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// </summary>
public string LocalPath { get; protected set; }
/// <summary>
/// How big is the channel (0 if it cannot be determined, e.g. FIFO, -1 if the channel data cannot be retrieved).
/// The actual data read/written so far (0 if it cannot be determined, e.g. FIFO, -1 if the channel data cannot be retrieved).
/// </summary>
public long Size { get; set; }
/// <summary>
/// How much of the channel was
/// </summary>
public long TotalSize { get; set; }
/// <summary>
/// String representation of the endpoint.
@ -2664,7 +2690,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
{
string uritype = this.UriType;
string localpath = this.LocalPath;
if (this.TotalSize == 0)
return string.Format("{0,4} {1,20:N0} {2}://{3}", this.Number, this.Size, uritype, localpath);
else
return string.Format("{0,4} {1,20:N0}/{2,20:N0} {3}://{4}", this.Number, this.Size, this.TotalSize, uritype, localpath);
}
/// <summary>
@ -2684,7 +2713,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
int sepindex = uri.IndexOf("://");
if (sepindex < 0)
throw new CalypsoDryadException("Channel URI " + uri + " does not contain separator ://");
throw new DryadException("Channel URI " + uri + " does not contain separator ://");
this.UriType = uri.Substring(0, sepindex);
// some HPC URIs use the compression scheme as an "option" (not really defined for file:// uris, but...)
@ -2737,10 +2766,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
this.Number = number;
int sepindex = uri.IndexOf("://");
if (sepindex < 0)
throw new CalypsoClusterException("Channel URI " + uri + " does not contain separator ://");
throw new ClusterException("Channel URI " + uri + " does not contain separator ://");
this.UriType = uri.Substring(0, sepindex);
this.LocalPath = uri.Substring(sepindex + 3);
this.Size = size;
this.TotalSize = size;
}
}
@ -2883,7 +2912,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
this.StdoutFile = job.ClusterConfiguration.ProcessStdoutFile(this.ProcessIdentifier, false, machine, job.Summary);
this.SetState(VertexState.Started);
if (approxStartTime == DateTime.MinValue)
throw new CalypsoDryadException("Unexpected small start time for vertex");
throw new DryadException("Unexpected small start time for vertex");
this.LogDirectory = job.ClusterConfiguration.ProcessLogDirectory(this.ProcessIdentifier, false, machine, job.Summary);
this.LogFilesPattern = job.ClusterConfiguration.VertexLogFilesPattern(false, job.Summary);
this.UniqueID = uniqueId;
@ -3049,7 +3078,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
{
Trace.TraceInformation("Vertex {0} which is not started is still running?", this.Name);
return;
//throw new CalypsoClusterException("Vertex which is not started is still running?");
//throw new ClusterException("Vertex which is not started is still running?");
}
if (this.Start > when)
// This can happen if the cluster clocks are not synchronized with the local machine clocks.
@ -3088,7 +3117,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
TimeSpan total = TimeSpan.FromSeconds(totSeconds);
DateTime totalTime = creation + total;
if (totSeconds < 0)
throw new CalypsoDryadException("Negative total time for vertex " + this.Name);
throw new DryadException("Negative total time for vertex " + this.Name);
// if the vertex has no machine just ignore the times
if (string.IsNullOrEmpty(this.Machine))
@ -3209,9 +3238,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <param name="uriprefix">If the channel is an output, prefix the path with this; this is null for inputs.</param>
/// <param name="skip">If true, do not return anything (still useful to advance the stream reader).</param>
/// <param name="fast">If true the channel sizes are not discovered; this is much faster, since no remote machines are queried for files.</param>
/// <param name="updateProgress">Delegate used to report progress.</param>
/// <param name="reporter">Delegate used to report errors.</param>
private Dictionary<int, ChannelEndpointDescription> DiscoverOriginalInfoChannels(ISharedStreamReader sr, string uriprefix, bool skip, bool fast, StatusReporter reporter, Action<int> updateProgress)
/// <param name="manager">Communication manager.</param>
private Dictionary<int, ChannelEndpointDescription> DiscoverOriginalInfoChannels(ISharedStreamReader sr, string uriprefix, bool skip, bool fast, CommManager manager)
{
bool isInput = uriprefix == null;
@ -3231,21 +3259,18 @@ namespace Microsoft.Research.Calypso.JobObjectModel
string channel = sr.ReadLine();
if (channel == null)
{
if (updateProgress != null)
updateProgress(100);
manager.Progress(100);
return null;
}
if (!skip)
{
ChannelEndpointDescription desc = new ChannelEndpointDescription(isInput, i, channel, uriprefix, fast, reporter);
ChannelEndpointDescription desc = new ChannelEndpointDescription(isInput, i, channel, uriprefix, fast, manager.Status);
channels.Add(i, desc);
if (updateProgress != null)
updateProgress(i * 100 / channelCount);
manager.Progress(i * 100 / channelCount);
}
}
if (updateProgress != null)
updateProgress(100);
manager.Progress(100);
if (skip)
return null;
return channels;
@ -3258,9 +3283,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <param name="inputs">If true discover the inputs.</param>
/// <param name="outputs">If true discover the outputs.</param>
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
/// <param name="progress">Delegate used to report progress.</param>
/// <param name="reporter">Delegate used to report errors.</param>
public bool DiscoverOriginalInfoChannels(bool inputs, bool outputs, bool fast, StatusReporter reporter, Action<int> progress)
/// <param name="manager">Communication manager.</param>
public bool DiscoverOriginalInfoChannels(bool inputs, bool outputs, bool fast, CommManager manager)
{
string filename = string.Format("vertex-{0}-{1}-rerun-originalInfo.txt", this.Number, this.Version);
bool success = true;
@ -3270,7 +3294,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
// skip discovery
inputs = false;
ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream();
var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, reporter, progress);
var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, manager);
if (channels == null)
{
if (inputs)
@ -3281,7 +3305,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
if (this.OutputChannels != null)
// skip discovery
outputs = false;
channels = this.DiscoverOriginalInfoChannels(sr, this.WorkDirectory.ToString(), !outputs, fast, reporter, progress);
channels = this.DiscoverOriginalInfoChannels(sr, this.WorkDirectory.ToString(), !outputs, fast, manager);
if (channels == null)
{
if (outputs)
@ -3300,28 +3324,27 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <param name="inputs">If true discover the inputs.</param>
/// <param name="outputs">If true discover the outputs.</param>
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
/// <param name="progress">Delegate used to report progress.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="manager">Communication manager.</param>
// ReSharper disable UnusedParameter.Global
public bool DiscoverScopeChannels(bool inputs, bool outputs, bool fast, StatusReporter reporter, Action<int> progress)
public bool DiscoverScopeChannels(bool inputs, bool outputs, bool fast, CommManager manager)
// ReSharper restore UnusedParameter.Global
{
// find the xml file
var files = this.WorkDirectory.GetFilesAndFolders("vcmdStart*.xml").ToList();
if (files.Count != 1)
{
reporter("Cannot locate vcmdStart*.xml file", StatusKind.Error);
manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error);
return false;
}
ISharedStreamReader sr = files.First().GetStream();
if (sr.Exception != null)
{
reporter("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);
manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);
return false;
}
// ReSharper disable PossibleNullReferenceException
XDocument plan = XDocument.Parse(sr.ReadToEnd());
XDocument plan = XDocument.Parse(sr.ReadToEnd(manager.Token));
if (inputs && this.InputChannels == null)
{
var channels = new Dictionary<int, ChannelEndpointDescription>();
@ -3365,9 +3388,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
/// <param name="inputs">If true discover the inputs.</param>
/// <param name="outputs">If true discover the outputs.</param>
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
/// <param name="progress">Delegate used to report progress.</param>
/// <param name="reporter">Delegate used to report errors.</param>
public bool DiscoverChannels(bool inputs, bool outputs, bool fast, StatusReporter reporter, Action<int> progress)
/// <param name="manager">Communication manager.</param>
public bool DiscoverChannels(bool inputs, bool outputs, bool fast, CommManager manager)
{
// check if the result is already cached
if ((this.InputChannels != null || !inputs) &&
@ -3395,7 +3417,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
if (wd is UNCFile)
{
result = this.DiscoverOriginalInfoChannels(inputs, outputs, fast, reporter, progress);
result = this.DiscoverOriginalInfoChannels(inputs, outputs, fast, manager);
}
else
{
@ -3448,9 +3470,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
internal void Update(string name, string guid)
{
if (this.State != VertexState.Cancelled && this.State != VertexState.Abandoned)
throw new CalypsoDryadException("Updating a non-cancelled/abandoned vertex");
throw new DryadException("Updating a non-cancelled/abandoned vertex");
if (this.Name != name)
throw new CalypsoDryadException("Vertex changed name");
throw new DryadException("Vertex changed name");
this.UniqueID = guid;
this.SetState(VertexState.Created);
// the stdoutfile is expected to change, so I don't invalidate the cache

12
JobBrowser/JOM/packages.config
View File

@ -1,12 +1,20 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />

6
JobBrowser/JOM/storage.cs
View File

@ -22,13 +22,13 @@ limitations under the License.
#undef USE_DSC
#undef USE_TIDYFS
using Microsoft.Research.Calypso.Tools;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Research.Tools;
namespace Microsoft.Research.Calypso.JobObjectModel
namespace Microsoft.Research.JobObjectModel
{
/// <summary>
@ -197,7 +197,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
{
var sourceInputs = plan.GetStageConnections(source, true).ToList();
if (sourceInputs.Count() != 1)
throw new CalypsoDryadException("Unexpected number of inputs for stage " + source.Name);
throw new DryadException("Unexpected number of inputs for stage " + source.Name);
source = sourceInputs.First().From;
}

50
JobBrowser/JobBrowser.sln
View File

@ -1,8 +1,6 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2013
VisualStudioVersion = 12.0.30110.0
MinimumVisualStudioVersion = 10.0.40219.1
# Visual Studio 2012
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JobBrowser", "JobBrowser\JobBrowser.csproj", "{EDDD2E0B-A52B-4E25-9436-B874017673FF}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools", "Tools\Tools.csproj", "{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}"
@ -13,72 +11,26 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UsefulForms", "UsefulForms\
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Debug|Mixed Platforms = Debug|Mixed Platforms
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|Any CPU = Release|Any CPU
Release|Mixed Platforms = Release|Mixed Platforms
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x64.ActiveCfg = Debug|x64
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x64.Build.0 = Debug|x64
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x86.ActiveCfg = Debug|Any CPU
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Any CPU.Build.0 = Release|Any CPU
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Mixed Platforms.Build.0 = Release|Any CPU
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x64.ActiveCfg = Release|x64
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x64.Build.0 = Release|x64
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x86.ActiveCfg = Release|Any CPU
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x64.ActiveCfg = Debug|x64
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x64.Build.0 = Debug|x64
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x86.ActiveCfg = Debug|Any CPU
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Any CPU.Build.0 = Release|Any CPU
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Mixed Platforms.Build.0 = Release|Any CPU
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x64.ActiveCfg = Release|x64
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x64.Build.0 = Release|x64
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x86.ActiveCfg = Release|Any CPU
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Any CPU.Build.0 = Debug|Any CPU
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x64.ActiveCfg = Debug|x64
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x64.Build.0 = Debug|x64
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x86.ActiveCfg = Debug|Any CPU
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Any CPU.ActiveCfg = Release|Any CPU
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Any CPU.Build.0 = Release|Any CPU
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Mixed Platforms.Build.0 = Release|Any CPU
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x64.ActiveCfg = Release|x64
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x64.Build.0 = Release|x64
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x86.ActiveCfg = Release|Any CPU
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Any CPU.Build.0 = Debug|Any CPU
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x64.ActiveCfg = Debug|x64
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x64.Build.0 = Debug|x64
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x86.ActiveCfg = Debug|Any CPU
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Any CPU.ActiveCfg = Release|Any CPU
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Any CPU.Build.0 = Release|Any CPU
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Mixed Platforms.Build.0 = Release|Any CPU
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x64.ActiveCfg = Release|x64
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x64.Build.0 = Release|x64
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x86.ActiveCfg = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

100
JobBrowser/JobBrowser/ClusterBrowser.cs
View File

@ -28,11 +28,11 @@ using System.Linq;
using System.Net;
using System.Windows.Forms;
using System.Diagnostics;
using Microsoft.Research.Calypso.JobObjectModel;
using Microsoft.Research.Calypso.Tools;
using Microsoft.Research.Calypso.UsefulForms;
using Microsoft.Research.JobObjectModel;
using Microsoft.Research.Tools;
using Microsoft.Research.UsefulForms;
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
/// <summary>
/// Class to browse jobs on cluster, copy, summarize and start visualization.
@ -73,7 +73,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
private TextWriterTraceListener LogFile;
private BackgroundWorkQueue queue;
private BackgroundWorker queueWorker;
/// <summary>
/// Jobs from the cluster.
@ -92,8 +91,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.InitializeComponent();
this.status = new StatusWriter(this.statuslabel, this.statusStrip, this.Status);
this.queueWorker = new BackgroundWorker();
this.queue = new BackgroundWorkQueue(this.queueWorker);
BackgroundWorker queueWorker = new BackgroundWorker();
this.queue = new BackgroundWorkQueue(queueWorker, null, null);
this.completeJobsList = new List<ClusterJobInformation>();
this.refreshTimer = new Timer();
@ -235,7 +234,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.toolStripMenuItem_job.Enabled = true;
var item = new BackgroundWorkItem<List<ClusterJobInformation>>(
(s, p) => BuildClusterJobList(s, p, this.clusterStatus, this.SelectedVirtualCluster),
m => BuildClusterJobList(m, this.clusterStatus, this.SelectedVirtualCluster),
this.JobListRetrieved,
"getJobs");
this.Queue(item);
@ -250,6 +249,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
private void JobListRetrieved(bool cancelled, List<ClusterJobInformation> jobs)
{
if (cancelled) return;
this.filteredDataGridView.DataGridView.ClearSelection();
this.completeJobsList = jobs;
this.clusterJobs.SetItems(this.completeJobsList);
@ -274,12 +275,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// Talk to the web server and build the list of clustr jobs; used it to populate the upper panel.
/// </summary>
/// <param name="virtualCluster">Virtual cluster selected; defined only for Scope clusters.</param>
/// <param name="progress">Reports progress.</param>
/// <param name="reporter">Reports status.</param>
/// <param name="manager">Communication manager.</param>
/// <param name="status">Cluster to scan.</param>
private static List<ClusterJobInformation> BuildClusterJobList(StatusReporter reporter, Action<int> progress, ClusterStatus status, string virtualCluster)
private static List<ClusterJobInformation> BuildClusterJobList(CommManager manager, ClusterStatus status, string virtualCluster)
{
return status.GetClusterJobList(virtualCluster, reporter, progress).ToList();
return status.GetClusterJobList(virtualCluster, manager).ToList();
}
/// <summary>
@ -311,7 +311,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
if (js == null)
return;
DryadLinqJobInfo job = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.clusterStatus.Config, js, false, this.Status, delegate {});
// TODO: this should run in the background
CommManager manager = new CommManager(this.Status, delegate { }, new System.Threading.CancellationTokenSource().Token);
DryadLinqJobInfo job = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.clusterStatus.Config, js, false, manager);
if (job != null)
{
JobBrowser browser = new JobBrowser(job);
@ -366,7 +368,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
IEnumerable<ClusterJobInformation> ti = this.SelectedJobs();
this.Status("Starting job browser...", StatusKind.LongOp);
IEnumerable<DryadLinqJobSummary> jobs = ti.Select(t => t.DiscoverDryadLinqJob(this.clusterStatus, this.Status)).ToList();
IEnumerable<DryadLinqJobInfo> detailed = jobs.Select(j => DryadLinqJobInfo.CreateDryadLinqJobInfo(this.clusterStatus.Config, j, false, this.Status, delegate { }));
CommManager manager = new CommManager(this.Status, delegate { }, new System.Threading.CancellationTokenSource().Token);
IEnumerable<DryadLinqJobInfo> detailed = jobs.Select(j => DryadLinqJobInfo.CreateDryadLinqJobInfo(this.clusterStatus.Config, j, false, manager));
foreach (DryadLinqJobInfo j in detailed)
{
if (j == null) continue;
@ -442,7 +446,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
IEnumerable<DryadLinqJobSummary> jobs = todo.Select(j => j.DiscoverDryadLinqJob(this.clusterStatus, this.Status)).Where(j => j != null);
var item = new BackgroundWorkItem<bool>(
(s, p) => ClusterWork.CancelJobs(jobs, this.clusterStatus, s, p),
m => ClusterWork.CancelJobs(jobs, this.clusterStatus, m),
(c, b) => { },
"cancel");
this.Queue(item);
@ -504,8 +508,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.autoRefreshToolStripMenuItem.Checked = this.formSettings.AutoRefresh;
this.AddClusterNameToMenu("<add>");
this.AddClusterNameToMenu("<scan>");
ClusterConfiguration.ReconstructKnownCluster(this.formSettings.KnownClusters);
int found = 0;
IEnumerable<string> clusters = ClusterConfiguration.GetKnownClusterNames();
foreach (string c in clusters)
{
@ -515,7 +522,12 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
{
(config as CacheClusterConfiguration).StartCaching();
}
found++;
}
if (found == 0)
// try to find them by scanning
this.ScanClusters();
}
/// <summary>
@ -539,6 +551,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
newItem.Click += this.AddNewCluster;
return;
}
if (clusterName == "<scan>")
{
newItem.Click += this.ScanClusters;
return;
}
var selItem = newItem.DropDownItems.Add("Select");
var delItem = newItem.DropDownItems.Add("Delete");
@ -548,6 +565,31 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
editItem.Click += editItem_Click;
}
/// <summary>
/// Scan the clusters we are subscribed to and add them to the list of known clusters.
/// </summary>
/// <param name="sender">Unused.</param>
/// <param name="e">Unused.</param>
private void ScanClusters(object sender, EventArgs e)
{
this.ScanClusters();
}
/// <summary>
/// Scan the clusters we are subscribed to and add them to the list of known clusters.
/// </summary>
private void ScanClusters()
{
this.Status("Scanning for known clusters", StatusKind.LongOp);
foreach (var conf in ClusterConfiguration.EnumerateSubscribedClusters())
{
ClusterConfiguration.AddKnownCluster(conf);
this.AddClusterNameToMenu(conf.Name);
this.Status("Adding cluster " + conf.Name, StatusKind.OK);
}
this.Status("Scan completed", StatusKind.OK);
}
/// <summary>
/// Edit a cluster.
/// </summary>
@ -684,7 +726,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
IEnumerable<DryadLinqJobSummary> jobs = todo.Select(j => j.DiscoverDryadLinqJob(this.clusterStatus, this.Status)).Where(j => j != null);
var item = new BackgroundWorkItem<List<DiagnosisLog>>(
(s, p) => ClusterWork.DiagnoseJobs(jobs, this.clusterStatus.Config, s, p),
m => ClusterWork.DiagnoseJobs(jobs, this.clusterStatus.Config, m),
DiagnosisResult.ShowDiagnosisResult,
"cancel");
this.Queue(item);
@ -895,18 +937,18 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// </summary>
/// <param name="jobs">Jobs to cancel.</param>
/// <param name="cluster">Cluster where the jobs are running.</param>
/// <param name="statusReporter">Delegate used to report errors.</param>
/// <returns>True if all cancellations succeed.</returns>
/// <param name="updateProgress">Delegate used to report progress.</param>
/// <param name="manager">Communicatoni manager.</param>
// ReSharper disable once UnusedParameter.Global
public static bool CancelJobs(IEnumerable<DryadLinqJobSummary> jobs, ClusterStatus cluster, StatusReporter statusReporter, Action<int> updateProgress)
public static bool CancelJobs(IEnumerable<DryadLinqJobSummary> jobs, ClusterStatus cluster, CommManager manager)
{
bool done = true;
foreach (DryadLinqJobSummary job in jobs)
{
manager.Token.ThrowIfCancellationRequested();
if (job.Status != ClusterJobInformation.ClusterJobStatus.Running)
{
statusReporter("Job " + job.Name + " does not appear to be running; will still try to cancel", StatusKind.Error);
manager.Status("Job " + job.Name + " does not appear to be running; will still try to cancel", StatusKind.Error);
}
bool success;
@ -923,9 +965,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
}
if (success)
statusReporter("Job " + job.Name + " cancelled", StatusKind.OK);
manager.Status("Job " + job.Name + " cancelled", StatusKind.OK);
else
statusReporter("Cancellation of " + job.Name + " failed " + reason, StatusKind.Error);
manager.Status("Cancellation of " + job.Name + " failed " + reason, StatusKind.Error);
done &= success;
}
return done;
@ -936,9 +978,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// </summary>
/// <param name="jobs">Jobs to diagnose.</param>
/// <param name="config">Cluster configuration.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="updateProgress">Delegate used to report progress.</param>
public static List<DiagnosisLog> DiagnoseJobs(IEnumerable<DryadLinqJobSummary> jobs, ClusterConfiguration config, StatusReporter reporter, Action<int> updateProgress)
/// <param name="manager">Communicatino manager.</param>
public static List<DiagnosisLog> DiagnoseJobs(IEnumerable<DryadLinqJobSummary> jobs, ClusterConfiguration config, CommManager manager)
{
var dryadLinqJobSummaries = jobs as DryadLinqJobSummary[] ?? jobs.ToArray();
int jobCount = dryadLinqJobSummaries.Count();
@ -949,15 +990,16 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
{
if (summary == null) continue;
JobFailureDiagnosis diagnosis = JobFailureDiagnosis.CreateJobFailureDiagnosis(config, summary, reporter, updateProgress);
reporter("Diagnosing " + summary.ShortName(), StatusKind.LongOp);
manager.Token.ThrowIfCancellationRequested();
JobFailureDiagnosis diagnosis = JobFailureDiagnosis.CreateJobFailureDiagnosis(config, summary, manager);
manager.Status("Diagnosing " + summary.ShortName(), StatusKind.LongOp);
DiagnosisLog log = diagnosis.Diagnose();
result.Add(log);
done++;
updateProgress(done * 100 / jobCount);
manager.Progress(done * 100 / jobCount);
}
reporter("Diagnosis complete", StatusKind.OK);
manager.Status("Diagnosis complete", StatusKind.OK);
return result;
}
}

61
JobBrowser/JobBrowser/ClusterBrowser.designer.cs generated
View File

@ -18,9 +18,9 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License.
*/
using Microsoft.Research.Calypso.Tools;
using Microsoft.Research.Tools;
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
partial class ClusterBrowser
{
@ -56,10 +56,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.openInJobBrowserToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.diagnoseToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
this.terminateToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
this.cancelToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
this.statusStrip = new System.Windows.Forms.StatusStrip();
this.statuslabel = new System.Windows.Forms.ToolStripStatusLabel();
this.toolStripStatusLabel_backgroundWork = new System.Windows.Forms.ToolStripStatusLabel();
this.toolStripStatusLabel_currentWork = new System.Windows.Forms.ToolStripStatusLabel();
this.toolStripProgressBar = new System.Windows.Forms.ToolStripProgressBar();
this.flowLayoutPanel_header = new System.Windows.Forms.FlowLayoutPanel();
this.label_vc = new System.Windows.Forms.Label();
@ -78,7 +77,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.logFileToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.autoRefreshToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.clusterToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.filteredDataGridView = new Microsoft.Research.Calypso.Tools.FilteredDataGridView();
this.filteredDataGridView = new FilteredDataGridView();
this.contextMenuStrip_job.SuspendLayout();
this.statusStrip.SuspendLayout();
this.flowLayoutPanel_header.SuspendLayout();
@ -92,7 +91,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.diagnoseToolStripMenuItem1,
this.terminateToolStripMenuItem1});
this.contextMenuStrip_job.Name = "contextMenuStrip_job";
this.contextMenuStrip_job.Size = new System.Drawing.Size(182, 92);
this.contextMenuStrip_job.Size = new System.Drawing.Size(182, 70);
//
// openInJobBrowserToolStripMenuItem
//
@ -114,15 +113,19 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.terminateToolStripMenuItem1.Name = "terminateToolStripMenuItem1";
this.terminateToolStripMenuItem1.Size = new System.Drawing.Size(181, 22);
this.terminateToolStripMenuItem1.Text = "Terminate";
this.terminateToolStripMenuItem1.Visible = false;
this.terminateToolStripMenuItem1.Click += new System.EventHandler(this.terminateToolStripMenuItem_Click);
//
// cancelToolStripMenuItem1
//
this.cancelToolStripMenuItem1.Name = "cancelToolStripMenuItem1";
this.cancelToolStripMenuItem1.Size = new System.Drawing.Size(186, 22);
this.cancelToolStripMenuItem1.Text = "Cancel current work";
this.cancelToolStripMenuItem1.Click += new System.EventHandler(this.cancelToolStripMenuItem_Click);
//
// statusStrip
//
this.statusStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] {
this.statuslabel,
this.toolStripStatusLabel_backgroundWork,
this.toolStripStatusLabel_currentWork,
this.toolStripProgressBar});
this.statusStrip.Location = new System.Drawing.Point(0, 431);
this.statusStrip.Name = "statusStrip";
@ -134,27 +137,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
//
this.statuslabel.Name = "statuslabel";
this.statuslabel.Overflow = System.Windows.Forms.ToolStripItemOverflow.Never;
this.statuslabel.Size = new System.Drawing.Size(833, 17);
this.statuslabel.Size = new System.Drawing.Size(1026, 17);
this.statuslabel.Spring = true;
this.statuslabel.Text = "Status displayed here";
this.statuslabel.TextAlign = System.Drawing.ContentAlignment.MiddleLeft;
//
// toolStripStatusLabel_backgroundWork
//
this.toolStripStatusLabel_backgroundWork.BorderStyle = System.Windows.Forms.Border3DStyle.SunkenOuter;
this.toolStripStatusLabel_backgroundWork.Name = "toolStripStatusLabel_backgroundWork";
this.toolStripStatusLabel_backgroundWork.Overflow = System.Windows.Forms.ToolStripItemOverflow.Never;
this.toolStripStatusLabel_backgroundWork.Size = new System.Drawing.Size(109, 17);
this.toolStripStatusLabel_backgroundWork.Text = "0 pending activities";
//
// toolStripStatusLabel_currentWork
//
this.toolStripStatusLabel_currentWork.BorderStyle = System.Windows.Forms.Border3DStyle.SunkenOuter;
this.toolStripStatusLabel_currentWork.Name = "toolStripStatusLabel_currentWork";
this.toolStripStatusLabel_currentWork.Overflow = System.Windows.Forms.ToolStripItemOverflow.Never;
this.toolStripStatusLabel_currentWork.Size = new System.Drawing.Size(84, 17);
this.toolStripStatusLabel_currentWork.Text = "Doing nothing";
//
// toolStripProgressBar
//
this.toolStripProgressBar.Name = "toolStripProgressBar";
@ -219,6 +206,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
// jobToolStripMenuItem_file
//
this.jobToolStripMenuItem_file.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
this.cancelToolStripMenuItem1,
this.newWindowToolStripMenuItem,
this.refreshToolStripMenuItem1,
this.exitToolStripMenuItem1});
@ -264,7 +252,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.toolStripMenuItem_job.Size = new System.Drawing.Size(37, 20);
this.toolStripMenuItem_job.Text = "&Job";
this.toolStripMenuItem_job.ToolTipText = "View job informaotion in detail.";
this.toolStripMenuItem_job.Visible = false;
//
// jobBrowserToolStripMenuItem
//
@ -279,6 +266,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.diagnoseToolStripMenuItem.Size = new System.Drawing.Size(163, 22);
this.diagnoseToolStripMenuItem.Text = "Diagnose";
this.diagnoseToolStripMenuItem.ToolTipText = "Attempt to diagnose job failures.";
this.diagnoseToolStripMenuItem.Visible = false;
this.diagnoseToolStripMenuItem.Click += new System.EventHandler(this.diagnoseToolStripMenuItem_Click);
//
// terminateToolStripMenuItem
@ -295,6 +283,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.openFromURLToolStripMenuItem.Size = new System.Drawing.Size(163, 22);
this.openFromURLToolStripMenuItem.Text = "Open job URL...";
this.openFromURLToolStripMenuItem.ToolTipText = "Open the job given a URL.";
this.openFromURLToolStripMenuItem.Visible = false;
this.openFromURLToolStripMenuItem.Click += new System.EventHandler(this.openFromURLToolStripMenuItem_Click);
//
// settingsToolStripMenuItem
@ -309,7 +298,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
// logFileToolStripMenuItem
//
this.logFileToolStripMenuItem.Name = "logFileToolStripMenuItem";
this.logFileToolStripMenuItem.Size = new System.Drawing.Size(139, 22);
this.logFileToolStripMenuItem.Size = new System.Drawing.Size(152, 22);
this.logFileToolStripMenuItem.Text = "Log file";
this.logFileToolStripMenuItem.ToolTipText = "When enabled logs errors in the selected file.";
this.logFileToolStripMenuItem.Click += new System.EventHandler(this.logFileToolStripMenuItem_Click);
@ -317,7 +306,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
// autoRefreshToolStripMenuItem
//
this.autoRefreshToolStripMenuItem.Name = "autoRefreshToolStripMenuItem";
this.autoRefreshToolStripMenuItem.Size = new System.Drawing.Size(139, 22);
this.autoRefreshToolStripMenuItem.Size = new System.Drawing.Size(152, 22);
this.autoRefreshToolStripMenuItem.Text = "Auto refresh";
this.autoRefreshToolStripMenuItem.Click += new System.EventHandler(this.autoRefreshToolStripMenuItem_Click);
//
@ -326,15 +315,15 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.clusterToolStripMenuItem.Name = "clusterToolStripMenuItem";
this.clusterToolStripMenuItem.Size = new System.Drawing.Size(56, 20);
this.clusterToolStripMenuItem.Text = "Cluster";
//
// filteredDataGridView
//
this.filteredDataGridView.BorderStyle = System.Windows.Forms.BorderStyle.FixedSingle;
this.filteredDataGridView.ContextMenuStrip = this.contextMenuStrip_job;
this.filteredDataGridView.Dock = System.Windows.Forms.DockStyle.Fill;
this.filteredDataGridView.Location = new System.Drawing.Point(0, 56);
this.filteredDataGridView.Location = new System.Drawing.Point(0, 24);
this.filteredDataGridView.Name = "filteredDataGridView";
this.filteredDataGridView.Size = new System.Drawing.Size(1143, 375);
this.filteredDataGridView.Size = new System.Drawing.Size(1143, 429);
this.filteredDataGridView.TabIndex = 15;
this.filteredDataGridView.CellFormatting += new System.Windows.Forms.DataGridViewCellFormattingEventHandler(this.filteredDataGridView_CellFormatting);
this.filteredDataGridView.CellMouseDoubleClick += new System.Windows.Forms.DataGridViewCellMouseEventHandler(this.filteredDataGridView_CellMouseDoubleClick);
@ -365,6 +354,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
}
private void cancelToolStripMenuItem_Click(object sender, System.EventArgs e)
{
this.queue.CancelCurrentWork();
}
#endregion
private System.Windows.Forms.StatusStrip statusStrip;
@ -378,13 +372,12 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
private System.Windows.Forms.ToolStripMenuItem refreshToolStripMenuItem1;
private System.Windows.Forms.ToolStripMenuItem openFromURLToolStripMenuItem;
private System.Windows.Forms.ToolStripMenuItem terminateToolStripMenuItem;
private System.Windows.Forms.ToolStripStatusLabel toolStripStatusLabel_backgroundWork;
private System.Windows.Forms.ToolStripStatusLabel toolStripStatusLabel_currentWork;
private System.Windows.Forms.ToolStripMenuItem diagnoseToolStripMenuItem;
private System.Windows.Forms.ContextMenuStrip contextMenuStrip_job;
private System.Windows.Forms.ToolStripMenuItem openInJobBrowserToolStripMenuItem;
private System.Windows.Forms.ToolStripMenuItem diagnoseToolStripMenuItem1;
private System.Windows.Forms.ToolStripMenuItem terminateToolStripMenuItem1;
private System.Windows.Forms.ToolStripMenuItem cancelToolStripMenuItem1;
private System.Windows.Forms.ToolStripProgressBar toolStripProgressBar;
private FilteredDataGridView filteredDataGridView;
private System.Windows.Forms.ToolStripMenuItem newWindowToolStripMenuItem;

3
JobBrowser/JobBrowser/ClusterBrowser.resx
View File

@ -123,9 +123,6 @@
<metadata name="statusStrip.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<value>17, 17</value>
</metadata>
<metadata name="backgroundWorker.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<value>253, 17</value>
</metadata>
<metadata name="menuStrip.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<value>419, 16</value>
</metadata>

2
JobBrowser/JobBrowser/ClusterConfigEditor.Designer.cs generated
View File

@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License.
*/
namespace Microsoft.Research.Calypso.JobObjectModel
namespace Microsoft.Research.JobObjectModel
{
partial class ClusterConfigEditor
{

4
JobBrowser/JobBrowser/ClusterConfigEditor.cs
View File

@ -23,9 +23,9 @@ using System;
using System.Collections.Generic;
using System.Drawing;
using System.Windows.Forms;
using Microsoft.Research.Calypso.Tools;
using Microsoft.Research.Tools;
namespace Microsoft.Research.Calypso.JobObjectModel
namespace Microsoft.Research.JobObjectModel
{
/// <summary>
/// Editor for cluster configuration.

94
JobBrowser/JobBrowser/Diagnosis.cs
View File

@ -19,15 +19,15 @@ limitations under the License.
*/
using Microsoft.Research.Calypso.JobObjectModel;
using Microsoft.Research.Calypso.Tools;
using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Microsoft.Research.JobObjectModel;
using Microsoft.Research.Tools;
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
/// <summary>
/// The result of a decision (ternary booleans?)
@ -236,30 +236,24 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// </summary>
public DryadLinqJobInfo Job { get; protected set; }
/// <summary>
/// Delegate used to report erorrs.
/// Communication manager.
/// </summary>
public StatusReporter Reporter { get; protected set; }
public CommManager Manager { get; protected set; }
/// <summary>
/// Plan of the job.
/// </summary>
public DryadJobStaticPlan StaticPlan { get; protected set; }
/// <summary>
/// Delegate used to report progress.
/// </summary>
public Action<int> ProgressReporter { get; protected set; }
/// <summary>
/// Create a FailureDiagnosis object.
/// </summary>
/// <param name="job">Job being diagnosed.</param>
/// <param name="plan">Static plan of the job.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="progressReporter">Delegate used to report progress.</param>
protected FailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, StatusReporter reporter, Action<int> progressReporter)
/// <param name="manager">Communication manager.</param>
protected FailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, CommManager manager)
{
this.Job = job;
this.StaticPlan = plan;
this.Reporter = reporter;
this.ProgressReporter = progressReporter;
this.Manager = manager;
this.Summary = job.Summary;
this.cluster = job.ClusterConfiguration;
}
@ -267,19 +261,18 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// <summary>
/// Try to find the job information from cluster and summary.
/// </summary>
/// <param name="status">Delegate used to report status.</param>
/// <param name="progress">Delegate used to report progress.</param>
protected void FindJobInfo(StatusReporter status, Action<int> progress)
/// <param name="manager">Communication manager.</param>
protected void FindJobInfo(CommManager manager)
{
DryadLinqJobInfo jobinfo = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.cluster, this.Summary, true, status, progress);
DryadLinqJobInfo jobinfo = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.cluster, this.Summary, true, manager);
if (jobinfo == null)
{
status("Cannot collect information for " + Summary.ShortName() + " to diagnose", StatusKind.Error);
manager.Status("Cannot collect information for " + Summary.ShortName() + " to diagnose", StatusKind.Error);
return;
}
this.Job = jobinfo;
this.StaticPlan = JobObjectModel.DryadJobStaticPlan.CreatePlan(jobinfo, status);
this.StaticPlan = JobObjectModel.DryadJobStaticPlan.CreatePlan(jobinfo, manager);
}
/// <summary>
@ -287,15 +280,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// </summary>
/// <param name="config">Cluster where job resides.</param>
/// <param name="summary">Job summary.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="progressReporter">Delegate used to report progress.</param>
protected FailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, StatusReporter reporter, Action<int> progressReporter)
/// <param name="manager">Communication manager.</param>
protected FailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, CommManager manager)
{
this.cluster = config;
this.Summary = summary;
this.Reporter = reporter;
this.ProgressReporter = progressReporter;
this.FindJobInfo(reporter, progressReporter);
this.Manager = manager;
this.FindJobInfo(manager);
}
/// <summary>
@ -327,11 +318,10 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// </summary>
/// <param name="vertex">Vertex to diagnose.</param>
/// <param name="job">Job containing the vertex.</param>
/// <param name="progressReporter">Delegate used to report progress.</param>
/// <param name="reporter">Delegate used to report status.</param>
/// <param name="plan">Plan of the executed job.</param>
protected VertexFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, ExecutedVertexInstance vertex, StatusReporter reporter, Action<int> progressReporter)
: base(job, plan, reporter, progressReporter)
/// <param name="manager">Communication manager.</param>
protected VertexFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, ExecutedVertexInstance vertex, CommManager manager)
: base(job, plan, manager)
{
this.Job = job;
this.Vertex = vertex;
@ -344,15 +334,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// </summary>
/// <param name="vertex">Vertex to diagnose.</param>
/// <param name="job">Job containing the vertex.</param>
/// <param name="progressReporter">Delegate used to report progress.</param>
/// <param name="reporter">Delegate used to report status.</param>
/// <param name="manager">Communication manager.</param>
/// <returns>A subclass of VertexFailureDiagnosis.</returns>
/// <param name="plan">Plan of the executed job.</param>
public static VertexFailureDiagnosis CreateVertexFailureDiagnosis(DryadLinqJobInfo job,
DryadJobStaticPlan plan,
ExecutedVertexInstance vertex,
StatusReporter reporter,
Action<int> progressReporter)
CommManager manager)
{
ClusterConfiguration config = job.ClusterConfiguration;
if (config is CacheClusterConfiguration)
@ -381,7 +369,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
DiagnosisLog log = new DiagnosisLog(this.Job, this.Summary);
log.AddMessage(new DiagnosisMessage(DiagnosisMessage.Importance.Final, "Diagnostic for " + this.VertexName, "Vertex state is " + this.Vertex.State));
this.Diagnose(log);
this.Reporter("Vertex diagnosis complete", StatusKind.OK);
this.Manager.Status("Vertex diagnosis complete", StatusKind.OK);
return log;
}
@ -409,7 +397,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// Detect whether the vertex had problems reading a particular channel.
/// </summary>
/// <returns>The channel that cannot be read, or null if that's not the problem.</returns>
public virtual ChannelEndpointDescription ChannelReadFailure()
/// <param name="manager">Communication manager.</param>
public virtual ChannelEndpointDescription ChannelReadFailure(CommManager manager)
{
List<string> stack = this.StackTrace().ToList();
if (stack.Count == 0)
@ -424,7 +413,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
bool success = int.TryParse(m.Groups[3].Value, out channelNo);
if (!success)
return null;
this.Vertex.DiscoverChannels(true, false, true, this.Reporter, null);
this.Vertex.DiscoverChannels(true, false, true, manager);
var channels = this.Vertex.InputChannels;
if (channels == null)
return null;
@ -604,11 +593,10 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// Create a class to diagnose the problems of a job.
/// </summary>
/// <param name="job">Job to diagnose.</param>
/// <param name="progress">Delegate used to report progress.</param>
/// <param name="status">Delegate used to report status.</param>
/// <param name="plan">Plan of the diagnosed job.</param>
protected JobFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, StatusReporter status, Action<int> progress)
: base(job, plan, status, progress)
/// <param name="manager">Communication manager.</param>
protected JobFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, CommManager manager)
: base(job, plan, manager)
{
this.diagnosisLog = new DiagnosisLog(job, job.Summary);
this.jobManager = this.Job.ManagerVertex;
@ -617,12 +605,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// <summary>
/// Create a class to diagnose the problems of a job.
/// </summary>
/// <param name="progress">Delegate used to report progress.</param>
/// <param name="status">Delegate used to report status.</param>
/// <param name="config">Cluster where job resides.</param>
/// <param name="manager">Communication manager.</param>
/// <param name="summary">Job summary.</param>
protected JobFailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, StatusReporter status, Action<int> progress)
: base(config, summary, status, progress)
protected JobFailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, CommManager manager)
: base(config, summary, manager)
{
this.diagnosisLog = new DiagnosisLog(this.Job, summary);
if (this.Job != null)
@ -825,11 +812,10 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// Create a suitable Job Failure diagnosis object for the job being analyzed.
/// </summary>
/// <param name="job">Job to diagnose.</param>
/// <param name="status">Delegate used to report errors.</param>
/// <param name="progress">Delegate used to report progress.</param>
/// <param name="manager">Communication manager.</param>
/// <returns>A subclass of JobFailureDiagnosis with the type appropriate for the job.</returns>
/// <param name="plan">Plan of the job being diagnosed.</param>
public static JobFailureDiagnosis CreateJobFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, StatusReporter status, Action<int> progress)
public static JobFailureDiagnosis CreateJobFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, CommManager manager)
{
ClusterConfiguration config = job.ClusterConfiguration;
if (config is CacheClusterConfiguration)
@ -843,11 +829,10 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// Create a suitable Job Failure diagnosis object for the job being analyzed.
/// </summary>
/// <param name="summary">Job to diagnose.</param>
/// <param name="status">Delegate used to report errors.</param>
/// <param name="progress">Delegate used to report progress.</param>
/// <param name="config">Cluster where job resides.</param>
/// <param name="manager">Communication manager.</param>
/// <returns>A subclass of JobFailureDiagnosis with the type appropriate for the job.</returns>
public static JobFailureDiagnosis CreateJobFailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, StatusReporter status, Action<int> progress)
public static JobFailureDiagnosis CreateJobFailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, CommManager manager)
{
if (config is CacheClusterConfiguration)
config = (config as CacheClusterConfiguration).ActualConfig(summary);
@ -860,7 +845,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// This is incomplete: e.g., it does not work for tidyfs streams.
/// </summary>
/// <returns>Yes if there were correlated failures.</returns>
protected Decision LookForCorrelatedReadFailures()
/// <param name="manager">Communication manager.</param>
protected Decision LookForCorrelatedReadFailures(CommManager manager)
{
// if we have more than this many failures we start to worry
const int maxFailures = 5;
@ -876,13 +862,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
int verticesDone = 0;
foreach (ExecutedVertexInstance v in failures)
{
var crf = VertexFailureDiagnosis.CreateVertexFailureDiagnosis(this.Job, this.StaticPlan, v, null, null).ChannelReadFailure();
var crf = VertexFailureDiagnosis.CreateVertexFailureDiagnosis(this.Job, this.StaticPlan, v, manager).ChannelReadFailure(manager);
if (crf != null)
{
channelsFailed.Add(crf);
}
verticesDone++;
this.ProgressReporter(verticesDone * 100 / totalFailures);
manager.Progress(verticesDone * 100 / totalFailures);
}
if (channelsFailed.Count() < maxFailures)
return Decision.No;

2
JobBrowser/JobBrowser/DiagnosisResult.Designer.cs generated
View File

@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License.
*/
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
using System;

4
JobBrowser/JobBrowser/DiagnosisResult.cs
View File

@ -20,11 +20,11 @@ limitations under the License.
*/
using System.Collections.Generic;
using Microsoft.Research.Calypso.JobObjectModel;
using System;
using System.Windows.Forms;
using Microsoft.Research.JobObjectModel;
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
/// <summary>
/// Display the diagnosis results.

2
JobBrowser/JobBrowser/DryadJobMain.cs
View File

@ -23,7 +23,7 @@ using System;
using System.Diagnostics;
using System.Windows.Forms;
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
static class Program
{

33
JobBrowser/JobBrowser/JobBrowser.Designer.cs generated
View File

@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License.
*/
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
partial class JobBrowser
{
@ -65,13 +65,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.menu = new System.Windows.Forms.MenuStrip();
this.jobToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.refreshToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.collectDataToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.hideCancelledVerticesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.exportToCSVToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.toolStripMenuItem_terminate = new System.Windows.Forms.ToolStripMenuItem();
this.packageCachedFilesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.diagnoseToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.closeToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.cancelCurrentWorkToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.stageToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.cacheLogsForAllVerticesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.vertexToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
@ -80,7 +80,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.profileLocallyCPUSamplingToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.profileLocallyMemorySamplingToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.jMStdoutMentionsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.jMLogsMentionsToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
this.diagnoseToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
this.viewToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.loadFileInEditorToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
@ -124,7 +123,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.menuItem_stageVertexProfileLocallyCPUSampling = new System.Windows.Forms.ToolStripMenuItem();
this.menuItem_stageVertexProfileLocallyMemorySampling = new System.Windows.Forms.ToolStripMenuItem();
this.jMStdoutLinesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.jMLogsMentionsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
this.diagnoseToolStripMenuItem2 = new System.Windows.Forms.ToolStripMenuItem();
this.splitContainer_vertexData = new System.Windows.Forms.SplitContainer();
this.dataGridView_vertexHeader = new System.Windows.Forms.DataGridView();
@ -147,7 +145,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.toolStripStatusLabel_currentWork = new System.Windows.Forms.ToolStripStatusLabel();
this.toolStripStatusLabel_backgroundWork = new System.Windows.Forms.ToolStripStatusLabel();
this.toolStripProgressBar = new System.Windows.Forms.ToolStripProgressBar();
this.backgroundWorker = new System.ComponentModel.BackgroundWorker();
this.menu.SuspendLayout();
((System.ComponentModel.ISupportInitialize)(this.splitContainer_jobAndRest)).BeginInit();
this.splitContainer_jobAndRest.Panel1.SuspendLayout();
@ -204,13 +201,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
//
this.jobToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
this.refreshToolStripMenuItem,
this.collectDataToolStripMenuItem,
this.hideCancelledVerticesToolStripMenuItem,
this.exportToCSVToolStripMenuItem,
this.toolStripMenuItem_terminate,
this.packageCachedFilesToolStripMenuItem,
this.diagnoseToolStripMenuItem,
this.closeToolStripMenuItem});
this.closeToolStripMenuItem,
this.cancelCurrentWorkToolStripMenuItem});
this.jobToolStripMenuItem.Name = "jobToolStripMenuItem";
this.jobToolStripMenuItem.Size = new System.Drawing.Size(37, 20);
this.jobToolStripMenuItem.Text = "Job";
@ -248,7 +245,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.toolStripMenuItem_terminate.Size = new System.Drawing.Size(195, 22);
this.toolStripMenuItem_terminate.Text = "Terminate job";
this.toolStripMenuItem_terminate.ToolTipText = "Requests the cluster to terminate the job execution.";
this.toolStripMenuItem_terminate.Visible = false;
this.toolStripMenuItem_terminate.Click += new System.EventHandler(this.toolStripMenuItem_terminate_Click);
//
// packageCachedFilesToolStripMenuItem
@ -279,6 +275,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.closeToolStripMenuItem.ToolTipText = "Save the settings and close the window.";
this.closeToolStripMenuItem.Click += new System.EventHandler(this.closeToolStripMenuItem_Click);
//
// cancelCurrentWorkToolStripMenuItem
//
this.cancelCurrentWorkToolStripMenuItem.Name = "cancelCurrentWorkToolStripMenuItem";
this.cancelCurrentWorkToolStripMenuItem.Size = new System.Drawing.Size(195, 22);
this.cancelCurrentWorkToolStripMenuItem.Text = "Cancel current work";
this.cancelCurrentWorkToolStripMenuItem.Click += new System.EventHandler(this.cancelCurrentWorkToolStripMenuItem_Click);
//
// stageToolStripMenuItem
//
this.stageToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
@ -304,7 +307,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.profileLocallyCPUSamplingToolStripMenuItem,
this.profileLocallyMemorySamplingToolStripMenuItem,
this.jMStdoutMentionsToolStripMenuItem,
this.jMLogsMentionsToolStripMenuItem1,
this.diagnoseToolStripMenuItem1});
this.vertexToolStripMenuItem.Enabled = false;
this.vertexToolStripMenuItem.Name = "vertexToolStripMenuItem";
@ -908,10 +910,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.menuItem_stageVertexProfileLocallyCPUSampling,
this.menuItem_stageVertexProfileLocallyMemorySampling,
this.jMStdoutLinesToolStripMenuItem,
this.jMLogsMentionsToolStripMenuItem,
this.diagnoseToolStripMenuItem2});
this.contextMenu_stageVertex.Name = "vertexContextMenuStrip";
this.contextMenu_stageVertex.Size = new System.Drawing.Size(256, 158);
this.contextMenu_stageVertex.Size = new System.Drawing.Size(256, 136);
//
// menuItem_stageVertexLocalDebugManaged
//
@ -1252,11 +1253,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.toolStripProgressBar.Name = "toolStripProgressBar";
this.toolStripProgressBar.Size = new System.Drawing.Size(100, 16);
//
// backgroundWorker
//
this.backgroundWorker.DoWork += new System.ComponentModel.DoWorkEventHandler(this.backgroundWorker_DoWork);
this.backgroundWorker.RunWorkerCompleted += new System.ComponentModel.RunWorkerCompletedEventHandler(this.backgroundWorker_RunWorkerCompleted);
//
// JobBrowser
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
@ -1352,7 +1348,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
private System.Windows.Forms.TextBox textBox_stageCode;
private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel3;
private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel4;
private System.Windows.Forms.ToolStripMenuItem collectDataToolStripMenuItem;
private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel5;
private System.Windows.Forms.Label label1;
private System.Windows.Forms.TextBox textBox_find;
@ -1362,7 +1357,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
private System.Windows.Forms.Label label_matches;
private System.Windows.Forms.CheckBox checkBox_refresh;
private System.Windows.Forms.Button button_filter;
private System.ComponentModel.BackgroundWorker backgroundWorker;
private System.Windows.Forms.ComboBox comboBox_vertexInformation;
private System.Windows.Forms.Label label_comboVertex;
private System.Windows.Forms.ToolStripMenuItem viewToolStripMenuItem;
@ -1380,9 +1374,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
private System.Windows.Forms.ToolStripMenuItem menuItem_stageVertexLocalDebugUnmanaged;
private System.Windows.Forms.ToolStripMenuItem debugLocallyUnmanagedToolStripMenuItem;
private System.Windows.Forms.ToolStripMenuItem jMStdoutMentionsToolStripMenuItem;
private System.Windows.Forms.ToolStripMenuItem jMLogsMentionsToolStripMenuItem1;
private System.Windows.Forms.ToolStripMenuItem jMStdoutLinesToolStripMenuItem;
private System.Windows.Forms.ToolStripMenuItem jMLogsMentionsToolStripMenuItem;
private System.Windows.Forms.TextBox textBox_stageFilter;
private System.Windows.Forms.Button button_stageFilter;
private System.Windows.Forms.Button button_clearStageFilter;
@ -1412,5 +1404,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
private System.Windows.Forms.ToolStripMenuItem cacheAllLogsToolStripMenuItem;
private System.Windows.Forms.ToolStripMenuItem exportToCSVToolStripMenuItem;
private System.Windows.Forms.ToolStripMenuItem cacheLogsForAllVerticesToolStripMenuItem;
private System.Windows.Forms.ToolStripMenuItem cancelCurrentWorkToolStripMenuItem;
}
}

264
JobBrowser/JobBrowser/JobBrowser.cs
View File

@ -34,10 +34,10 @@ using System.Threading;
using System.Windows.Forms;
using Microsoft.Msagl.GraphViewerGdi;
using Microsoft.Msagl.Splines;
using Microsoft.Research.Calypso.JobObjectModel;
using Microsoft.Research.Calypso.Tools;
using Microsoft.Research.JobObjectModel;
using Microsoft.Research.Tools;
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
/// <summary>
/// A form to display information about a DryadLinq job.
@ -66,39 +66,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// </summary>
private BackgroundWorkQueue queue;
/// <summary>
/// Activity to perform by the backgroundWorker.
/// </summary>
class BackgroundWorkInfo
{
public enum WorkKind
{
LoadJobInfo,
};
public WorkKind Work; // kind of operation to perform in the background
public bool Success; // if true the work succeeded
public DateTime workStartTime; // when work is started
public override string ToString()
{
return this.Work.ToString();
}
/// <summary>
/// Add yourself to the list of pending work.
/// </summary>
/// <param name="list">List of pending work.</param>
internal void AddTo(List<BackgroundWorkInfo> list)
{
list.Add(this);
}
}
/// <summary>
/// List of work activities to perform.
/// </summary>
readonly List<BackgroundWorkInfo> pendingWork;
// window regions starting from left-top in order going down
#region JOB_HEADER
@ -352,10 +319,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.InitializeComponent();
this.queueWorker = new BackgroundWorker();
this.queue = new BackgroundWorkQueue(this.queueWorker);
this.queue = new BackgroundWorkQueue(this.queueWorker, this.toolStripStatusLabel_currentWork, this.toolStripStatusLabel_backgroundWork);
this.WarnedAboutDebugging = false;
this.pendingWork = new List<BackgroundWorkInfo>();
this.status = new StatusWriter(this.toolStripStatusLabel, this.statusStrip, this.Status);
this.refreshTimer = new System.Windows.Forms.Timer();
@ -558,16 +524,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.RefreshDisplay();
}
/// <summary>
/// Load information about the current job.
/// </summary>
private void LoadJobDetails()
{
BackgroundWorkInfo work = new BackgroundWorkInfo();
work.Work = BackgroundWorkInfo.WorkKind.LoadJobInfo;
this.StartBackgroundWork(work);
}
/// <summary>
/// Loading the job information has completed.
/// <param name="timeToLoad">Time it took to load the job information.</param>
@ -857,7 +813,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
case ExecutedVertexInstance.VertexState.Failed:
return Color.Tomato;
default:
throw new CalypsoDryadException("Unexpected vertex state " + state);
throw new DryadException("Unexpected vertex state " + state);
}
}
@ -986,22 +942,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
yield return new Tuple<double, Color>(unknown, VertexStateColor(ExecutedVertexInstance.VertexState.Unknown));
}
private static DryadJobStaticPlan CreatePlan(DryadLinqJobInfo job, StatusReporter status)
{
status("Constructing static plan", StatusKind.LongOp);
try
{
var result = JobObjectModel.DryadJobStaticPlan.CreatePlan(job, status);
return result;
}
catch (Exception ex)
{
status("Exception during building of static plan: " + ex.Message, StatusKind.Error);
Trace.TraceInformation(ex.ToString());
return null;
}
}
/// <summary>
/// Refresh and redisplay the query plan.
/// </summary>
@ -1010,7 +950,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.richTextBox_file.Text = "";
var item = new BackgroundWorkItem<DryadJobStaticPlan>(
(s, p) => JobObjectModel.DryadJobStaticPlan.CreatePlan(this.Job, this.Status),
m => JobObjectModel.DryadJobStaticPlan.CreatePlan(this.Job, m),
this.PlanComputed,
"refresh plan");
this.Queue(item);
@ -1677,10 +1617,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// </summary>
/// <param name="path">Cluster object whose contents is read.</param>
/// <param name="pattern">Pattern to filter contents, for folders.</param>
/// <param name="status">Used to report status.</param>
/// <returns>The file contents.</returns>
/// <param name="progress">Progress reporter.</param>
private static FileContents GetContents(StatusReporter status, Action<int> progress, IClusterResidentObject path, string pattern)
/// <param name="manager">Communication manager.</param>
private static FileContents GetContents(CommManager manager, IClusterResidentObject path, string pattern)
{
if (path == null)
{
@ -1705,6 +1644,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
int displayed = 0;
foreach (IClusterResidentObject d in dirs)
{
manager.Token.ThrowIfCancellationRequested();
if (d.Exception != null)
{
error += " [Error " + d.Exception.Message + "]";
@ -1730,7 +1670,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
}
else
{
status("Extracting contents of " + path, StatusKind.LongOp);
manager.Status("Extracting contents of " + path, StatusKind.LongOp);
ISharedStreamReader sr = path.GetStream();
if (sr.Exception != null)
{
@ -1741,7 +1681,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
{
if (path.Size == 0)
error += "[empty]";
var contents = sr.ReadToEnd();
var contents = sr.ReadToEnd(manager.Token);
return new FileContents(contents, error, linkCache);
}
}
@ -1756,7 +1696,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
private void DisplayContents1(IClusterResidentObject path, string pattern)
{
var item = new BackgroundWorkItem<FileContents>(
(s, p) => GetContents(s, p, path, pattern),
m => GetContents(m, path, pattern),
this.ShowContents,
"Read file");
this.Queue(item);
@ -1892,7 +1832,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
{
this.label_title.Text = "Inputs";
this.Status("Discovering vertex channel information", StatusKind.LongOp);
bool found = this.currentVertex.DiscoverChannels(true, false, false, this.Status, this.UpdateProgress);
// TODO: this should run in the background
CommManager manager = new CommManager(this.Status, this.UpdateProgress, new CancellationTokenSource().Token);
bool found = this.currentVertex.DiscoverChannels(true, false, false, manager);
if (found)
{
this.richTextBox_file.SuspendLayout();
@ -1922,7 +1864,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
{
this.label_title.Text = "Outputs";
this.Status("Discovering vertex channel information", StatusKind.LongOp);
bool found = this.currentVertex.DiscoverChannels(false, true, false, this.Status, this.UpdateProgress);
// TODO: this should run in the background
CommManager manager = new CommManager(this.Status, this.UpdateProgress, new CancellationTokenSource().Token);
bool found = this.currentVertex.DiscoverChannels(false, true, false, manager);
if (found)
{
this.richTextBox_file.SuspendLayout();
@ -2319,7 +2263,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
this.Status("Refreshing...", StatusKind.LongOp);
this.Job.InvalidateCaches();
this.stageColorMap = null; // force recomputation
this.LoadJobDetails();
this.RefreshJob();
}
/// <summary>
@ -2340,8 +2284,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
private void JobBrowser_FormClosing(object sender, FormClosingEventArgs e)
{
this.refreshTimer.Stop();
this.pendingWork.Clear();
this.queue.Stop();
this.formSettings.WarnedAboutDebugging = this.WarnedAboutDebugging;
this.formSettings.WarnedAboutProfiling = this.WarnedAboutProfiling;
this.formSettings.Location = this.Location;
@ -2496,107 +2439,37 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// <param name="e">Unused.</param>
private void JobBrowser_Shown(object sender, EventArgs e)
{
this.LoadJobDetails();
}
#region BACKGROUND_WORK
/// <summary>
/// Start performing a piece of background work.
/// </summary>
/// <param name="work">Work to perform.</param>
private void StartBackgroundWork(BackgroundWorkInfo work)
{
if (this.backgroundWorker.IsBusy)
{
work.AddTo(this.pendingWork);
this.toolStripStatusLabel_backgroundWork.Text = this.pendingWork.Count() + " tasks pending.";
this.Status("Queued task for execution", StatusKind.OK);
return;
}
backgroundWorker.RunWorkerAsync(work);
this.RefreshJob();
}
/// <summary>
/// Show the work currently being done.
/// Refresh the job details.
/// </summary>
/// <param name="msg">Message to display (work description).</param>
private void ShowCurrentWork(string msg)
private void RefreshJob()
{
if (this.InvokeRequired)
this.Invoke(new Action<string>(this.ShowCurrentWork), msg);
else
this.toolStripStatusLabel_currentWork.Text = msg;
DryadLinqJobInfo job = this.Job;
DateTime start = DateTime.Now;
var item = new BackgroundWorkItem<TimeSpan>(
m =>
{
job.CollectEssentialInformation(m);
return DateTime.Now - start;
},
this.JobInfoLoaded,
"refreshJob");
this.Queue(item);
}
/// <summary>
/// Perform some background work.
/// Called after a job has been loaded.
/// </summary>
/// <param name="sender">Unused.</param>
/// <param name="e">Event describing the work to perform.</param>
private void backgroundWorker_DoWork(object sender, DoWorkEventArgs e)
/// <param name="cancelled">If true the loading has been cancelled.</param>
/// <param name="loadTime">Time to load job.</param>
private void JobInfoLoaded(bool cancelled, TimeSpan loadTime)
{
BackgroundWorkInfo work = (BackgroundWorkInfo)e.Argument;
work.workStartTime = DateTime.Now;
this.ShowCurrentWork("Doing " + work + ".");
if (cancelled) return;
switch (work.Work)
{
case BackgroundWorkInfo.WorkKind.LoadJobInfo:
{
work.Success = this.Job.CollectEssentialInformation(this.Status, this.UpdateProgress);
break;
}
}
e.Result = work;
}
/// <summary>
/// Background work has terminated.
/// </summary>
/// <param name="sender">Unused.</param>
/// <param name="e">Event describing the result.</param>
private void backgroundWorker_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
{
this.ShowCurrentWork("Doing nothing.");
if (e.Cancelled)
{
this.Status("Background work was cancelled", StatusKind.OK);
goto end;
}
else if (e.Error != null)
{
this.Status("Exception during background work: " + e.Error.Message, StatusKind.Error);
Trace.TraceInformation(e.ToString());
goto end;
}
if (e.Result == null)
{
Trace.TraceInformation("Null result from background work!");
// I don't know why this happens
goto end;
}
BackgroundWorkInfo result = (BackgroundWorkInfo)e.Result;
if (result.Success)
{
switch (result.Work)
{
case BackgroundWorkInfo.WorkKind.LoadJobInfo:
this.LoadJobCompleted(DateTime.Now - result.workStartTime);
break;
}
this.Status("Completed " + result, StatusKind.OK);
}
else
{
// failed in background work
goto end;
}
// do not overwrite the error message if the job did not succeed
// refresh the stage view too
this.LoadJobCompleted(loadTime);
string s = this.currentStage != null ? this.currentStage.Name : null;
if (this.doingStartup && string.IsNullOrEmpty(s))
{
@ -2613,17 +2486,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
{
this.SetTable(this.currentTable.Refresh(this.Job, this.Status, !this.hideCancelledVerticesToolStripMenuItem.Checked));
}
end:
if (this.pendingWork.Any())
{
BackgroundWorkInfo work = this.pendingWork[0];
this.pendingWork.RemoveAt(0);
this.toolStripStatusLabel_backgroundWork.Text = this.pendingWork.Count() + " tasks pending.";
this.StartBackgroundWork(work);
}
}
#endregion
#region MOUSE_DYNAMIC_VIEWS
/// <summary>
@ -2879,7 +2742,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
lv.Show();
var item = new BackgroundWorkItem<bool>(
(s, p) => ScanJMStdout(this.currentVertex, this.Job.ManagerVertex.StdoutFile, lv),
m => ScanJMStdout(this.currentVertex, this.Job.ManagerVertex.StdoutFile, lv),
(c, b) => { },
"findStdout");
this.Queue(item);
@ -2936,7 +2799,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// <param name="e">Unused.</param>
private void diagnoseToolStripMenuItem_Click(object sender, EventArgs e)
{
JobFailureDiagnosis diagnosis = JobFailureDiagnosis.CreateJobFailureDiagnosis(this.Job, this.staticPlan, this.Status, this.UpdateProgress);
CommManager manager = new CommManager(this.Status, this.UpdateProgress, new CancellationToken());
JobFailureDiagnosis diagnosis = JobFailureDiagnosis.CreateJobFailureDiagnosis(this.Job, this.staticPlan, manager);
DiagnosisLog log = diagnosis.Diagnose();
this.DisplayDiagnosis(log);
}
@ -3328,7 +3192,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
ClusterStatus clusterStatus = this.Job.ClusterConfiguration.CreateClusterStatus();
var item = new BackgroundWorkItem<bool>(
(s, p) => ClusterWork.CancelJobs(job, clusterStatus, s, p),
m => ClusterWork.CancelJobs(job, clusterStatus, m),
(c, b) => { },
"cancel");
this.Queue(item);
@ -3344,7 +3208,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
if (this.currentVertex == null)
return;
VertexFailureDiagnosis vfd = VertexFailureDiagnosis.CreateVertexFailureDiagnosis(this.Job, this.staticPlan, this.currentVertex, this.Status, this.UpdateProgress);
// TODO: this should run in the background
CommManager manager = new CommManager(this.Status, this.UpdateProgress, new CancellationToken());
VertexFailureDiagnosis vfd = VertexFailureDiagnosis.CreateVertexFailureDiagnosis(this.Job, this.staticPlan, this.currentVertex, manager);
DiagnosisLog log = vfd.Diagnose();
this.DisplayDiagnosis(log);
}
@ -3387,16 +3253,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// <param name="e">Unused.</param>
private void dataGridView_Scroll(object sender, ScrollEventArgs e)
{
if (e.Type == ScrollEventType.EndScroll)
{
VScrollBar scrollbar = sender as VScrollBar;
if (scrollbar == null)
return;
DataGridView view = scrollbar.Parent as DataGridView;
if (view == null)
return;
view.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.DisplayedCells);
}
}
/// <summary>
@ -3427,9 +3283,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// <param name="e">Unused.</param>
private void cacheAllLogsToolStripMenuItem_Click(object sender, EventArgs e)
{
BackgroundWorkInfo work = new BackgroundWorkInfo();
work.Work = BackgroundWorkInfo.WorkKind.LoadJobInfo;
this.StartBackgroundWork(work);
this.RefreshJob();
IClusterResidentObject folder = this.richtextBoxShownFile;
if (folder == null || folder.Exception != null || !folder.RepresentsAFolder)
@ -3518,7 +3372,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
List<ExecutedVertexInstance> vertices = this.stageData.ToList();
var item = new BackgroundWorkItem<bool>(
(s, p) => CacheAllVertices(this.Job.ClusterConfiguration, this.Job.Summary, vertices, s, p),
m => CacheAllVertices(this.Job.ClusterConfiguration, this.Job.Summary, vertices, m),
(c, b) => { },
"cacheAll");
this.Queue(item);
@ -3528,23 +3382,27 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// Cache the vertices in the list; executed on the background thread.
/// </summary>
/// <returns>True: success.</returns>
/// <param name="manager">Communication manager.</param>
/// <param name="config">Cluster configuration.</param>
/// <param name="summary">Job to cache.</param>
/// <param name="vertices">Vertices to cache.</param>
private static bool CacheAllVertices(
ClusterConfiguration config, DryadLinqJobSummary summary, List<ExecutedVertexInstance> vertices,
StatusReporter status, Action<int> progress)
CommManager manager)
{
int done = 0;
int todo = vertices.Count;
int files = 0;
status("Caching data for " + todo + " vertices", StatusKind.LongOp);
manager.Status("Caching data for " + todo + " vertices", StatusKind.LongOp);
foreach (ExecutedVertexInstance v in vertices)
{
files += CacheVertexInfo(config, summary, v);
done++;
progress(done / todo);
manager.Progress(done / todo);
}
progress(100);
status("Cached " + files + " files", StatusKind.OK);
manager.Progress(100);
manager.Status("Cached " + files + " files", StatusKind.OK);
return true;
}
@ -3576,6 +3434,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
}
ISharedStreamReader reader = file.GetStream();
// ReSharper disable once UnusedVariable
foreach (string line in reader.ReadAllLines())
{
// discard; causes caching
@ -3584,6 +3443,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
}
return cached;
}
private void cancelCurrentWorkToolStripMenuItem_Click(object sender, EventArgs e)
{
this.queue.CancelCurrentWork();
}
}
/// <summary>

147
JobBrowser/JobBrowser/JobBrowser.csproj
View File

@ -1,5 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@ -8,8 +9,8 @@
<ProjectGuid>{EDDD2E0B-A52B-4E25-9436-B874017673FF}</ProjectGuid>
<OutputType>Exe</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>DryadAnalysis</RootNamespace>
<AssemblyName>JobBrowser</AssemblyName>
<RootNamespace>JobBrowser</RootNamespace>
<AssemblyName>DryadLinqBrowser</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<StartupObject>
@ -22,6 +23,7 @@
<UpgradeBackupLocation>
</UpgradeBackupLocation>
<OldToolsVersion>3.5</OldToolsVersion>
<TargetFrameworkProfile />
<PublishUrl>publish\</PublishUrl>
<Install>true</Install>
<InstallFrom>Disk</InstallFrom>
@ -36,29 +38,6 @@
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
<UseApplicationTrust>false</UseApplicationTrust>
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>TRACE;DEBUG</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<DocumentationFile>bin\Debug\JobBrowser.xml</DocumentationFile>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
@ -79,7 +58,48 @@
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup>
<RunPostBuildEvent>OnBuildSuccess</RunPostBuildEvent>
</PropertyGroup>
<ItemGroup>
<Reference Include="EnvDTE, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE.dll</HintPath>
</Reference>
<Reference Include="EnvDTE80, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE80.dll</HintPath>
</Reference>
<Reference Include="EnvDTE90, Version=9.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE90.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Hpc.Scheduler, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\Daphne\Dependences\Microsoft.Hpc.Scheduler.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Hpc.Scheduler.Properties, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\..\Daphne\Dependences\Microsoft.Hpc.Scheduler.Properties.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Msagl, Version=3.0.0.0, Culture=neutral, PublicKeyToken=7f84f738e1fc245c, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
@ -92,13 +112,71 @@
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
</Reference>
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Core">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
</Reference>
<Reference Include="System.Data.Linq" />
<Reference Include="System.Net" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.Extensions">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.Primitives">
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WebRequest" />
<Reference Include="System.Runtime.Serialization" />
<Reference Include="System.ServiceModel" />
<Reference Include="System.Web" />
<Reference Include="System.Windows.Forms" />
<Reference Include="System.Xml.Linq">
<RequiredTargetFramework>3.5</RequiredTargetFramework>
@ -186,11 +264,11 @@
<Name>JobObjectModel</Name>
</ProjectReference>
<ProjectReference Include="..\Tools\Tools.csproj">
<Project>{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}</Project>
<Project>{20b91aaf-aad4-47df-9f1d-494de6e066f9}</Project>
<Name>Tools</Name>
</ProjectReference>
<ProjectReference Include="..\UsefulForms\UsefulForms.csproj">
<Project>{27635A68-ADFC-4192-9262-B4E6ECDDCE09}</Project>
<Project>{27635a68-adfc-4192-9262-b4e6ecddce09}</Project>
<Name>UsefulForms</Name>
</ProjectReference>
</ItemGroup>
@ -234,6 +312,19 @@
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
</Target>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
</Target>
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

3
JobBrowser/JobBrowser/JobBrowser.resx
View File

@ -126,9 +126,6 @@
<metadata name="statusStrip.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<value>17, 17</value>
</metadata>
<metadata name="backgroundWorker.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<value>209, 17</value>
</metadata>
<metadata name="$this.TrayHeight" type="System.Int32, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<value>42</value>
</metadata>

8
JobBrowser/JobBrowser/LocalDebuggingAndProfiling.cs
View File

@ -24,13 +24,13 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Microsoft.Research.Calypso.JobObjectModel;
using Microsoft.Research.Calypso.Tools;
using Microsoft.Research.Calypso.UsefulForms;
using Microsoft.Research.JobObjectModel;
using Microsoft.Research.Tools;
using Microsoft.Research.UsefulForms;
using Microsoft.Win32;
using System.Diagnostics;
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
/// <summary>
/// Class that encapsulates the steps required to debug a vertex locally.

4
JobBrowser/JobBrowser/LogViewer.Designer.cs generated
View File

@ -18,9 +18,9 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License.
*/
using Microsoft.Research.Calypso.Tools;
using Microsoft.Research.Tools;
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
partial class LogViewer
{

22
JobBrowser/JobBrowser/LogViewer.cs
View File

@ -19,15 +19,15 @@ limitations under the License.
*/
using Microsoft.Research.Calypso.JobObjectModel;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Windows.Forms;
using Microsoft.Research.Calypso.Tools;
using Microsoft.Research.JobObjectModel;
using Microsoft.Research.Tools;
namespace Microsoft.Research.Calypso.DryadAnalysis
namespace Microsoft.Research.DryadAnalysis
{
/// <summary>
/// A log viewer displays fragments of logs or other text files.
@ -37,7 +37,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
bool canceled;
DGVData<TextFileLine> shownText;
DGVData<PositionedCosmosLogEntry> shownLogLines;
DGVData<PositionedDryadLogEntry> shownLogLines;
StatusWriter status;
/// <summary>
@ -77,7 +77,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
}
else
{
this.shownLogLines = new DGVData<PositionedCosmosLogEntry>();
this.shownLogLines = new DGVData<PositionedDryadLogEntry>();
this.filteredDataGridView.SetDataSource(this.shownLogLines);
foreach (string s in new string[] { "Malformed", "IsError", "OriginalLogLine", "File", "LineNo" })
{
@ -132,7 +132,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
long bytes = 0;
List<TextFileLine> toAddText = new List<TextFileLine>();
List<PositionedCosmosLogEntry> toAddLog = new List<PositionedCosmosLogEntry>();
List<PositionedDryadLogEntry> toAddLog = new List<PositionedDryadLogEntry>();
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
@ -141,7 +141,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
toAddText.Add(new TextFileLine(lineno, line));
else
{
PositionedCosmosLogEntry cle = new PositionedCosmosLogEntry(filename, lineno, line);
PositionedDryadLogEntry cle = new PositionedDryadLogEntry(filename, lineno, line);
if (cle.Malformed)
{
Trace.TraceInformation("Malformed log entry: " + cle.OriginalLogLine);
@ -204,7 +204,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
}
else
{
PositionedCosmosLogEntry cle = new PositionedCosmosLogEntry(file, lineno, text);
PositionedDryadLogEntry cle = new PositionedDryadLogEntry(file, lineno, text);
if (cle.Malformed)
return;
this.shownLogLines.AddItem(cle);
@ -274,7 +274,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
var rows = this.filteredDataGridView.DataGridView.SelectedRows;
for (int i = 0; i < rows.Count; i++)
{
PositionedCosmosLogEntry entry = ((PositionedCosmosLogEntry)rows[i].DataBoundItem);
PositionedDryadLogEntry entry = ((PositionedDryadLogEntry)rows[i].DataBoundItem);
position += entry.File + ":" + entry.LineNo + Environment.NewLine;
}
MessageBox.Show(position, "File containing log entries");
@ -319,7 +319,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// <summary>
/// Cosmos log entry with position information.
/// </summary>
public class PositionedCosmosLogEntry : CosmosLogEntry
public class PositionedDryadLogEntry : DryadLogEntry
{
/// <summary>
/// File containing the log entry.
@ -336,7 +336,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
/// <param name="file">File containing the log entry.</param>
/// <param name="lineno">Line number.</param>
/// <param name="line">Line contents.</param>
public PositionedCosmosLogEntry(string file, long lineno, string line)
public PositionedDryadLogEntry(string file, long lineno, string line)
: base(line)
{
this.File = file;

12
JobBrowser/JobBrowser/Properties/AssemblyInfo.cs
View File

@ -25,12 +25,12 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("DryadAnalysis")]
[assembly: AssemblyTitle("DryadLinqBrowser")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyProduct("DryadAnalysis")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyCompany("MSIT")]
[assembly: AssemblyProduct("DryadLinqBrowser")]
[assembly: AssemblyCopyright("Copyright © Microsoft")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.0.0")]
[assembly: AssemblyFileVersion("0.1.0.0")]
[assembly: AssemblyVersion("1.0.1.0")]
[assembly: AssemblyFileVersion("1.0.1.0")]

4
JobBrowser/JobBrowser/jobschedule.cs
View File

@ -19,13 +19,13 @@ limitations under the License.
*/
using Microsoft.Research.Calypso.Tools;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using Microsoft.Research.Tools;
namespace Microsoft.Research.Calypso.JobObjectModel
namespace Microsoft.Research.JobObjectModel
{
/// <summary>
/// Information about the dynamic execution schedule of a job.

20
JobBrowser/JobBrowser/packages.config
View File

@ -1,4 +1,22 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
</packages>

48
JobBrowser/Microsoft.Research.JobBrowser.nuspec
View File

@ -1,48 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
<metadata>
<id>Microsoft.Research.Calypso</id>
<version>0.7.0-beta024</version>
<title>Calypso Job Browser for DryadLINQ</title>
<authors>msrsvc</authors>
<owners>msrsvc,Microsoft</owners>
<licenseUrl>http://www.apache.org/licenses/LICENSE-2.0</licenseUrl>
<projectUrl>http://research.microsoft.com/en-us/um/siliconvalley/projects/BigDataDev/</projectUrl>
<requireLicenseAcceptance>true</requireLicenseAcceptance>
<description>The DryadLINQ job browser is a graphical user interface which provides a unified view of a large part of the distributed state of a DryadLINQ job. In this document we provide a brief overview of the main capabilities of the DryadLINQ job browser.</description>
<copyright>© Microsoft Corporation. All rights reserved.</copyright>
<language>en-US</language>
<tags>DryadLINQ Dryad Azure</tags>
<dependencies>
<group targetFramework=".NETFramework4.5">
<dependency id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" />
<dependency id="Microsoft.Research.Peloponnese" version="0.7.0-beta024" />
</group>
</dependencies>
<frameworkAssemblies>
<frameworkAssembly assemblyName="System" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Core" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Data" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Data.DataSetExtensions" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Data.Linq" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Deployment" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Drawing" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Runtime.Serialization" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.ServiceModel" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Windows.Forms" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Xml" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Xml.Linq" targetFramework=".NETFramework4.5" />
</frameworkAssemblies>
</metadata>
<files>
<file src="Calypso\bin\x64\Release\DistributedDataCollection.dll" target="lib\DistributedDataCollection.dll" />
<file src="Calypso\bin\x64\Release\DistributedDataCollection.pdb" target="lib\DistributedDataCollection.pdb" />
<file src="Calypso\bin\x64\Release\JobBrowser.exe" target="lib\JobBrowser.exe" />
<file src="Calypso\bin\x64\Release\JobBrowser.exe.config" target="lib\JobBrowser.exe.config" />
<file src="Calypso\bin\x64\Release\JobBrowser.pdb" target="lib\JobBrowser.pdb" />
<file src="Calypso\bin\x64\Release\Microsoft.Research.Calypso.Tools.dll" target="lib\Microsoft.Research.Calypso.Tools.dll" />
<file src="Calypso\bin\x64\Release\Microsoft.Research.Calypso.Tools.pdb" target="lib\Microsoft.Research.Calypso.Tools.pdb" />
<file src="Calypso\bin\x64\Release\Microsoft.Research.Calypso.UsefulForms.dll" target="lib\Microsoft.Research.Calypso.UsefulForms.dll" />
<file src="Calypso\bin\x64\Release\Microsoft.Research.Calypso.UsefulForms.pdb" target="lib\Microsoft.Research.Calypso.UsefulForms.pdb" />
</files>
</package>

2
JobBrowser/Tools/Graphlayout.cs
View File

@ -24,7 +24,7 @@ using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Drawing2D;
namespace Microsoft.Research.Calypso.Tools
namespace Microsoft.Research.Tools
{
/// <summary>
/// Deals with the plane representation of a graph.

8
JobBrowser/Tools/Properties/AssemblyInfo.cs
View File

@ -28,9 +28,9 @@ using System.Runtime.InteropServices;
[assembly: AssemblyTitle("Tools")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyCompany("MSIT")]
[assembly: AssemblyProduct("Tools")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyCopyright("Copyright © MSIT 2008")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.0.0")]
[assembly: AssemblyFileVersion("0.1.0.0")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

27
JobBrowser/Tools/Tools.csproj
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -9,7 +9,7 @@
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Microsoft.Research.Artemis.Tools</RootNamespace>
<AssemblyName>Microsoft.Research.DryadLinq.JobBrowser.Tools</AssemblyName>
<AssemblyName>Microsoft.Research.Calypso.Tools</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<FileUpgradeFlags>
@ -33,28 +33,6 @@
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<DocumentationFile>bin\Debug\Microsoft.Research.DryadLinq.JobBrowser.Tools.xml</DocumentationFile>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath>
@ -84,6 +62,7 @@
<Reference Include="System.Windows.Forms" />
<Reference Include="System.Data" />
<Reference Include="System.Xml" />
<Reference Include="System.Xml.Linq" />
</ItemGroup>
<ItemGroup>
<Compile Include="DrawingSurface.cs" />

2
JobBrowser/Tools/drawingSurface.cs
View File

@ -19,7 +19,7 @@ limitations under the License.
*/
namespace Microsoft.Research.Calypso.Tools
namespace Microsoft.Research.Tools
{
using System;
using System.Drawing;

2
JobBrowser/Tools/partitionedTables.cs
View File

@ -19,7 +19,7 @@ limitations under the License.
*/
namespace Microsoft.Research.Calypso.Tools
namespace Microsoft.Research.Tools
{
using System.Collections.Generic;
using System.IO;

111
JobBrowser/Tools/tools.cs
View File

@ -31,6 +31,7 @@ using System.Net;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
using System.Windows.Forms;
using System.Xml;
using System.Xml.Serialization;
using System.Text.RegularExpressions;
@ -38,7 +39,7 @@ using System.Threading;
using System.Security.Cryptography;
// Implement here generally-useful tools.
namespace Microsoft.Research.Calypso.Tools
namespace Microsoft.Research.Tools
{
/// <summary>
/// An error handling function.
@ -66,6 +67,38 @@ namespace Microsoft.Research.Calypso.Tools
LongOp,
};
/// <summary>
/// Communication management with background activities.
/// </summary>
public struct CommManager
{
/// <summary>
/// Used to report status.
/// </summary>
public StatusReporter Status;
/// <summary>
/// Used to report progress.
/// </summary>
public Action<int> Progress;
/// <summary>
/// Used to cancel activities.
/// </summary>
public CancellationToken Token;
/// <summary>
/// Create a communication manager.
/// </summary>
/// <param name="status">Status to report errors.</param>
/// <param name="progress">Action to report progress.</param>
/// <param name="token">Token to cancel computations.</param>
public CommManager(StatusReporter status, Action<int> progress, CancellationToken token)
{
this.Status = status;
this.Progress = progress;
this.Token = token;
}
}
/// <summary>
/// Untyped version of work item.
/// </summary>
@ -100,6 +133,10 @@ namespace Microsoft.Research.Calypso.Tools
/// <param name="ex">Exception that occurred during background work (or null).</param>
/// </summary>
void RunContinuation(Exception ex);
/// <summary>
/// Can be used to cancel this work item.
/// </summary>
CancellationTokenSource TokenSource { get; }
}
/// <summary>
@ -111,7 +148,7 @@ namespace Microsoft.Research.Calypso.Tools
/// <summary>
/// Computation to invoke. If the computation is not cancelled the result is passed as the second argument to the continuation.
/// </summary>
public Func<StatusReporter, Action<int>, T> Computation { get; protected set; }
public Func<CommManager, T> Computation { get; protected set; }
/// <summary>
/// Function to call when the work is completed. The first argument is 'true' if the computation was not cancelled. The second argument is the result of the computation.
@ -141,6 +178,10 @@ namespace Microsoft.Research.Calypso.Tools
/// Queue containing item.
/// </summary>
private BackgroundWorkQueue queue;
/// <summary>
/// Source for cancellation token.
/// </summary>
public CancellationTokenSource TokenSource { get; protected set; }
// ReSharper disable ConvertToConstant.Local
bool TraceAsync =
@ -159,7 +200,7 @@ namespace Microsoft.Research.Calypso.Tools
/// <param name="computation">Computation to perform on a background thread. Ideally this should always be a static method.</param>
/// <param name="continuation">Continuation to invoke on the foreground thread when work is done.</param>
/// <param name="description">Description of the background work.</param>
public BackgroundWorkItem(Func<StatusReporter, Action<int>, T> computation, Action<bool, T> continuation, string description)
public BackgroundWorkItem(Func<CommManager, T> computation, Action<bool, T> continuation, string description)
{
this.Description = description;
this.Computation = computation;
@ -167,23 +208,22 @@ namespace Microsoft.Research.Calypso.Tools
this.reporter = null;
this.queue = null;
this.Id = crtid++;
this.TokenSource = new CancellationTokenSource();
}
/// <summary>
/// Perform the background work.
/// </summary>
/// <param name="queue">Worker which does the work.</param>
/// <param name="reporter">Delegate used to report errors.</param>
/// <param name="q">Worker which does the work.</param>
/// <param name="rep">Delegate used to report errors.</param>
/// <param name="progressReporter">Delegate used to report progress.</param>
/// <param name="cancel">If true for an item, cancel it.</param>
// ReSharper disable ParameterHidesMember
public void Queue(BackgroundWorkQueue queue, StatusReporter reporter, Action<int> progressReporter, Func<IBackgroundWorkItem, bool> cancel)
// ReSharper restore ParameterHidesMember
public void Queue(BackgroundWorkQueue q, StatusReporter rep, Action<int> progressReporter, Func<IBackgroundWorkItem, bool> cancel)
{
if (TraceAsync)
Console.WriteLine("{0} Queueing {1}", Utilities.PreciseTime, this.Description);
this.queue = queue;
this.reporter = reporter;
this.queue = q;
this.reporter = rep;
this.progress = progressReporter;
this.queue.CancelMatching(cancel);
this.queue.Enqueue(this);
@ -200,7 +240,8 @@ namespace Microsoft.Research.Calypso.Tools
Console.WriteLine("{0} Running function {1}", Utilities.PreciseTime, this.Description);
try
{
this.Result = this.Computation(this.reporter, this.progress);
CommManager manager = new CommManager(this.reporter, this.progress, this.TokenSource.Token);
this.Result = this.Computation(manager);
}
catch (Exception ex)
{
@ -235,6 +276,7 @@ namespace Microsoft.Research.Calypso.Tools
if (TraceAsync)
Console.WriteLine("{1}/{0}: Cancelling", this.Description, this.Id);
this.Cancelled = true;
this.TokenSource.Cancel();
this.queue.CancelMe(this);
}
@ -266,12 +308,18 @@ namespace Microsoft.Research.Calypso.Tools
/// </summary>
IBackgroundWorkItem current;
private ToolStripStatusLabel currentItemLabel, queueSizeLabel;
/// <summary>
/// Create a background work queue servicing a specified worker.
/// </summary>
/// <param name="worker">Worker to use.</param>
public BackgroundWorkQueue(BackgroundWorker worker)
/// <param name="current">Label where the current work is displayed.</param>
/// <param name="queue">Label where the queue size is displayed.</param>
public BackgroundWorkQueue(BackgroundWorker worker, ToolStripStatusLabel current, ToolStripStatusLabel queue)
{
this.currentItemLabel = current;
this.queueSizeLabel = queue;
if (worker == null)
throw new ArgumentNullException("worker");
this.BackgroundWorker = worker;
@ -280,6 +328,7 @@ namespace Microsoft.Research.Calypso.Tools
this.BackgroundWorker.DoWork += this.worker_DoWork;
this.queue = new List<IBackgroundWorkItem>();
this.current = null;
this.stopped = false;
}
/// <summary>
@ -289,7 +338,7 @@ namespace Microsoft.Research.Calypso.Tools
/// <param name="e">Unused.</param>
void worker_DoWork(object sender, DoWorkEventArgs e)
{
if (this.current == null)
if (this.stopped || this.current == null)
return;
#if DEBUG_WORKQUEUE
#endif
@ -306,6 +355,8 @@ namespace Microsoft.Research.Calypso.Tools
e.Cancel = true;
}
private bool stopped;
/// <summary>
/// Called when the worker is completed.
/// </summary>
@ -319,6 +370,8 @@ namespace Microsoft.Research.Calypso.Tools
#endif
IBackgroundWorkItem crt = this.current;
this.current = null;
if (this.currentItemLabel != null)
this.currentItemLabel.Text = "";
crt.RunContinuation(e.Error);
}
this.Kick();
@ -348,7 +401,11 @@ namespace Microsoft.Research.Calypso.Tools
if (this.current != null)
throw new Exception("current is not null");
this.current = this.queue[0];
if (this.currentItemLabel != null)
this.currentItemLabel.Text = "Doing " + this.current.Description;
this.queue.RemoveAt(0);
if (this.queueSizeLabel != null)
this.queueSizeLabel.Text = "Pending " + this.queue.Count + " items";
this.Start();
}
@ -411,9 +468,22 @@ namespace Microsoft.Research.Calypso.Tools
}
}
/// <summary>
/// Stop the queue.
/// </summary>
public void Stop()
{
// TODO
this.stopped = true;
this.CancelCurrentWork();
}
/// <summary>
/// Cancel the currently running work.
/// </summary>
public void CancelCurrentWork()
{
if (this.current == null) return;
this.current.Cancel();
}
}
@ -3409,7 +3479,8 @@ namespace Microsoft.Research.Calypso.Tools
/// Read the stream to the end from the current position.
/// </summary>
/// <returns>The contents of the stream.</returns>
string ReadToEnd();
/// <param name="token">Can be used to cancel the reading.</param>
string ReadToEnd(CancellationToken token);
/// <summary>
/// Read all the lines remaining in the stream.
@ -3465,11 +3536,15 @@ namespace Microsoft.Research.Calypso.Tools
/// Read the whole stream to the end.
/// </summary>
/// <returns>A string containing the whole contents of the stream.</returns>
public virtual string ReadToEnd()
/// <param name="token">Can be used to cancel the reading.</param>
public virtual string ReadToEnd(CancellationToken token)
{
StringBuilder result = new StringBuilder();
foreach (string s in this.ReadAllLines())
{
token.ThrowIfCancellationRequested();
result.AppendLine(s);
}
return result.ToString();
}
@ -3612,11 +3687,13 @@ namespace Microsoft.Research.Calypso.Tools
/// Read the stream to the end from the current position.
/// </summary>
/// <returns>The contents of the stream.</returns>
public override string ReadToEnd()
/// <param name="token">Can be used to cancel the reading.</param>
public override string ReadToEnd(CancellationToken token)
{
string result = this.actualReader.ReadToEnd();
if (this.cacheWriter != null)
{
token.ThrowIfCancellationRequested();
this.cacheWriter.Write(result);
this.cacheWriter.Close();
if (this.onClose != null)

2
JobBrowser/UsefulForms/FilteredDataGridView.Designer.cs generated
View File

@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License.
*/
namespace Microsoft.Research.Calypso.Tools
namespace Microsoft.Research.Tools
{
partial class FilteredDataGridView
{

2
JobBrowser/UsefulForms/FilteredDataGridView.cs
View File

@ -23,7 +23,7 @@ using System;
using System.Collections.Generic;
using System.Windows.Forms;
namespace Microsoft.Research.Calypso.Tools
namespace Microsoft.Research.Tools
{
/// <summary>
/// A data grid that filters the contents.

2
JobBrowser/UsefulForms/PasswordDialog.Designer.cs generated
View File

@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License.
*/
namespace Microsoft.Research.Calypso.UsefulForms
namespace Microsoft.Research.UsefulForms
{
partial class PasswordDialog
{

2
JobBrowser/UsefulForms/PasswordDialog.cs
View File

@ -21,7 +21,7 @@ limitations under the License.
using System.Windows.Forms;
namespace Microsoft.Research.Calypso.UsefulForms
namespace Microsoft.Research.UsefulForms
{
/// <summary>
/// Prompt user for password.

8
JobBrowser/UsefulForms/Properties/AssemblyInfo.cs
View File

@ -28,9 +28,9 @@ using System.Runtime.InteropServices;
[assembly: AssemblyTitle("UsefulForms")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Microsoft Corporation")]
[assembly: AssemblyCompany("Microsoft")]
[assembly: AssemblyProduct("UsefulForms")]
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
[assembly: AssemblyCopyright("Copyright © Microsoft 2010")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.1.0.0")]
[assembly: AssemblyFileVersion("0.1.0.0")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

2
JobBrowser/UsefulForms/Status.cs
View File

@ -23,7 +23,7 @@ using System.Drawing;
using System;
using System.Windows.Forms;
namespace Microsoft.Research.Calypso.Tools
namespace Microsoft.Research.Tools
{
/// <summary>
/// Delegate used for invoking status messages from accross threads.

2
JobBrowser/UsefulForms/UrlDialog.Designer.cs generated
View File

@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License.
*/
namespace Microsoft.Research.Calypso.UsefulForms
namespace Microsoft.Research.UsefulForms
{
partial class CustomDialog
{

2
JobBrowser/UsefulForms/UrlDialog.cs
View File

@ -19,7 +19,7 @@ limitations under the License.
*/
namespace Microsoft.Research.Calypso.UsefulForms
namespace Microsoft.Research.UsefulForms
{
using System.Windows.Forms;

28
JobBrowser/UsefulForms/UsefulForms.csproj
View File

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@ -9,7 +9,7 @@
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>UsefulForms</RootNamespace>
<AssemblyName>Microsoft.Research.DryadLinq.JobBrowser.UsefulForms</AssemblyName>
<AssemblyName>Microsoft.Research.Calypso.UsefulForms</AssemblyName>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<FileUpgradeFlags>
@ -33,27 +33,6 @@
<BootstrapperEnabled>true</BootstrapperEnabled>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x64\Debug\</OutputPath>
@ -72,6 +51,9 @@
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup>
<RunPostBuildEvent>OnBuildSuccess</RunPostBuildEvent>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Core">

2
JobBrowser/UsefulForms/WarningBox.Designer.cs generated
View File

@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
limitations under the License.
*/
namespace Microsoft.Research.Calypso.UsefulForms
namespace Microsoft.Research.UsefulForms
{
partial class WarningBox
{

2
JobBrowser/UsefulForms/WarningBox.cs
View File

@ -22,7 +22,7 @@ limitations under the License.
using System;
using System.Windows.Forms;
namespace Microsoft.Research.Calypso.UsefulForms
namespace Microsoft.Research.UsefulForms
{
/// <summary>
/// A box displaying a warning, which can be turned off.

BIN
JobBrowser/doc/clusterBrowser.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 25 KiB

BIN
JobBrowser/doc/clusterEditor.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.4 KiB

BIN
JobBrowser/doc/clusterManipulation.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.1 KiB

BIN
JobBrowser/doc/clusterMenu.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.2 KiB

BIN
JobBrowser/doc/dynamicPlan.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.5 KiB

BIN
JobBrowser/doc/filteringVertex.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.9 KiB

BIN
JobBrowser/doc/jobBrowser.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 496 KiB

BIN
JobBrowser/doc/jobBrowserManual.docx
View File

Binary file not shown.

BIN
JobBrowser/doc/jobFiltering.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.8 KiB

BIN
JobBrowser/doc/jobMenu.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.0 KiB

BIN
JobBrowser/doc/jobPlan.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

BIN
JobBrowser/doc/jobSchedule.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 19 KiB

BIN
JobBrowser/doc/jobSummary.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.8 KiB

BIN
JobBrowser/doc/stageCode.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.1 KiB

BIN
JobBrowser/doc/stageFiltering.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.6 KiB

BIN
JobBrowser/doc/stageStatistics.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

BIN
JobBrowser/doc/stageVertices.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 22 KiB

BIN
JobBrowser/doc/staticPlan.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 197 KiB

BIN
JobBrowser/doc/vertexState.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 27 KiB

BIN
JobBrowser/doc/vertexStatistics.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 14 KiB

BIN
JobBrowser/doc/viewMenu.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.0 KiB

27
JobBrowser/packages/Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1/Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1.nuspec vendored
View File

@ -1,27 +0,0 @@
<?xml version="1.0"?>
<package xmlns="http://schemas.microsoft.com/packaging/2013/01/nuspec.xsd">
<metadata minClientVersion="2.5">
<id>Microsoft.Research.DryadLINQ.MSAGL</id>
<version>3.0.0.1</version>
<title>MSAGL Binary Package solely for use with DryadLINQ</title>
<authors>msrsvc</authors>
<owners>msrsvc</owners>
<licenseUrl>http://research.microsoft.com/en-us/um/siliconvalley/projects/BigDataDev/MSAGL_License.txt</licenseUrl>
<projectUrl>http://research.microsoft.com/en-us/projects/msagl/</projectUrl>
<requireLicenseAcceptance>true</requireLicenseAcceptance>
<description>This is a binary package of the Microsoft Automatic Graph Layout engine solely for use with DryadLINQ. Please see the linked license for details.</description>
<copyright>© Microsoft Corporation. All rights reserved.</copyright>
<language>en-US</language>
<frameworkAssemblies>
<frameworkAssembly assemblyName="System" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Core" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Data.DataSetExtensions" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Data.Linq" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Drawing" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Runtime.Serialization" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.ServiceModel" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Xml" targetFramework=".NETFramework4.5" />
<frameworkAssembly assemblyName="System.Xml.Linq" targetFramework=".NETFramework4.5" />
</frameworkAssemblies>
</metadata>
</package>

BIN
JobBrowser/packages/Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1/lib/Microsoft.Msagl.Drawing.dll vendored
View File

Binary file not shown.

BIN
JobBrowser/packages/Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1/lib/Microsoft.Msagl.GraphViewerGdi.dll vendored
View File

Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More