Initial v0.1.2 commit
This commit is contained in:
Michael Isard
parent
cce4d976b2
commit
5ad103cba7
|
|
@ -1,6 +1,6 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
|
|
@ -51,15 +51,36 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions">
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||||
|
|
@ -72,17 +93,29 @@
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
<Reference Include="Newtonsoft.Json">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
|
||||||
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="System" />
|
<Reference Include="System" />
|
||||||
<Reference Include="System.Core" />
|
<Reference Include="System.Core" />
|
||||||
<Reference Include="System.Data.Services.Client" />
|
<Reference Include="System.Data.Services.Client" />
|
||||||
|
<Reference Include="System.Net" />
|
||||||
|
<Reference Include="System.Net.Http" />
|
||||||
|
<Reference Include="System.Net.Http.Extensions">
|
||||||
|
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.Primitives">
|
||||||
|
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.WebRequest" />
|
||||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||||
|
|
@ -105,14 +138,19 @@
|
||||||
<None Include="packages.config" />
|
<None Include="packages.config" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||||
|
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
|
||||||
|
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
|
||||||
|
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
|
||||||
|
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
|
||||||
|
</Target>
|
||||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||||
</Target>
|
</Target>
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||||
Other similar extension points exist, see Microsoft.Common.targets.
|
Other similar extension points exist, see Microsoft.Common.targets.
|
||||||
<Target Name="BeforeBuild">
|
<Target Name="BeforeBuild">
|
||||||
|
|
@ -120,4 +158,4 @@
|
||||||
<Target Name="AfterBuild">
|
<Target Name="AfterBuild">
|
||||||
</Target>
|
</Target>
|
||||||
-->
|
-->
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
|
||||||
// You can specify all the values or you can default the Build and Revision Numbers
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
// by using the '*' as shown below:
|
// by using the '*' as shown below:
|
||||||
// [assembly: AssemblyVersion("1.0.*")]
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
[assembly: AssemblyVersion("0.1.0.0")]
|
[assembly: AssemblyVersion("0.1.2.0")]
|
||||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
[assembly: AssemblyFileVersion("0.1.2.0")]
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,21 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
|
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||||
|
|
@ -71,15 +71,39 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||||
|
|
@ -92,6 +116,10 @@
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||||
|
|
@ -108,6 +136,17 @@
|
||||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="System.Data.Services.Client" />
|
<Reference Include="System.Data.Services.Client" />
|
||||||
|
<Reference Include="System.Net" />
|
||||||
|
<Reference Include="System.Net.Http" />
|
||||||
|
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.WebRequest" />
|
||||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||||
|
|
@ -164,14 +203,19 @@
|
||||||
</ProjectReference>
|
</ProjectReference>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||||
|
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
|
||||||
|
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
|
||||||
|
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
|
||||||
|
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
|
||||||
|
</Target>
|
||||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||||
</Target>
|
</Target>
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||||
Other similar extension points exist, see Microsoft.Common.targets.
|
Other similar extension points exist, see Microsoft.Common.targets.
|
||||||
<Target Name="BeforeBuild">
|
<Target Name="BeforeBuild">
|
||||||
|
|
@ -179,4 +223,4 @@
|
||||||
<Target Name="AfterBuild">
|
<Target Name="AfterBuild">
|
||||||
</Target>
|
</Target>
|
||||||
-->
|
-->
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
|
||||||
// You can specify all the values or you can default the Build and Revision Numbers
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
// by using the '*' as shown below:
|
// by using the '*' as shown below:
|
||||||
// [assembly: AssemblyVersion("1.0.*")]
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
[assembly: AssemblyVersion("0.1.0.0")]
|
[assembly: AssemblyVersion("0.1.2.0")]
|
||||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
[assembly: AssemblyFileVersion("0.1.2.0")]
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,21 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
|
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
@ -0,0 +1,270 @@
|
||||||
|
using Microsoft.Research.DryadLinq;
|
||||||
|
using Microsoft.Research.Peloponnese.Storage;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
namespace DryadLinqTests
|
||||||
|
{
|
||||||
|
public class ApplyAndForkTests
|
||||||
|
{
|
||||||
|
public static IEnumerable<int> NonHomomorphic_Unary_Func(IEnumerable<int> input)
|
||||||
|
{
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
// [DistributiveOverConcat]
|
||||||
|
public static IEnumerable<int> Homomorphic_Unary_Func(IEnumerable<int> input)
|
||||||
|
{
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IEnumerable<int> NonHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
|
||||||
|
{
|
||||||
|
return left;
|
||||||
|
}
|
||||||
|
|
||||||
|
// [LeftDistributiveOverConcat]
|
||||||
|
public static IEnumerable<int> LeftHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
|
||||||
|
{
|
||||||
|
return left;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Note: an apply function must only consume each enumerable once, and it must produce an enumerable
|
||||||
|
// So for a simple pass-through function that does a little work, we must enumerate only once.
|
||||||
|
// Else we get the error: "An HpcLinq channel can't be read more than once."
|
||||||
|
// [DistributiveOverConcat]
|
||||||
|
public static IEnumerable<int> FullHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
|
||||||
|
{
|
||||||
|
long cLeft = 0;
|
||||||
|
foreach (int x in left)
|
||||||
|
{
|
||||||
|
cLeft++;
|
||||||
|
yield return x;
|
||||||
|
}
|
||||||
|
|
||||||
|
long cRight = 0;
|
||||||
|
foreach (int x in right)
|
||||||
|
{
|
||||||
|
cRight++;
|
||||||
|
yield return x;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cLeft == 0)
|
||||||
|
throw new Exception("a node received empty left-data");
|
||||||
|
|
||||||
|
if (cRight == 0)
|
||||||
|
throw new Exception("a node received empty right-data");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static bool NonHomomorphicUnaryApply()
|
||||||
|
{
|
||||||
|
var context = new DryadLinqContext(Config.cluster);
|
||||||
|
context.LocalExecution = false;
|
||||||
|
bool passed = true;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
string outFile = "unittest/output/NonHomomorphicUnaryApply.out";
|
||||||
|
|
||||||
|
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
|
||||||
|
"unittest/inputdata/SimpleFile.txt"));
|
||||||
|
|
||||||
|
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
|
||||||
|
IQueryable<int> pt1 = simple.Select(x => x.First());
|
||||||
|
|
||||||
|
var q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
|
||||||
|
var jobInfo = q1.ToStore<int>(outFile).Submit();
|
||||||
|
jobInfo.Wait();
|
||||||
|
|
||||||
|
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
|
||||||
|
}
|
||||||
|
catch (DryadLinqException)
|
||||||
|
{
|
||||||
|
passed &= false;
|
||||||
|
}
|
||||||
|
return passed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool HomomorphicUnaryApply()
|
||||||
|
{
|
||||||
|
var context = new DryadLinqContext(Config.cluster);
|
||||||
|
context.LocalExecution = false;
|
||||||
|
bool passed = true;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
string outFile = "unittest/output/HomomorphicUnaryApply.out";
|
||||||
|
|
||||||
|
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
|
||||||
|
"unittest/inputdata/SimpleFile.txt"));
|
||||||
|
|
||||||
|
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
|
||||||
|
IQueryable<int> pt1 = simple.Select(x => x.First());
|
||||||
|
|
||||||
|
var q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
|
||||||
|
var jobInfo = q1.ToStore<int>(outFile).Submit();
|
||||||
|
jobInfo.Wait();
|
||||||
|
|
||||||
|
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
|
||||||
|
}
|
||||||
|
catch (DryadLinqException)
|
||||||
|
{
|
||||||
|
passed &= false;
|
||||||
|
}
|
||||||
|
return passed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool NonHomomorphicBinaryApply()
|
||||||
|
{
|
||||||
|
var context = new DryadLinqContext(Config.cluster);
|
||||||
|
context.LocalExecution = false;
|
||||||
|
bool passed = true;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
string outFile = "unittest/output/NonHomomorphicBinaryApply.out";
|
||||||
|
|
||||||
|
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
|
||||||
|
"unittest/inputdata/SimpleFile.txt"));
|
||||||
|
|
||||||
|
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
|
||||||
|
IQueryable<int> pt1 = simple.Select(x => x.First());
|
||||||
|
|
||||||
|
var q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
|
||||||
|
var jobInfo = q1.ToStore<int>(outFile).Submit();
|
||||||
|
jobInfo.Wait();
|
||||||
|
|
||||||
|
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
|
||||||
|
}
|
||||||
|
catch (DryadLinqException)
|
||||||
|
{
|
||||||
|
passed &= false;
|
||||||
|
}
|
||||||
|
return passed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool LeftHomomorphicBinaryApply()
|
||||||
|
{
|
||||||
|
var context = new DryadLinqContext(Config.cluster);
|
||||||
|
context.LocalExecution = false;
|
||||||
|
bool passed = true;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
string outFile = "unittest/output/LeftHomomorphicBinaryApply.out";
|
||||||
|
|
||||||
|
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
|
||||||
|
"unittest/inputdata/SimpleFile.txt"));
|
||||||
|
|
||||||
|
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
|
||||||
|
IQueryable<int> pt1 = simple.Select(x => x.First());
|
||||||
|
|
||||||
|
var q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
|
||||||
|
var jobInfo = q1.ToStore<int>(outFile).Submit();
|
||||||
|
jobInfo.Wait();
|
||||||
|
|
||||||
|
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
|
||||||
|
}
|
||||||
|
catch (DryadLinqException)
|
||||||
|
{
|
||||||
|
passed &= false;
|
||||||
|
}
|
||||||
|
return passed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool FullHomomorphicBinaryApply_DifferentDataSets()
|
||||||
|
{
|
||||||
|
var context = new DryadLinqContext(Config.cluster);
|
||||||
|
context.LocalExecution = false;
|
||||||
|
bool passed = true;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
string outFile = "unittest/output/FullHomomorphicBinaryApply_DifferentDataSets.out";
|
||||||
|
|
||||||
|
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
|
||||||
|
"unittest/inputdata/SimpleFile.txt"));
|
||||||
|
|
||||||
|
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
|
||||||
|
IQueryable<int> pt1 = simple.Select(x => x.First());
|
||||||
|
IQueryable<int> pt2 = simple.Select(x => x.First());
|
||||||
|
|
||||||
|
var q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
|
||||||
|
var jobInfo = q1.ToStore<int>(outFile).Submit();
|
||||||
|
jobInfo.Wait();
|
||||||
|
|
||||||
|
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
|
||||||
|
}
|
||||||
|
catch (DryadLinqException)
|
||||||
|
{
|
||||||
|
passed &= false;
|
||||||
|
}
|
||||||
|
return passed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool FullHomomorphicBinaryApply_IdenticalDataSets()
|
||||||
|
{
|
||||||
|
var context = new DryadLinqContext(Config.cluster);
|
||||||
|
context.LocalExecution = false;
|
||||||
|
bool passed = true;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
string outFile = "unittest/output/FullHomomorphicBinaryApply_2.out";
|
||||||
|
|
||||||
|
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
|
||||||
|
"unittest/inputdata/SimpleFile.txt"));
|
||||||
|
|
||||||
|
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
|
||||||
|
IQueryable<int> pt1 = simple.Select(x => x.First());
|
||||||
|
var q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
|
||||||
|
var jobInfo = q1.ToStore<int>(outFile).Submit();
|
||||||
|
jobInfo.Wait();
|
||||||
|
|
||||||
|
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
|
||||||
|
}
|
||||||
|
catch (DryadLinqException)
|
||||||
|
{
|
||||||
|
passed &= false;
|
||||||
|
}
|
||||||
|
return passed;
|
||||||
|
}
|
||||||
|
|
||||||
|
[Associative(typeof(AssociativeRecursive1))]
|
||||||
|
public static string IntToStringCSVAggregator(string agg, int next)
|
||||||
|
{
|
||||||
|
return agg + "," + next.ToString();
|
||||||
|
}
|
||||||
|
public class AssociativeRecursive1 : IAssociative<string>
|
||||||
|
{
|
||||||
|
public string Seed()
|
||||||
|
{
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
public string RecursiveAccumulate(string first, string second)
|
||||||
|
{
|
||||||
|
return first + second;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool Aggregate_WithCombiner()
|
||||||
|
{
|
||||||
|
var context = new DryadLinqContext(Config.cluster);
|
||||||
|
context.LocalExecution = false;
|
||||||
|
bool passed = true;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
|
||||||
|
"unittest/inputdata/SimpleFile.txt"));
|
||||||
|
|
||||||
|
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
|
||||||
|
IQueryable<int> pt1 = simple.Select(x => x.First());
|
||||||
|
|
||||||
|
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
|
||||||
|
|
||||||
|
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
|
||||||
|
}
|
||||||
|
catch (DryadLinqException)
|
||||||
|
{
|
||||||
|
passed &= false;
|
||||||
|
}
|
||||||
|
return passed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,6 +1,6 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
|
|
@ -63,37 +63,78 @@
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Hadoop.Client">
|
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge">
|
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Research.Peloponnese.Utils">
|
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions">
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
|
<Reference Include="Microsoft.WindowsAzure.Management">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Newtonsoft.Json">
|
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="System" />
|
<Reference Include="System" />
|
||||||
<Reference Include="System.Core" />
|
<Reference Include="System.Core" />
|
||||||
<Reference Include="System.Data.Services.Client" />
|
<Reference Include="System.Data.Services.Client" />
|
||||||
|
<Reference Include="System.Net" />
|
||||||
|
<Reference Include="System.Net.Http" />
|
||||||
|
<Reference Include="System.Net.Http.Extensions">
|
||||||
|
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.Primitives">
|
||||||
|
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.WebRequest" />
|
||||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||||
|
|
@ -106,8 +147,14 @@
|
||||||
<Reference Include="System.Xml" />
|
<Reference Include="System.Xml" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
<Compile Include="ApplyAndForkTests.cs" />
|
||||||
|
<Compile Include="BasicAPITests.cs" />
|
||||||
|
<Compile Include="GroupByReduceTests.cs" />
|
||||||
<Compile Include="Program.cs" />
|
<Compile Include="Program.cs" />
|
||||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||||
|
<Compile Include="RangePartitionAPICoverageTests.cs" />
|
||||||
|
<Compile Include="Utils.cs" />
|
||||||
|
<Compile Include="Validate.cs" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="App.config" />
|
<None Include="App.config" />
|
||||||
|
|
@ -137,14 +184,19 @@
|
||||||
</BootstrapperPackage>
|
</BootstrapperPackage>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||||
|
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
|
||||||
|
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
|
||||||
|
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
|
||||||
|
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
|
||||||
|
</Target>
|
||||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||||
</Target>
|
</Target>
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||||
Other similar extension points exist, see Microsoft.Common.targets.
|
Other similar extension points exist, see Microsoft.Common.targets.
|
||||||
<Target Name="BeforeBuild">
|
<Target Name="BeforeBuild">
|
||||||
|
|
@ -152,4 +204,4 @@
|
||||||
<Target Name="AfterBuild">
|
<Target Name="AfterBuild">
|
||||||
</Target>
|
</Target>
|
||||||
-->
|
-->
|
||||||
</Project>
|
</Project>
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -21,6 +21,7 @@ using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
using System.IO;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using Microsoft.Research.DryadLinq;
|
using Microsoft.Research.DryadLinq;
|
||||||
|
|
||||||
|
|
@ -32,64 +33,32 @@ namespace DryadLinqTests
|
||||||
{
|
{
|
||||||
public static void Main(string[] args)
|
public static void Main(string[] args)
|
||||||
{
|
{
|
||||||
//Test1(args);
|
//BasicAPITests.ToStoreThrowsForNonQuery();
|
||||||
//Test2(args);
|
//BasicAPITests.ToStoreGetEnumeratorThrows();
|
||||||
//Test3(args);
|
//BasicAPITests.GetEnumeratorNonToStoreTerminated();
|
||||||
//Test4(args);
|
//BasicAPITests.ToStoreSubmitGetEnumerator();
|
||||||
//Test5(args);
|
//BasicAPITests.SubmitNonToStoreTerminated();
|
||||||
}
|
//BasicAPITests.MaterializeToStoreTerminated();
|
||||||
|
//BasicAPITests.MaterializeNonToStoreTerminated();
|
||||||
|
//BasicAPITests.EnumeratePlainData();
|
||||||
|
|
||||||
public static void Test1(string[] args)
|
//BasicAPITests.RepeatSubmit();
|
||||||
{
|
//BasicAPITests.RepeatMaterialize();
|
||||||
DryadLinqContext context = new DryadLinqContext(1, "partfile");
|
//BasicAPITests.MaterializeMentionsSameQueryTwice();
|
||||||
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
|
//BasicAPITests.QueryOnDataBackedDLQ();
|
||||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
//BasicAPITests.Bug11781_CountandFirstOrDefault();
|
||||||
var lines = input.Where(x => x.Line.Contains("white"));
|
//BasicAPITests.Bug11782_Aggregate();
|
||||||
var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
|
//BasicAPITests.Bug11782_LowLevelQueryableManipulation();
|
||||||
result.SubmitAndWait();
|
//BasicAPITests.Bug11638_LongWhere();
|
||||||
}
|
//BasicAPITests.AssumeRangePartition();
|
||||||
|
//BasicAPITests.Bug11638_LongMethods();
|
||||||
|
//BasicAPITests.ContextConfigIsReadOnly();
|
||||||
|
//BasicAPITests.ToggleSpeculativeDuplication();
|
||||||
|
//BasicAPITests.Bug15068_ConfigResourcesAPI();
|
||||||
|
//BasicAPITests.Bug_16341_SubmitThrowsForDifferentContexts();
|
||||||
|
//BasicAPITests.Bug14449_ContextShouldExposeVersionIDs();
|
||||||
|
//BasicAPITests.Bug_16341_VariousTestsForSubmit();
|
||||||
|
|
||||||
public static void Test2(string[] args)
|
|
||||||
{
|
|
||||||
DryadLinqContext context = new DryadLinqContext(1, "partfile:///d:/DryadLinqTemp/PartFiles");
|
|
||||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
|
||||||
var q1 = input.Where(x => x.Line.Contains("white"));
|
|
||||||
var q2 = input.Where(x => x.Line.Contains("the"));
|
|
||||||
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
|
|
||||||
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
|
|
||||||
DryadLinqQueryable.SubmitAndWait(res1, res2);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void Test3(string[] args)
|
|
||||||
{
|
|
||||||
DryadLinqContext context = new DryadLinqContext(1, "partfile:///d:/DryadLinqTemp/PartFiles");
|
|
||||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
|
||||||
var words = input.SelectMany(x => x.Line.Split(' '));
|
|
||||||
var groups = words.GroupBy(x => x);
|
|
||||||
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
|
|
||||||
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
|
|
||||||
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
|
|
||||||
result.SubmitAndWait();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void Test4(string[] args)
|
|
||||||
{
|
|
||||||
DryadLinqContext context = new DryadLinqContext("svc-d2-01");
|
|
||||||
var input = context.FromStore<LineRecord>("hdfs://svc-d2-01:8033/user/misard/foo.txt");
|
|
||||||
var lines = input.Where(x => x.Line.Contains("white"));
|
|
||||||
var result = lines.ToStore("hdfs://svc-d2-01:8033/user/yuanbyu/foo.txt", true);
|
|
||||||
result.SubmitAndWait();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void Test5(string[] args)
|
|
||||||
{
|
|
||||||
DryadLinqContext context = new DryadLinqContext(1);
|
|
||||||
Uri dataUri = AzureUtils.ToAzureUri("msrsvc", "I4JPlk0bZ6YWypg+RJamyq0us1b+kCcuoeKlPhfiHTcVW7P4xvuzURvlRShSo1O3UDhcL2LiY4kMaarD+p1lKg==", "test", "testwrite");
|
|
||||||
IEnumerable<LineRecord> lines = DataProvider.ReadData<LineRecord>(context, dataUri);
|
|
||||||
foreach (var x in lines)
|
|
||||||
{
|
|
||||||
Console.WriteLine(x);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
|
||||||
// You can specify all the values or you can default the Build and Revision Numbers
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
// by using the '*' as shown below:
|
// by using the '*' as shown below:
|
||||||
// [assembly: AssemblyVersion("1.0.*")]
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
[assembly: AssemblyVersion("0.1.0.0")]
|
[assembly: AssemblyVersion("0.1.2.0")]
|
||||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
[assembly: AssemblyFileVersion("0.1.2.0")]
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace DryadLinqTests
|
||||||
|
{
|
||||||
|
public class RangePartitionAPICoverageTests
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,157 @@
|
||||||
|
using Microsoft.Research.DryadLinq;
|
||||||
|
using Microsoft.WindowsAzure.Storage;
|
||||||
|
using Microsoft.WindowsAzure.Storage.Blob;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
|
||||||
|
namespace DryadLinqTests
|
||||||
|
{
|
||||||
|
public static class Config
|
||||||
|
{
|
||||||
|
public static string accountName = @"MyAccountName";
|
||||||
|
public static string storageKey = @"MyStorageKey";
|
||||||
|
public static string containerName = @"MyContainerName";
|
||||||
|
public static string cluster = "MyCcluster";
|
||||||
|
}
|
||||||
|
|
||||||
|
public class DataGenerator
|
||||||
|
{
|
||||||
|
public DataGenerator()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IEnumerable<IEnumerable<int>> CreateSimpleFileSets()
|
||||||
|
{
|
||||||
|
IEnumerable<IEnumerable<int>> data = new int[][]
|
||||||
|
{
|
||||||
|
new[] { 1, 2, 3, 4 },
|
||||||
|
new[] { 5, 6, 7, 8 },
|
||||||
|
new[] { 9, 10, 11, 12 },
|
||||||
|
};
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IEnumerable<IEnumerable<int>> CreateGroupByReduceDataSet()
|
||||||
|
{
|
||||||
|
// we need quite a few elements to ensure the combiner will be activated in Stage#1 groupBy.
|
||||||
|
// 33 elements per partition should suffice, but 100 per partition is safer.
|
||||||
|
IEnumerable<IEnumerable<int>> data = new int[][]
|
||||||
|
{
|
||||||
|
Enumerable.Range(1,100).ToArray(),
|
||||||
|
Enumerable.Range(101,100).ToArray(),
|
||||||
|
};
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public class Utils
|
||||||
|
{
|
||||||
|
public static bool DeleteFile(string accountName, string accountKey, string containerName, string fileName, bool delSubDirs)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + accountName + ";AccountKey=" + accountKey);
|
||||||
|
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
|
||||||
|
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
|
||||||
|
container.CreateIfNotExists();
|
||||||
|
BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
|
||||||
|
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
|
||||||
|
container.SetPermissions(containerPermissions);
|
||||||
|
|
||||||
|
if (false == delSubDirs)
|
||||||
|
{
|
||||||
|
CloudBlockBlob remoteFile = container.GetBlockBlobReference(fileName);
|
||||||
|
remoteFile.DeleteIfExists();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (true == delSubDirs)
|
||||||
|
{
|
||||||
|
foreach (IListBlobItem item in container.ListBlobs(fileName, true))
|
||||||
|
{
|
||||||
|
CloudBlockBlob blob = (CloudBlockBlob)item;
|
||||||
|
blob.DeleteIfExists();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
public static bool FileExists(string accountName, string accountKey, string containerName, string fileName)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + accountName + ";AccountKey=" + accountKey);
|
||||||
|
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
|
||||||
|
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
|
||||||
|
container.CreateIfNotExists();
|
||||||
|
BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
|
||||||
|
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
|
||||||
|
container.SetPermissions(containerPermissions);
|
||||||
|
|
||||||
|
CloudBlockBlob remoteFile = container.GetBlockBlobReference(fileName);
|
||||||
|
if (!remoteFile.Exists())
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
catch (Exception)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static DryadLinqContext MakeBasicConfig(string cluster) //???
|
||||||
|
{
|
||||||
|
var context = new DryadLinqContext(cluster);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
context.JobFriendlyName = "DryadLinq_DevUnitTests";
|
||||||
|
context.CompileForVertexDebugging = true;
|
||||||
|
context.JobEnvironmentVariables.Add("DummyEnvVar", "hello"); //note: this is consumed by a unit-test.
|
||||||
|
|
||||||
|
if (File.Exists("Microsoft.Hpc.Linq.pdb")) // TODO: fix references
|
||||||
|
{
|
||||||
|
context.ResourcesToAdd.Add("Microsoft.Hpc.Linq.pdb");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (File.Exists("Microsoft.Hpc.Dsc.Client.pdb")) // TODO: fix references
|
||||||
|
{
|
||||||
|
context.ResourcesToAdd.Add("Microsoft.Hpc.Dsc.Client.pdb");
|
||||||
|
}
|
||||||
|
|
||||||
|
// To prevent job from running forever, and blocking other test
|
||||||
|
context.JobRuntimeLimit = (int)TimeSpan.FromMinutes(30).TotalSeconds;
|
||||||
|
|
||||||
|
|
||||||
|
//config.AllowConcurrentUserDelegatesInSingleProcess = false;
|
||||||
|
|
||||||
|
// If we are on Azure, we have to set the nodeGroup to "NodeRole" so that the default of "ComputeNodes" is not used
|
||||||
|
// This fixes "FromEnumerableTests" on Azure which queries the active node-group.
|
||||||
|
// Note also, the headnode for an azure deployment defaults to "HPCCluster" (at least from James' script)
|
||||||
|
int onAzureInt = 0;
|
||||||
|
string onAzureString = Environment.GetEnvironmentVariable("CCP_SCHEDULERONAZURE");
|
||||||
|
if (onAzureString != null)
|
||||||
|
{
|
||||||
|
int.TryParse(onAzureString, out onAzureInt);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (onAzureInt == 1)
|
||||||
|
{
|
||||||
|
context.NodeGroup = "NodeRole";
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
catch (DryadLinqException)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
return context;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,267 @@
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
// file: Validate.cs
|
||||||
|
//
|
||||||
|
// summary: Implements the validate class
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
namespace BenchmarkFramework {
|
||||||
|
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
/// <summary> Validation utils </summary>
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public class Validate {
|
||||||
|
|
||||||
|
public static void
|
||||||
|
Check<T>(
|
||||||
|
IEnumerable<T>[] ss,
|
||||||
|
IComparer<T> comparer = null,
|
||||||
|
bool sort = true,
|
||||||
|
bool verbose = false,
|
||||||
|
IComparer<T> sortcomparer = null
|
||||||
|
) {
|
||||||
|
|
||||||
|
if(ss.Length == 0) return;
|
||||||
|
|
||||||
|
if(comparer == null) {
|
||||||
|
comparer = Comparer<T>.Default;
|
||||||
|
if(comparer == null) {
|
||||||
|
throw new ArgumentNullException("Can't not be null.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(sortcomparer == null)
|
||||||
|
sortcomparer = comparer;
|
||||||
|
|
||||||
|
T[][] aa = new T[ss.Length][];
|
||||||
|
for(int i = 0; i < aa.Length; i++) {
|
||||||
|
aa[i] = ss[i].ToArray();
|
||||||
|
if(sort) Array.Sort(aa[i], sortcomparer);
|
||||||
|
}
|
||||||
|
int len = aa[0].Length;
|
||||||
|
for(int i = 1; i < aa.Length; i++) {
|
||||||
|
if(aa[i].Length != len) {
|
||||||
|
throw new Exception("Wrong number of elements.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for(int i = 0; i < len; i++) {
|
||||||
|
T elem = aa[0][i];
|
||||||
|
for(int j = 1; j < aa.Length; j++) {
|
||||||
|
if(verbose) {
|
||||||
|
//TestOutput.WriteLine("Comparing {0} to {1}", elem.ToString(), aa[j][i].ToString());
|
||||||
|
}
|
||||||
|
if(comparer.Compare(elem, aa[j][i]) != 0) {
|
||||||
|
throw new Exception("Elements failed to match: " + elem + " != " + aa[j][i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//public static void
|
||||||
|
//GroupCheck<K, V>(
|
||||||
|
// IEnumerable<IGrouping<K, V>>[] ss,
|
||||||
|
// IComparer<K> kComparer = null,
|
||||||
|
// IComparer<V> vComparer = null
|
||||||
|
// ) {
|
||||||
|
|
||||||
|
// if(ss.Length == 0) return;
|
||||||
|
|
||||||
|
// if(kComparer == null) {
|
||||||
|
// kComparer = Comparer<K>.Default;
|
||||||
|
// if(kComparer == null) {
|
||||||
|
// throw new ArgumentNullException("Can't not be null.");
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// if(vComparer == null) {
|
||||||
|
// vComparer = Comparer<V>.Default;
|
||||||
|
// if(vComparer == null) {
|
||||||
|
// throw new ArgumentNullException("Can't not be null.");
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// IGrouping<K, V>[][] aa = new IGrouping<K, V>[ss.Length][];
|
||||||
|
// for(int i = 0; i < aa.Length; i++) {
|
||||||
|
// aa[i] = ss[i].ToArray();
|
||||||
|
// K[] keys = aa[i].Select(x => x.Key).ToArray();
|
||||||
|
// Array.Sort(keys, aa[i], kComparer);
|
||||||
|
// }
|
||||||
|
// int len = aa[0].Length;
|
||||||
|
// for(int i = 1; i < aa.Length; i++) {
|
||||||
|
// if(aa[i].Length != len) {
|
||||||
|
// throw new Exception("Wrong number of elements.");
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// for(int i = 0; i < len; i++) {
|
||||||
|
// IEnumerable<V> elem = aa[0][i];
|
||||||
|
// for(int j = 1; j < aa.Length; j++) {
|
||||||
|
// Check(new IEnumerable<V>[] { elem, aa[j][i] }, vComparer);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//}
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
/// <summary> Tolerant float comparer. Floating point differences between
|
||||||
|
/// GPU and CPU cause the default comparer to fail sometimes even when
|
||||||
|
/// the result is correct. Use this comparer to introduce some tolerance
|
||||||
|
/// for this
|
||||||
|
/// </summary>
|
||||||
|
///
|
||||||
|
/// <remarks> Crossbac, 2/19/2013. </remarks>
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public class TolerantDoubleComparer : IComparer<double> {
|
||||||
|
Double EPSILON;
|
||||||
|
public TolerantDoubleComparer(Double _epsilon = 0.000001f) {
|
||||||
|
EPSILON = _epsilon;
|
||||||
|
}
|
||||||
|
public int Compare(Double a, Double b) {
|
||||||
|
Double delta = a - b;
|
||||||
|
if(Math.Abs(delta) <= EPSILON)
|
||||||
|
return 0;
|
||||||
|
return delta < 0.0f ? -1 : 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
/// <summary> Tolerant float comparer. Floating point differences between
|
||||||
|
/// GPU and CPU cause the default comparer to fail sometimes even when
|
||||||
|
/// the result is correct. Use this comparer to introduce some tolerance
|
||||||
|
/// for this
|
||||||
|
/// </summary>
|
||||||
|
///
|
||||||
|
/// <remarks> Crossbac, 2/19/2013. </remarks>
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public class TolerantFloatComparer : IComparer<float> {
|
||||||
|
float EPSILON;
|
||||||
|
public TolerantFloatComparer(float _epsilon = 0.000001f) {
|
||||||
|
EPSILON = _epsilon;
|
||||||
|
}
|
||||||
|
public int Compare(float a, float b) {
|
||||||
|
float delta = a - b;
|
||||||
|
if(Math.Abs(delta) <= EPSILON)
|
||||||
|
return 0;
|
||||||
|
return delta < 0.0f ? -1 : 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
/// <summary> Tolerant float comparer. Floating point differences between
|
||||||
|
/// GPU and CPU cause the default comparer to fail sometimes even when
|
||||||
|
/// the result is correct. Use this comparer to introduce some tolerance
|
||||||
|
/// for this
|
||||||
|
/// </summary>
|
||||||
|
///
|
||||||
|
/// <remarks> Crossbac, 2/19/2013. </remarks>
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public class TolerantVectorComparer : IComparer<Vector> {
|
||||||
|
float EPSILON;
|
||||||
|
public TolerantVectorComparer(float _epsilon = 0.0001f) {
|
||||||
|
EPSILON = _epsilon;
|
||||||
|
}
|
||||||
|
public int Compare(Vector a, Vector b) {
|
||||||
|
for(int i = 0; i < a.m_elems.Length; i++) {
|
||||||
|
float delta = a.m_elems[i] - b.m_elems[i];
|
||||||
|
if(Math.Abs(delta) > EPSILON)
|
||||||
|
return delta < 0.0f ? -1 : 1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
/// <summary> Interface for epsilon comparable single. </summary>
|
||||||
|
///
|
||||||
|
/// <remarks> Crossbac, 1/16/2014. </remarks>
|
||||||
|
///
|
||||||
|
/// <typeparam name="T"> Generic type parameter. </typeparam>
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public interface IEpsilonComparableSingle<T> {
|
||||||
|
int EpsilonCompare(T a, T b, float epsilon);
|
||||||
|
}
|
||||||
|
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
/// <summary> Interface for epsilon comparable double. </summary>
|
||||||
|
///
|
||||||
|
/// <remarks> Crossbac, 1/16/2014. </remarks>
|
||||||
|
///
|
||||||
|
/// <typeparam name="T"> Generic type parameter. </typeparam>
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public interface IEpsilonComparableDouble<T> {
|
||||||
|
int EpsilonCompare(T a, T b, double epsilon);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
/// <summary> Tolerant float comparer. Floating point differences between
|
||||||
|
/// GPU and CPU cause the default comparer to fail sometimes even when
|
||||||
|
/// the result is correct. Use this comparer to introduce some tolerance
|
||||||
|
/// for this
|
||||||
|
/// </summary>
|
||||||
|
///
|
||||||
|
/// <remarks> Crossbac, 2/19/2013. </remarks>
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public class EpsilonComparer<T> : IComparer<T> where T : IEpsilonComparableSingle<T> {
|
||||||
|
float EPSILON;
|
||||||
|
public EpsilonComparer(float _epsilon = 0.0001f) {
|
||||||
|
EPSILON = _epsilon;
|
||||||
|
}
|
||||||
|
public int Compare(T a, T b) {
|
||||||
|
return a.EpsilonCompare(a, b, EPSILON);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
/// <summary> Tolerant float comparer for images. Floating point differences between
|
||||||
|
/// GPU and CPU cause the default comparer to fail sometimes even when
|
||||||
|
/// the result is correct. Use this comparer to introduce some tolerance
|
||||||
|
/// for this
|
||||||
|
/// </summary>
|
||||||
|
///
|
||||||
|
/// <remarks> Crossbac, 2/19/2013. </remarks>
|
||||||
|
///-------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public class TolerantImageComparer : IComparer<Image> {
|
||||||
|
float EPSILON;
|
||||||
|
public TolerantImageComparer(float _epsilon = 0.0001f) {
|
||||||
|
EPSILON = _epsilon;
|
||||||
|
}
|
||||||
|
public int Compare(Image a, Image b) {
|
||||||
|
for(int i = 0; i < a.m_elems.Length; i++) {
|
||||||
|
float delta = a.m_elems[i] - b.m_elems[i];
|
||||||
|
if(Math.Abs(delta) > EPSILON)
|
||||||
|
return delta < 0.0f ? -1 : 1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Compare two instances of Pair<int,int>.
|
||||||
|
/// Implement for concrete type as Pair<T1,T2> cannot straighforwardly implement IComparable.
|
||||||
|
/// </summary>
|
||||||
|
///
|
||||||
|
/// <remarks> jcurrey, 3/11/2013. </remarks>
|
||||||
|
public class PairIntIntComparer : IComparer<Pair<int, int>> {
|
||||||
|
public int Compare(Pair<int, int> a, Pair<int, int> b) {
|
||||||
|
int keyComparison = a.Key.CompareTo(b.Key);
|
||||||
|
if(keyComparison == 0) {
|
||||||
|
return a.Value.CompareTo(b.Value);
|
||||||
|
} else {
|
||||||
|
return keyComparison;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
*/
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -1,13 +1,21 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
|
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||||
<ItemGroup Label="ProjectConfigurations">
|
<ItemGroup Label="ProjectConfigurations">
|
||||||
<ProjectConfiguration Include="Debug|x64">
|
<ProjectConfiguration Include="Debug|x64">
|
||||||
<Configuration>Debug</Configuration>
|
<Configuration>Debug</Configuration>
|
||||||
|
|
@ -149,13 +149,13 @@
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||||
<ImportGroup Label="ExtensionTargets">
|
<ImportGroup Label="ExtensionTargets">
|
||||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||||
</ImportGroup>
|
</ImportGroup>
|
||||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||||
</Target>
|
</Target>
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||||
<ItemGroup Label="ProjectConfigurations">
|
<ItemGroup Label="ProjectConfigurations">
|
||||||
<ProjectConfiguration Include="Debug|x64">
|
<ProjectConfiguration Include="Debug|x64">
|
||||||
<Configuration>Debug</Configuration>
|
<Configuration>Debug</Configuration>
|
||||||
|
|
@ -123,13 +123,13 @@
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||||
<ImportGroup Label="ExtensionTargets">
|
<ImportGroup Label="ExtensionTargets">
|
||||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||||
</ImportGroup>
|
</ImportGroup>
|
||||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||||
</Target>
|
</Target>
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
|
||||||
// You can specify all the values or you can default the Build and Revision Numbers
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
// by using the '*' as shown below:
|
// by using the '*' as shown below:
|
||||||
// [assembly: AssemblyVersion("1.0.*")]
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
[assembly: AssemblyVersion("0.1.0.0")]
|
[assembly: AssemblyVersion("0.1.2.0")]
|
||||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
[assembly: AssemblyFileVersion("0.1.2.0")]
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
|
|
@ -53,15 +53,39 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.166.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||||
|
|
@ -74,6 +98,10 @@
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||||
|
|
@ -85,6 +113,17 @@
|
||||||
<Reference Include="System" />
|
<Reference Include="System" />
|
||||||
<Reference Include="System.Core" />
|
<Reference Include="System.Core" />
|
||||||
<Reference Include="System.Data.Services.Client" />
|
<Reference Include="System.Data.Services.Client" />
|
||||||
|
<Reference Include="System.Net" />
|
||||||
|
<Reference Include="System.Net.Http" />
|
||||||
|
<Reference Include="System.Net.Http.Extensions, Version=2.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.Primitives, Version=4.2.19.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.WebRequest" />
|
||||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\..\..\..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
<HintPath>..\..\..\..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||||
|
|
@ -112,14 +151,19 @@
|
||||||
<None Include="packages.config" />
|
<None Include="packages.config" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||||
|
<Import Project="..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
|
||||||
|
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
|
||||||
|
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
|
||||||
|
<Error Condition="Exists('..\..\..\..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
|
||||||
|
</Target>
|
||||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||||
</Target>
|
</Target>
|
||||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||||
Other similar extension points exist, see Microsoft.Common.targets.
|
Other similar extension points exist, see Microsoft.Common.targets.
|
||||||
<Target Name="BeforeBuild">
|
<Target Name="BeforeBuild">
|
||||||
|
|
@ -127,4 +171,4 @@
|
||||||
<Target Name="AfterBuild">
|
<Target Name="AfterBuild">
|
||||||
</Target>
|
</Target>
|
||||||
-->
|
-->
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -1,13 +1,21 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
|
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||||
<ItemGroup Label="ProjectConfigurations">
|
<ItemGroup Label="ProjectConfigurations">
|
||||||
<ProjectConfiguration Include="Debug|x64">
|
<ProjectConfiguration Include="Debug|x64">
|
||||||
<Configuration>Debug</Configuration>
|
<Configuration>Debug</Configuration>
|
||||||
|
|
@ -124,13 +124,13 @@
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||||
<ImportGroup Label="ExtensionTargets">
|
<ImportGroup Label="ExtensionTargets">
|
||||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||||
</ImportGroup>
|
</ImportGroup>
|
||||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||||
</Target>
|
</Target>
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||||
<ItemGroup Label="ProjectConfigurations">
|
<ItemGroup Label="ProjectConfigurations">
|
||||||
<ProjectConfiguration Include="Debug|x64">
|
<ProjectConfiguration Include="Debug|x64">
|
||||||
<Configuration>Debug</Configuration>
|
<Configuration>Debug</Configuration>
|
||||||
|
|
@ -223,13 +223,13 @@
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||||
<ImportGroup Label="ExtensionTargets">
|
<ImportGroup Label="ExtensionTargets">
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||||
</ImportGroup>
|
</ImportGroup>
|
||||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||||
</Target>
|
</Target>
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -5,10 +5,6 @@
|
||||||
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
|
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
|
||||||
<Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
|
<Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
|
||||||
</Filter>
|
</Filter>
|
||||||
<Filter Include="Resource Files">
|
|
||||||
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
|
|
||||||
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx</Extensions>
|
|
||||||
</Filter>
|
|
||||||
<Filter Include="Source Files">
|
<Filter Include="Source Files">
|
||||||
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
|
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
|
||||||
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
|
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="Native" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
@ -19,30 +19,31 @@ limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
using Microsoft.Research.Calypso.Tools;
|
using System.Security.Cryptography.X509Certificates;
|
||||||
|
using System.Xml.Linq;
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Text.RegularExpressions;
|
|
||||||
using System.Net;
|
using System.Net;
|
||||||
using System.Diagnostics;
|
|
||||||
|
|
||||||
using Microsoft.Research.Peloponnese.Storage;
|
using Microsoft.Research.Peloponnese.Storage;
|
||||||
|
using Microsoft.Research.Tools;
|
||||||
|
using Microsoft.WindowsAzure.Management.HDInsight;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
namespace Microsoft.Research.JobObjectModel
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Error during conversation with cluster.
|
/// Error during conversation with cluster.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CalypsoClusterException : Exception
|
public sealed class ClusterException : Exception
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Create an exception about handling a cluster.
|
/// Create an exception about handling a cluster.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="message">Exception message.</param>
|
/// <param name="message">Exception message.</param>
|
||||||
public CalypsoClusterException(string message) : base(message) { }
|
public ClusterException(string message) : base(message) { }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -606,13 +607,22 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public abstract string Initialize();
|
public abstract string Initialize();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Enumerate all clusters this user is subscribed to.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A list of clusters.</returns>
|
||||||
|
public static IEnumerable<ClusterConfiguration> EnumerateSubscribedClusters()
|
||||||
|
{
|
||||||
|
return AzureDfsClusterConfiguration.EnumerateAzureDfsSubscribedClusters();
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Create serialization data structure for this configuration.
|
/// Create serialization data structure for this configuration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The corresponding serialization.</returns>
|
/// <returns>The corresponding serialization.</returns>
|
||||||
public ClusterConfigurationSerialization ExtractData()
|
public ClusterConfigurationSerialization ExtractData()
|
||||||
{
|
{
|
||||||
ClusterConfigurationSerialization result = new ClusterConfigurationSerialization()
|
ClusterConfigurationSerialization result = new ClusterConfigurationSerialization
|
||||||
{
|
{
|
||||||
Type = this.TypeOfCluster,
|
Type = this.TypeOfCluster,
|
||||||
Name = this.Name,
|
Name = this.Name,
|
||||||
|
|
@ -925,7 +935,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
IClusterResidentObject dir = this.ProcessWorkDirectory(job.ManagerProcessGuid, true, job.Machine, job); // immutable
|
IClusterResidentObject dir = this.ProcessWorkDirectory(job.ManagerProcessGuid, true, job.Machine, job); // immutable
|
||||||
var matchingfiles = dir.GetFilesAndFolders("DryadLinqProgram__*.xml").ToList();
|
var matchingfiles = dir.GetFilesAndFolders("DryadLinqProgram__*.xml").ToList();
|
||||||
if (matchingfiles.Count() != 1)
|
if (matchingfiles.Count() != 1)
|
||||||
throw new CalypsoClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
|
throw new ClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
|
||||||
IClusterResidentObject result = matchingfiles.First();
|
IClusterResidentObject result = matchingfiles.First();
|
||||||
result.ShouldCacheLocally = true; // immutable
|
result.ShouldCacheLocally = true; // immutable
|
||||||
return result;
|
return result;
|
||||||
|
|
@ -1306,7 +1316,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
//IClusterResidentObject dir = this.ProcessWorkDirectory(new DryadProcessIdentifier("Process.000.001"), true, job.Machine, job);
|
//IClusterResidentObject dir = this.ProcessWorkDirectory(new DryadProcessIdentifier("Process.000.001"), true, job.Machine, job);
|
||||||
var matchingfiles = dir.GetFilesAndFolders("DryadLinqProgram__*.xml").ToList();
|
var matchingfiles = dir.GetFilesAndFolders("DryadLinqProgram__*.xml").ToList();
|
||||||
if (matchingfiles.Count() != 1)
|
if (matchingfiles.Count() != 1)
|
||||||
throw new CalypsoClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
|
throw new ClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
|
||||||
IClusterResidentObject result = matchingfiles.First();
|
IClusterResidentObject result = matchingfiles.First();
|
||||||
result.ShouldCacheLocally = true; // immutable
|
result.ShouldCacheLocally = true; // immutable
|
||||||
return result;
|
return result;
|
||||||
|
|
@ -1395,10 +1405,67 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Enumerate all the clusters this user is subscribed to.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>The list of clusters this user is subscribed to.</returns>
|
||||||
|
public static IEnumerable<ClusterConfiguration> EnumerateAzureDfsSubscribedClusters()
|
||||||
|
{
|
||||||
|
var store = new X509Store();
|
||||||
|
store.Open(OpenFlags.ReadOnly);
|
||||||
|
var configDir = Path.Combine(
|
||||||
|
Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
|
||||||
|
"Windows Azure Powershell");
|
||||||
|
var defaultFile = Path.Combine(configDir, "WindowsAzureProfile.xml");
|
||||||
|
if (File.Exists(defaultFile))
|
||||||
|
{
|
||||||
|
using (FileStream s = new FileStream(defaultFile, FileMode.Open, FileAccess.Read))
|
||||||
|
{
|
||||||
|
XDocument doc = XDocument.Load(s);
|
||||||
|
XNamespace ns = doc.Root.GetDefaultNamespace();
|
||||||
|
IEnumerable<XElement> subs = doc.Descendants(ns + "AzureSubscriptionData");
|
||||||
|
foreach (XElement sub in subs)
|
||||||
|
{
|
||||||
|
string thumbprint = sub.Descendants(ns + "ManagementCertificate").Single().Value;
|
||||||
|
string subId = sub.Descendants(ns + "SubscriptionId").Single().Value;
|
||||||
|
Guid subGuid = new Guid(subId);
|
||||||
|
|
||||||
|
X509Certificate2 cert = store.Certificates.Cast<X509Certificate2>().First(item => item.Thumbprint == thumbprint);
|
||||||
|
|
||||||
|
HDInsightCertificateCredential sCred = new HDInsightCertificateCredential(subGuid, cert);
|
||||||
|
IHDInsightClient sClient = HDInsightClient.Connect(sCred);
|
||||||
|
var clusters = sClient.ListClusters();
|
||||||
|
foreach (var cluster in clusters)
|
||||||
|
{
|
||||||
|
var account = cluster.DefaultStorageAccount;
|
||||||
|
var accountName = account.Name.Split('.').First();
|
||||||
|
Console.WriteLine("Cluster " + cluster.Name + " uses account " + accountName + " with key " + account.Key);
|
||||||
|
|
||||||
|
AzureDfsClusterConfiguration config = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
config = new AzureDfsClusterConfiguration();
|
||||||
|
config.AzureClient = new AzureDfsClient(accountName, account.Key, "dryad-jobs");
|
||||||
|
config.Name = cluster.Name;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Console.WriteLine("Exception while reconstructing cluster " + cluster.Name + ": " + ex);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (config != null)
|
||||||
|
yield return config;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Azure account name.
|
/// Azure account name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string AccountName { get; set; }
|
public
|
||||||
|
string AccountName { get; set; }
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Azure account key.
|
/// Azure account key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|
@ -1567,7 +1634,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
IClusterResidentObject dir = this.ProcessWorkDirectory(job.ManagerProcessGuid, true, job.Machine, job); // immutable
|
IClusterResidentObject dir = this.ProcessWorkDirectory(job.ManagerProcessGuid, true, job.Machine, job); // immutable
|
||||||
var matchingfiles = dir.GetFilesAndFolders("DryadLinqProgram__*.xml").ToList();
|
var matchingfiles = dir.GetFilesAndFolders("DryadLinqProgram__*.xml").ToList();
|
||||||
if (matchingfiles.Count() != 1)
|
if (matchingfiles.Count() != 1)
|
||||||
throw new CalypsoClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
|
throw new ClusterException("Could not find query plan file; got " + matchingfiles.Count() + " possible matches");
|
||||||
IClusterResidentObject result = matchingfiles.First();
|
IClusterResidentObject result = matchingfiles.First();
|
||||||
(result as AzureDfsFile).IsDfsStream = true;
|
(result as AzureDfsFile).IsDfsStream = true;
|
||||||
result.ShouldCacheLocally = true; // immutable
|
result.ShouldCacheLocally = true; // immutable
|
||||||
|
|
|
||||||
|
|
@ -23,11 +23,10 @@ using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text;
|
using Microsoft.Research.Peloponnese.Storage;
|
||||||
using System.Threading.Tasks;
|
using Microsoft.Research.Tools;
|
||||||
using Microsoft.Research.Calypso.Tools;
|
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
namespace Microsoft.Research.JobObjectModel
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Dynamic information of all the jobs and machines in a cluster.
|
/// Dynamic information of all the jobs and machines in a cluster.
|
||||||
|
|
@ -119,22 +118,20 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// The cached of tasks on the cluster.
|
/// The cached of tasks on the cluster.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="virtualCluster">Virtual cluster selected; defined only for Scope clusters.</param>
|
/// <param name="virtualCluster">Virtual cluster selected; defined only for Scope clusters.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reportProgress">Used to report progress.</param>
|
public IEnumerable<ClusterJobInformation> GetClusterJobList(string virtualCluster, CommManager manager)
|
||||||
public IEnumerable<ClusterJobInformation> GetClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
|
|
||||||
{
|
{
|
||||||
this.RecomputeClusterJobList(virtualCluster, reporter, reportProgress);
|
this.RecomputeClusterJobList(virtualCluster, manager);
|
||||||
return this.clusterJobs.Values.ToList();
|
return this.clusterJobs.Values.ToList();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Force the recomputation of the cluster job list.
|
/// Force the recomputation of the cluster job list.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
|
||||||
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
||||||
/// <param name="reportProgress">Used to report progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
// ReSharper disable once UnusedParameter.Global
|
// ReSharper disable once UnusedParameter.Global
|
||||||
protected abstract void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress);
|
protected abstract void RecomputeClusterJobList(string virtualCluster, CommManager manager);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Discover the (unique) dryadlinq job corresponding to a cluster job.
|
/// Discover the (unique) dryadlinq job corresponding to a cluster job.
|
||||||
|
|
@ -160,12 +157,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="job">Job to discover.</param>
|
/// <param name="job">Job to discover.</param>
|
||||||
/// <returns>The cluster job, or null if not found.</returns>
|
/// <returns>The cluster job, or null if not found.</returns>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reportProgress">Used to report progress.</param>
|
public virtual ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, CommManager manager)
|
||||||
public virtual ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, StatusReporter reporter, Action<int> reportProgress)
|
|
||||||
{
|
{
|
||||||
if (this.clusterJobs == null)
|
if (this.clusterJobs == null)
|
||||||
this.RecomputeClusterJobList(job.VirtualCluster, reporter, reportProgress);
|
this.RecomputeClusterJobList(job.VirtualCluster, manager);
|
||||||
return this.clusterJobs[job.ClusterJobId];
|
return this.clusterJobs[job.ClusterJobId];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -173,13 +169,12 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// Refresh the job summary status.
|
/// Refresh the job summary status.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="summary">Summary to refresh.</param>
|
/// <param name="summary">Summary to refresh.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reportProgress">Used to report progress.</param>
|
public virtual void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
|
||||||
public virtual void RefreshStatus(DryadLinqJobSummary summary, StatusReporter reporter, Action<int> reportProgress)
|
|
||||||
{
|
{
|
||||||
// refresh the whole list
|
// refresh the whole list
|
||||||
this.RecomputeClusterJobList(summary.VirtualCluster, reporter, reportProgress);
|
this.RecomputeClusterJobList(summary.VirtualCluster, manager);
|
||||||
ClusterJobInformation info = this.DiscoverClusterJob(summary, reporter, reportProgress);
|
ClusterJobInformation info = this.DiscoverClusterJob(summary, manager);
|
||||||
if (info == null)
|
if (info == null)
|
||||||
{
|
{
|
||||||
summary.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
summary.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
||||||
|
|
@ -226,10 +221,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Recompute the list of jobs on the cluster and add them to the clusterJobs field.
|
/// Recompute the list of jobs on the cluster and add them to the clusterJobs field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
|
||||||
/// <param name="virtualCluster">Unused.</param>
|
/// <param name="virtualCluster">Unused.</param>
|
||||||
/// <param name="reportProgress">Used to report progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
protected override void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
|
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
|
||||||
{
|
{
|
||||||
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
||||||
if (string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
|
if (string.IsNullOrEmpty(CachedClusterResidentObject.CacheDirectory))
|
||||||
|
|
@ -242,32 +236,32 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
string[] files = Directory.GetFiles(joblist, "*.xml");
|
string[] files = Directory.GetFiles(joblist, "*.xml");
|
||||||
foreach (var file in files)
|
foreach (var file in files)
|
||||||
{
|
{
|
||||||
|
manager.Token.ThrowIfCancellationRequested();
|
||||||
DryadLinqJobSummary job = Utilities.LoadXml<DryadLinqJobSummary>(file);
|
DryadLinqJobSummary job = Utilities.LoadXml<DryadLinqJobSummary>(file);
|
||||||
string cjid = job.Cluster + "-" + job.ClusterJobId; // there may be two jobs with same id from different clusters
|
string cjid = job.Cluster + "-" + job.ClusterJobId; // there may be two jobs with same id from different clusters
|
||||||
ClusterJobInformation ci = new ClusterJobInformation(this.Config.Name, job.Cluster, cjid, job.Name, job.User, job.Date, job.EndTime - job.Date, job.Status);
|
ClusterJobInformation ci = new ClusterJobInformation(this.Config.Name, job.Cluster, cjid, job.Name, job.User, job.Date, job.EndTime - job.Date, job.Status);
|
||||||
ci.SetAssociatedSummary(job);
|
ci.SetAssociatedSummary(job);
|
||||||
if (this.clusterJobs.ContainsKey(cjid))
|
if (this.clusterJobs.ContainsKey(cjid))
|
||||||
{
|
{
|
||||||
reporter("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error);
|
manager.Status("Duplicate job id, cannot insert in cache " + job.AsIdentifyingString(), StatusKind.Error);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
this.clusterJobs.Add(cjid, ci);
|
this.clusterJobs.Add(cjid, ci);
|
||||||
}
|
}
|
||||||
reportProgress(100);
|
manager.Progress(100);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Refresh the job summary status.
|
/// Refresh the job summary status.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="job">Summary to refresh.</param>
|
/// <param name="job">Summary to refresh.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reportProgress">Used to report progres.</param>
|
public override void RefreshStatus(DryadLinqJobSummary job, CommManager manager)
|
||||||
public override void RefreshStatus(DryadLinqJobSummary job, StatusReporter reporter, Action<int> reportProgress)
|
|
||||||
{
|
{
|
||||||
ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job);
|
ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job);
|
||||||
ClusterStatus actualStatus = actual.CreateClusterStatus();
|
ClusterStatus actualStatus = actual.CreateClusterStatus();
|
||||||
actualStatus.RefreshStatus(job, reporter, reportProgress);
|
actualStatus.RefreshStatus(job, manager);
|
||||||
ClusterJobInformation info = actualStatus.DiscoverClusterJob(job, reporter, reportProgress);
|
ClusterJobInformation info = actualStatus.DiscoverClusterJob(job, manager);
|
||||||
if (info == null)
|
if (info == null)
|
||||||
{
|
{
|
||||||
job.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
job.Status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
||||||
|
|
@ -303,13 +297,12 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="job">Cluster job.</param>
|
/// <param name="job">Cluster job.</param>
|
||||||
/// <returns>Throws an exception.</returns>
|
/// <returns>Throws an exception.</returns>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reportProgress">Used to report progress.</param>
|
public override ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, CommManager manager)
|
||||||
public override ClusterJobInformation DiscoverClusterJob(DryadLinqJobSummary job, StatusReporter reporter, Action<int> reportProgress)
|
|
||||||
{
|
{
|
||||||
ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job);
|
ClusterConfiguration actual = (this.Config as CacheClusterConfiguration).ActualConfig(job);
|
||||||
ClusterStatus actualStatus = actual.CreateClusterStatus();
|
ClusterStatus actualStatus = actual.CreateClusterStatus();
|
||||||
return actualStatus.DiscoverClusterJob(job, reporter, reportProgress);
|
return actualStatus.DiscoverClusterJob(job, manager);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -358,10 +351,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Force the recomputation of the cluster job list.
|
/// Force the recomputation of the cluster job list.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
|
||||||
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
||||||
/// <param name="reportProgress">Used to report progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
protected override void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
|
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
|
||||||
{
|
{
|
||||||
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
||||||
if (!Directory.Exists(this.config.JobsFolder))
|
if (!Directory.Exists(this.config.JobsFolder))
|
||||||
|
|
@ -371,6 +363,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
int done = 0;
|
int done = 0;
|
||||||
foreach (var job in subfolders)
|
foreach (var job in subfolders)
|
||||||
{
|
{
|
||||||
|
manager.Token.ThrowIfCancellationRequested();
|
||||||
string jobId = Path.GetFileName(job);
|
string jobId = Path.GetFileName(job);
|
||||||
ClusterJobInformation info = this.GetJobInfo(job, jobId);
|
ClusterJobInformation info = this.GetJobInfo(job, jobId);
|
||||||
if (info != null)
|
if (info != null)
|
||||||
|
|
@ -378,9 +371,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
// ReSharper disable once AssignNullToNotNullAttribute
|
// ReSharper disable once AssignNullToNotNullAttribute
|
||||||
this.clusterJobs.Add(jobId, info);
|
this.clusterJobs.Add(jobId, info);
|
||||||
}
|
}
|
||||||
reportProgress(done++ *100/subfolders.Length);
|
manager.Progress(done++ *100/subfolders.Length);
|
||||||
}
|
}
|
||||||
reportProgress(100);
|
manager.Progress(100);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -478,10 +471,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Force the recomputation of the cluster job list.
|
/// Force the recomputation of the cluster job list.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
|
||||||
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
/// <param name="virtualCluster">Virtual cluster to use (defined only for some cluster types).</param>
|
||||||
/// <param name="reportProgress">Used to report progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
protected override void RecomputeClusterJobList(string virtualCluster, StatusReporter reporter, Action<int> reportProgress)
|
protected override void RecomputeClusterJobList(string virtualCluster, CommManager manager)
|
||||||
{
|
{
|
||||||
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
this.clusterJobs = new Dictionary<string, ClusterJobInformation>();
|
||||||
var jobs = this.config.AzureClient.EnumerateDirectory("").ToList();
|
var jobs = this.config.AzureClient.EnumerateDirectory("").ToList();
|
||||||
|
|
@ -489,15 +481,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
int done = 0;
|
int done = 0;
|
||||||
foreach (var job in jobs)
|
foreach (var job in jobs)
|
||||||
{
|
{
|
||||||
|
manager.Token.ThrowIfCancellationRequested();
|
||||||
ClusterJobInformation info = this.GetJobInfo(job);
|
ClusterJobInformation info = this.GetJobInfo(job);
|
||||||
if (info != null)
|
if (info != null)
|
||||||
{
|
{
|
||||||
// ReSharper disable once AssignNullToNotNullAttribute
|
// ReSharper disable once AssignNullToNotNullAttribute
|
||||||
this.clusterJobs.Add(job, info);
|
this.clusterJobs.Add(job, info);
|
||||||
}
|
}
|
||||||
reportProgress(100*done++/jobs.Count);
|
manager.Progress(100*done++/jobs.Count);
|
||||||
}
|
}
|
||||||
reportProgress(100);
|
manager.Progress(100);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -547,23 +540,12 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
DateTime lastHeartBeat = DateTime.MinValue;
|
DateTime lastHeartBeat = DateTime.MinValue;
|
||||||
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
ClusterJobInformation.ClusterJobStatus status = ClusterJobInformation.ClusterJobStatus.Unknown;
|
||||||
bool found = false;
|
bool found = false;
|
||||||
|
string jobName = jobRootFolder;
|
||||||
|
|
||||||
var jobsFolders = this.config.AzureClient.EnumerateDirectory(jobRootFolder).ToList();
|
var jobsFolders = this.config.AzureClient.EnumerateDirectory(jobRootFolder).ToList();
|
||||||
foreach (var file in jobsFolders)
|
foreach (var file in jobsFolders)
|
||||||
{
|
{
|
||||||
if (file.Contains("DryadLinqProgram__"))
|
if (file.EndsWith("heartbeat"))
|
||||||
{
|
|
||||||
var blob = this.config.AzureClient.Container.GetBlockBlobReference(file);
|
|
||||||
blob.FetchAttributes();
|
|
||||||
var props = blob.Properties;
|
|
||||||
if (props.LastModified.HasValue)
|
|
||||||
{
|
|
||||||
date = props.LastModified.Value.DateTime;
|
|
||||||
date = date.ToLocalTime();
|
|
||||||
}
|
|
||||||
found = true;
|
|
||||||
}
|
|
||||||
else if (file.EndsWith("heartbeat"))
|
|
||||||
{
|
{
|
||||||
var blob = this.config.AzureClient.Container.GetPageBlobReference(file);
|
var blob = this.config.AzureClient.Container.GetPageBlobReference(file);
|
||||||
blob.FetchAttributes();
|
blob.FetchAttributes();
|
||||||
|
|
@ -582,6 +564,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
case "running":
|
case "running":
|
||||||
status = ClusterJobInformation.ClusterJobStatus.Running;
|
status = ClusterJobInformation.ClusterJobStatus.Running;
|
||||||
break;
|
break;
|
||||||
|
case "killed":
|
||||||
|
status = ClusterJobInformation.ClusterJobStatus.Cancelled;
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
Console.WriteLine("Unknown status " + st);
|
Console.WriteLine("Unknown status " + st);
|
||||||
break;
|
break;
|
||||||
|
|
@ -599,6 +584,31 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
status = ClusterJobInformation.ClusterJobStatus.Failed;
|
status = ClusterJobInformation.ClusterJobStatus.Failed;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (props.ContainsKey("jobname"))
|
||||||
|
{
|
||||||
|
jobName = props["jobname"];
|
||||||
|
}
|
||||||
|
if (props.ContainsKey("starttime"))
|
||||||
|
{
|
||||||
|
var t = props["starttime"];
|
||||||
|
if (DateTime.TryParse(t, out date))
|
||||||
|
date = date.ToLocalTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
found = true;
|
||||||
|
}
|
||||||
|
else if (file.Contains("DryadLinqProgram__") &&
|
||||||
|
// newer heartbeats contain the date
|
||||||
|
date != DateTime.MinValue)
|
||||||
|
{
|
||||||
|
var blob = this.config.AzureClient.Container.GetBlockBlobReference(file);
|
||||||
|
blob.FetchAttributes();
|
||||||
|
var props = blob.Properties;
|
||||||
|
if (props.LastModified.HasValue)
|
||||||
|
{
|
||||||
|
date = props.LastModified.Value.DateTime;
|
||||||
|
date = date.ToLocalTime();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -607,7 +617,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
TimeSpan running = TimeSpan.Zero;
|
TimeSpan running = TimeSpan.Zero;
|
||||||
if (date != DateTime.MinValue && lastHeartBeat != DateTime.MinValue)
|
if (date != DateTime.MinValue && lastHeartBeat != DateTime.MinValue)
|
||||||
running = lastHeartBeat - date;
|
running = lastHeartBeat - date;
|
||||||
var info = new ClusterJobInformation(this.config.Name, "", jobRootFolder, jobRootFolder, Environment.UserName, date, running, status);
|
var info = new ClusterJobInformation(this.config.Name, "", jobRootFolder, jobName, Environment.UserName, date, running, status);
|
||||||
return info;
|
return info;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -615,9 +625,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// Refresh the job summary status.
|
/// Refresh the job summary status.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="summary">Summary to refresh.</param>
|
/// <param name="summary">Summary to refresh.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reportProgress">Used to report progress.</param>
|
public override void RefreshStatus(DryadLinqJobSummary summary, CommManager manager)
|
||||||
public override void RefreshStatus(DryadLinqJobSummary summary, StatusReporter reporter, Action<int> reportProgress)
|
|
||||||
{
|
{
|
||||||
// refresh the whole list
|
// refresh the whole list
|
||||||
ClusterJobInformation info = this.GetJobInfo(summary.JobID);
|
ClusterJobInformation info = this.GetJobInfo(summary.JobID);
|
||||||
|
|
@ -636,7 +645,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <returns>True if the cancellation succeeded.</returns>
|
/// <returns>True if the cancellation succeeded.</returns>
|
||||||
public override bool CancelJob(DryadLinqJobSummary job)
|
public override bool CancelJob(DryadLinqJobSummary job)
|
||||||
{
|
{
|
||||||
throw new InvalidOperationException();
|
AzureUtils.KillJob(this.config.AccountName, this.config.AccountKey, this.config.Container, job.ClusterJobId);
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||||
|
|
@ -10,7 +10,7 @@
|
||||||
<OutputType>Library</OutputType>
|
<OutputType>Library</OutputType>
|
||||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||||
<RootNamespace>DistributedDataCollection</RootNamespace>
|
<RootNamespace>DistributedDataCollection</RootNamespace>
|
||||||
<AssemblyName>Microsoft.Research.DryadLinq.JobBrowser.DistributedDataCollection</AssemblyName>
|
<AssemblyName>DistributedDataCollection</AssemblyName>
|
||||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||||
<FileAlignment>512</FileAlignment>
|
<FileAlignment>512</FileAlignment>
|
||||||
<StartupObject>
|
<StartupObject>
|
||||||
|
|
@ -37,28 +37,6 @@
|
||||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||||
<TargetFrameworkProfile />
|
<TargetFrameworkProfile />
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
|
||||||
<DebugSymbols>true</DebugSymbols>
|
|
||||||
<DebugType>full</DebugType>
|
|
||||||
<Optimize>false</Optimize>
|
|
||||||
<OutputPath>bin\Debug\</OutputPath>
|
|
||||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
<DocumentationFile>bin\Debug\Microsoft.Research.DryadLinq.JobBrowser.DistributedDataCollection.xml</DocumentationFile>
|
|
||||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
|
||||||
<Prefer32Bit>false</Prefer32Bit>
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
|
||||||
<DebugType>pdbonly</DebugType>
|
|
||||||
<Optimize>true</Optimize>
|
|
||||||
<OutputPath>bin\Release\</OutputPath>
|
|
||||||
<DefineConstants>TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
|
||||||
<Prefer32Bit>false</Prefer32Bit>
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||||
<DebugSymbols>true</DebugSymbols>
|
<DebugSymbols>true</DebugSymbols>
|
||||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||||
|
|
@ -91,43 +69,59 @@
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Hadoop.Client">
|
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Hpc.Scheduler, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
|
||||||
<SpecificVersion>False</SpecificVersion>
|
|
||||||
<HintPath>..\Dependencies\Microsoft.Hpc.Scheduler.dll</HintPath>
|
|
||||||
</Reference>
|
|
||||||
<Reference Include="Microsoft.Hpc.Scheduler.Properties, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
|
||||||
<SpecificVersion>False</SpecificVersion>
|
|
||||||
<HintPath>..\Dependencies\Microsoft.Hpc.Scheduler.Properties.dll</HintPath>
|
|
||||||
</Reference>
|
|
||||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks">
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions">
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
|
<Reference Include="Microsoft.WindowsAzure.Management">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
<Reference Include="Newtonsoft.Json">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
|
||||||
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="System" />
|
<Reference Include="System" />
|
||||||
|
|
@ -135,6 +129,15 @@
|
||||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="System.Data.Services.Client" />
|
<Reference Include="System.Data.Services.Client" />
|
||||||
|
<Reference Include="System.Net" />
|
||||||
|
<Reference Include="System.Net.Http" />
|
||||||
|
<Reference Include="System.Net.Http.Extensions">
|
||||||
|
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.Primitives">
|
||||||
|
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.WebRequest" />
|
||||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||||
|
|
@ -153,7 +156,7 @@
|
||||||
<Compile Include="ClusterAccess.cs" />
|
<Compile Include="ClusterAccess.cs" />
|
||||||
<Compile Include="ClusterConfiguration.cs" />
|
<Compile Include="ClusterConfiguration.cs" />
|
||||||
<Compile Include="ClusterStatus.cs" />
|
<Compile Include="ClusterStatus.cs" />
|
||||||
<Compile Include="Cosmos.cs" />
|
<Compile Include="dryadlog.cs" />
|
||||||
<Compile Include="jobinfo.cs" />
|
<Compile Include="jobinfo.cs" />
|
||||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||||
<Compile Include="Storage.cs" />
|
<Compile Include="Storage.cs" />
|
||||||
|
|
@ -186,15 +189,19 @@
|
||||||
<None Include="packages.config" />
|
<None Include="packages.config" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.0-beta023\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.0-beta023\build\Microsoft.Research.Peloponnese.targets')" />
|
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
|
||||||
|
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
|
||||||
|
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
|
||||||
|
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
|
||||||
|
</Target>
|
||||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||||
</Target>
|
</Target>
|
||||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||||
Other similar extension points exist, see Microsoft.Common.targets.
|
Other similar extension points exist, see Microsoft.Common.targets.
|
||||||
<Target Name="BeforeBuild">
|
<Target Name="BeforeBuild">
|
||||||
|
|
@ -202,4 +209,4 @@
|
||||||
<Target Name="AfterBuild">
|
<Target Name="AfterBuild">
|
||||||
</Target>
|
</Target>
|
||||||
-->
|
-->
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -28,9 +28,9 @@ using System.Runtime.InteropServices;
|
||||||
[assembly: AssemblyTitle("DistributedDataCollection")]
|
[assembly: AssemblyTitle("DistributedDataCollection")]
|
||||||
[assembly: AssemblyDescription("")]
|
[assembly: AssemblyDescription("")]
|
||||||
[assembly: AssemblyConfiguration("")]
|
[assembly: AssemblyConfiguration("")]
|
||||||
[assembly: AssemblyCompany("Microsoft Corporation")]
|
[assembly: AssemblyCompany("")]
|
||||||
[assembly: AssemblyProduct("DistributedDataCollection")]
|
[assembly: AssemblyProduct("DistributedDataCollection")]
|
||||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
|
[assembly: AssemblyCopyright("Copyright © 2009")]
|
||||||
[assembly: AssemblyTrademark("")]
|
[assembly: AssemblyTrademark("")]
|
||||||
[assembly: AssemblyCulture("")]
|
[assembly: AssemblyCulture("")]
|
||||||
|
|
||||||
|
|
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
|
||||||
// You can specify all the values or you can default the Build and Revision Numbers
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
// by using the '*' as shown below:
|
// by using the '*' as shown below:
|
||||||
// [assembly: AssemblyVersion("1.0.*")]
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
[assembly: AssemblyVersion("0.1.0.0")]
|
[assembly: AssemblyVersion("1.0.0.0")]
|
||||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||||
|
|
|
||||||
|
|
@ -2,10 +2,6 @@
|
||||||
<configuration>
|
<configuration>
|
||||||
<runtime>
|
<runtime>
|
||||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||||
<dependentAssembly>
|
|
||||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
|
||||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
|
||||||
</dependentAssembly>
|
|
||||||
<dependentAssembly>
|
<dependentAssembly>
|
||||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||||
|
|
@ -22,6 +18,10 @@
|
||||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||||
</dependentAssembly>
|
</dependentAssembly>
|
||||||
|
<dependentAssembly>
|
||||||
|
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||||
|
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||||
|
</dependentAssembly>
|
||||||
</assemblyBinding>
|
</assemblyBinding>
|
||||||
</runtime>
|
</runtime>
|
||||||
</configuration>
|
</configuration>
|
||||||
|
|
@ -20,15 +20,15 @@ limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
using Microsoft.Research.Calypso.Tools;
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using Microsoft.Research.Peloponnese.Storage;
|
using Microsoft.Research.Peloponnese.Storage;
|
||||||
|
using Microsoft.Research.Tools;
|
||||||
using Microsoft.WindowsAzure.Storage.Blob;
|
using Microsoft.WindowsAzure.Storage.Blob;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
namespace Microsoft.Research.JobObjectModel
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A cluster-resident object is a file or a folder.
|
/// A cluster-resident object is a file or a folder.
|
||||||
|
|
@ -158,7 +158,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
if (file.Job == null)
|
if (file.Job == null)
|
||||||
return;
|
return;
|
||||||
if (string.IsNullOrEmpty(file.LocalCachePath))
|
if (string.IsNullOrEmpty(file.LocalCachePath))
|
||||||
throw new CalypsoClusterException("Missing expected LocalCachePath");
|
throw new ClusterException("Missing expected LocalCachePath");
|
||||||
|
|
||||||
CachedClusterResidentObject.RecordCachedFile(file.Job, file.LocalCachePath);
|
CachedClusterResidentObject.RecordCachedFile(file.Job, file.LocalCachePath);
|
||||||
}
|
}
|
||||||
|
|
@ -375,7 +375,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
{
|
{
|
||||||
// cache it
|
// cache it
|
||||||
if (this.RepresentsAFolder)
|
if (this.RepresentsAFolder)
|
||||||
throw new CalypsoClusterException("Cannot cache folders");
|
throw new ClusterException("Cannot cache folders");
|
||||||
|
|
||||||
StreamWriter writer = this.CreateTempStream();
|
StreamWriter writer = this.CreateTempStream();
|
||||||
return new FileSharedStreamReader(this.Pathname.ToString(), writer, this.OnClose);
|
return new FileSharedStreamReader(this.Pathname.ToString(), writer, this.OnClose);
|
||||||
|
|
@ -504,7 +504,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
get
|
get
|
||||||
{
|
{
|
||||||
if (this.RepresentsAFolder)
|
if (this.RepresentsAFolder)
|
||||||
throw new CalypsoClusterException("Cannot get size of a folder");
|
throw new ClusterException("Cannot get size of a folder");
|
||||||
if (File.Exists(this.LocalCachePath))
|
if (File.Exists(this.LocalCachePath))
|
||||||
{
|
{
|
||||||
FileInfo info = new FileInfo(this.LocalCachePath);
|
FileInfo info = new FileInfo(this.LocalCachePath);
|
||||||
|
|
@ -732,7 +732,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <returns>Throws an exception.</returns>
|
/// <returns>Throws an exception.</returns>
|
||||||
public IEnumerable<IClusterResidentObject> GetFilesAndFolders(string match)
|
public IEnumerable<IClusterResidentObject> GetFilesAndFolders(string match)
|
||||||
{
|
{
|
||||||
throw new CalypsoClusterException("Object is not a folder");
|
throw new ClusterException("Object is not a folder");
|
||||||
}
|
}
|
||||||
|
|
||||||
private long size;
|
private long size;
|
||||||
|
|
@ -787,7 +787,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <returns>Throws an exception.</returns>
|
/// <returns>Throws an exception.</returns>
|
||||||
public IClusterResidentObject GetFile(string filename)
|
public IClusterResidentObject GetFile(string filename)
|
||||||
{
|
{
|
||||||
throw new CalypsoClusterException("Object is not a folder");
|
throw new ClusterException("Object is not a folder");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -797,7 +797,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <returns>Throws an exception.</returns>
|
/// <returns>Throws an exception.</returns>
|
||||||
public IClusterResidentObject GetFolder(string foldername)
|
public IClusterResidentObject GetFolder(string foldername)
|
||||||
{
|
{
|
||||||
throw new CalypsoClusterException("Object is not a folder");
|
throw new ClusterException("Object is not a folder");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1011,13 +1011,24 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
this.client.ContainerName,
|
this.client.ContainerName,
|
||||||
this.path);
|
this.path);
|
||||||
}
|
}
|
||||||
StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, false, 1024 * 1024);
|
|
||||||
|
long size = this.Size;
|
||||||
|
int bufferSize = 1024*1024;
|
||||||
|
if (size >= 0)
|
||||||
|
{
|
||||||
|
bufferSize = (int)(size/10);
|
||||||
|
if (bufferSize < 1024*1024)
|
||||||
|
bufferSize = 1024*1024;
|
||||||
|
if (bufferSize > 20*1024*1024)
|
||||||
|
bufferSize = 20*1024*1024;
|
||||||
|
}
|
||||||
|
StreamReader reader = new StreamReader(stream, System.Text.Encoding.UTF8, false, bufferSize);
|
||||||
|
|
||||||
if (this.ShouldCacheLocally && this.LocalCachePath != null)
|
if (this.ShouldCacheLocally && this.LocalCachePath != null)
|
||||||
{
|
{
|
||||||
// cache it
|
// cache it
|
||||||
if (this.RepresentsAFolder)
|
if (this.RepresentsAFolder)
|
||||||
throw new CalypsoClusterException("Cannot cache folders");
|
throw new ClusterException("Cannot cache folders");
|
||||||
StreamWriter writer = this.CreateTempStream();
|
StreamWriter writer = this.CreateTempStream();
|
||||||
return new SharedStreamReader(reader, writer, this.OnClose);
|
return new SharedStreamReader(reader, writer, this.OnClose);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -19,17 +19,17 @@ limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
namespace Microsoft.Research.JobObjectModel
|
||||||
{
|
{
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
using System;
|
using System;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Information about a standard Cosmos log entry.
|
/// Information about a standard Dryad log entry.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Serializable]
|
[Serializable]
|
||||||
public class CosmosLogEntry : IParse
|
public class DryadLogEntry : IParse
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Message severity.
|
/// Message severity.
|
||||||
|
|
@ -98,7 +98,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Allocate an empty log entry
|
/// Allocate an empty log entry
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CosmosLogEntry()
|
public DryadLogEntry()
|
||||||
{
|
{
|
||||||
this.Malformed = true;
|
this.Malformed = true;
|
||||||
}
|
}
|
||||||
|
|
@ -107,7 +107,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// Allocate a log entry from a given string.
|
/// Allocate a log entry from a given string.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="line">String to initialize the log entry.</param>
|
/// <param name="line">String to initialize the log entry.</param>
|
||||||
public CosmosLogEntry(string line)
|
public DryadLogEntry(string line)
|
||||||
{
|
{
|
||||||
this.Malformed = true;
|
this.Malformed = true;
|
||||||
// ReSharper disable once DoNotCallOverridableMethodsInConstructor
|
// ReSharper disable once DoNotCallOverridableMethodsInConstructor
|
||||||
|
|
@ -167,7 +167,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// An extended log entry is like a cosmos log entry, but it has a prefix: GUID,Machine
|
/// An extended log entry is like a cosmos log entry, but it has a prefix: GUID,Machine
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Serializable]
|
[Serializable]
|
||||||
public class ExtendedLogEntry : CosmosLogEntry
|
public class ExtendedLogEntry : DryadLogEntry
|
||||||
{
|
{
|
||||||
static Regex loglineregex =
|
static Regex loglineregex =
|
||||||
new Regex(@"([-0-9A-F]+), # 1 guid, inserted by reader
|
new Regex(@"([-0-9A-F]+), # 1 guid, inserted by reader
|
||||||
|
|
@ -27,24 +27,23 @@ using System.Collections.Generic;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text;
|
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
using System.Xml.Linq;
|
using System.Xml.Linq;
|
||||||
using Microsoft.Research.Calypso.Tools;
|
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
using Microsoft.Research.Tools;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
namespace Microsoft.Research.JobObjectModel
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Exception throw by Calypso when it cannot understand the structure of a Dryad/DryadLINQ job.
|
/// Exception thrown when we cannot understand the structure of a Dryad/DryadLINQ job.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class CalypsoDryadException : Exception
|
public class DryadException : Exception
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Create a new CalypsoDryadException.
|
/// Create a new DryadException.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="message">Message conveyed by the exception.</param>
|
/// <param name="message">Message conveyed by the exception.</param>
|
||||||
public CalypsoDryadException(string message) : base(message) { }
|
public DryadException(string message) : base(message) { }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -195,7 +194,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
this.SuccessfulVertices++;
|
this.SuccessfulVertices++;
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
throw new CalypsoDryadException("Unexpected vertex state " + vertex.State);
|
throw new DryadException("Unexpected vertex state " + vertex.State);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.TotalInitiatedVertices -= this.AbandonedVertices;
|
this.TotalInitiatedVertices -= this.AbandonedVertices;
|
||||||
|
|
@ -423,22 +422,21 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <param name="cf">Cluster configuration.</param>
|
/// <param name="cf">Cluster configuration.</param>
|
||||||
/// <param name="summary">Summary description of the job.</param>
|
/// <param name="summary">Summary description of the job.</param>
|
||||||
/// <returns>The Dryad job description, or null.</returns>
|
/// <returns>The Dryad job description, or null.</returns>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
|
||||||
/// <param name="fill">If true, fill all the information, otherwise the user will have to call FillInformation on the result later.</param>
|
/// <param name="fill">If true, fill all the information, otherwise the user will have to call FillInformation on the result later.</param>
|
||||||
/// <param name="updateProgress">Delegate used to report progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
public static DryadLinqJobInfo CreateDryadLinqJobInfo(ClusterConfiguration cf, DryadLinqJobSummary summary, bool fill, StatusReporter reporter, Action<int> updateProgress)
|
public static DryadLinqJobInfo CreateDryadLinqJobInfo(ClusterConfiguration cf, DryadLinqJobSummary summary, bool fill, CommManager manager)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
DryadLinqJobInfo job = new DryadLinqJobInfo(cf, summary);
|
DryadLinqJobInfo job = new DryadLinqJobInfo(cf, summary);
|
||||||
if (fill)
|
if (fill)
|
||||||
job.CollectEssentialInformation(reporter, updateProgress);
|
job.CollectEssentialInformation(manager);
|
||||||
return job;
|
return job;
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
Trace.TraceInformation(e.ToString());
|
Trace.TraceInformation(e.ToString());
|
||||||
reporter("Could not collect job information for " + summary.Name + ": " + e.Message, StatusKind.Error);
|
manager.Status("Could not collect job information for " + summary.Name + ": " + e.Message, StatusKind.Error);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -506,7 +504,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
|
|
||||||
if (this.stdoutpath == null)
|
if (this.stdoutpath == null)
|
||||||
{
|
{
|
||||||
throw new CalypsoClusterException("Could not locate JM standard output file in folder " + jmdir);
|
throw new ClusterException("Could not locate JM standard output file in folder " + jmdir);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -515,9 +513,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Refresh the job status.
|
/// Refresh the job status.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="updateProgress">Used to report progress.</param>
|
public void RefreshJobStatus(CommManager manager)
|
||||||
public void RefreshJobStatus(StatusReporter reporter, Action<int> updateProgress)
|
|
||||||
{
|
{
|
||||||
// skip if job is finished
|
// skip if job is finished
|
||||||
if (this.Summary.Status == ClusterJobInformation.ClusterJobStatus.Failed ||
|
if (this.Summary.Status == ClusterJobInformation.ClusterJobStatus.Failed ||
|
||||||
|
|
@ -526,18 +523,17 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
return;
|
return;
|
||||||
|
|
||||||
ClusterStatus status = this.ClusterConfiguration.CreateClusterStatus();
|
ClusterStatus status = this.ClusterConfiguration.CreateClusterStatus();
|
||||||
status.RefreshStatus(this.Summary, reporter, updateProgress);
|
status.RefreshStatus(this.Summary, manager);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Fill the job info by parsing the stdout.txt.
|
/// Fill the job info by parsing the stdout.txt.
|
||||||
/// <param name="statusReporter">Delegate used to report errors.</param>
|
/// <returns>The updated job.</returns>
|
||||||
/// <returns>True if it succeeds, false otherwise.</returns>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="updateProgress">Delegate used to report progress.</param>
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool CollectEssentialInformation(StatusReporter statusReporter, Action<int> updateProgress)
|
public bool CollectEssentialInformation(CommManager manager)
|
||||||
{
|
{
|
||||||
this.RefreshJobStatus(statusReporter, updateProgress);
|
this.RefreshJobStatus(manager);
|
||||||
if (this.ManagerVertex == null)
|
if (this.ManagerVertex == null)
|
||||||
{
|
{
|
||||||
this.ManagerVertex = new ExecutedVertexInstance(this, -1, 0, "JobManager", "", this.Summary.Date);
|
this.ManagerVertex = new ExecutedVertexInstance(this, -1, 0, "JobManager", "", this.Summary.Date);
|
||||||
|
|
@ -562,13 +558,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
|
|
||||||
if (this.stdoutpath == null)
|
if (this.stdoutpath == null)
|
||||||
return false;
|
return false;
|
||||||
bool success = this.ParseStdout(this.stdoutpath, statusReporter, updateProgress);
|
bool success = this.ParseStdout(this.stdoutpath, manager);
|
||||||
updateProgress(100);
|
manager.Progress(100);
|
||||||
if (!success)
|
if (!success)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
this.JobInfoCannotBeCollected = false;
|
this.JobInfoCannotBeCollected = false;
|
||||||
statusReporter("Stdout parsed", StatusKind.OK);
|
manager.Status("Stdout parsed", StatusKind.OK);
|
||||||
|
|
||||||
this.LastUpdatetime = DateTime.Now;
|
this.LastUpdatetime = DateTime.Now;
|
||||||
if (this.Summary.Status == ClusterJobInformation.ClusterJobStatus.Running)
|
if (this.Summary.Status == ClusterJobInformation.ClusterJobStatus.Running)
|
||||||
|
|
@ -600,13 +596,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
{
|
{
|
||||||
Match m = numberre.Match(vertexlist);
|
Match m = numberre.Match(vertexlist);
|
||||||
if (!m.Success)
|
if (!m.Success)
|
||||||
throw new CalypsoDryadException("Could not find vertex number in " + vertexlist);
|
throw new DryadException("Could not find vertex number in " + vertexlist);
|
||||||
string number = m.Groups[1].Value;
|
string number = m.Groups[1].Value;
|
||||||
|
|
||||||
// now scan a balanced number of parantheses
|
// now scan a balanced number of parantheses
|
||||||
string rest = m.Groups[2].Value;
|
string rest = m.Groups[2].Value;
|
||||||
if (rest[0] != '(')
|
if (rest[0] != '(')
|
||||||
throw new CalypsoDryadException("Expecting open parens after vertex number");
|
throw new DryadException("Expecting open parens after vertex number");
|
||||||
int opened = 0;
|
int opened = 0;
|
||||||
int i;
|
int i;
|
||||||
for (i = 0; i < rest.Length; i++)
|
for (i = 0; i < rest.Length; i++)
|
||||||
|
|
@ -624,7 +620,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (opened != 0 || i <= 2)
|
if (opened != 0 || i <= 2)
|
||||||
throw new CalypsoDryadException("did not find matched parantheses in vertex name in " + vertexlist + ", can't parse");
|
throw new DryadException("did not find matched parantheses in vertex name in " + vertexlist + ", can't parse");
|
||||||
string name = rest.Substring(1, i - 2); // skip first and last paranthesis
|
string name = rest.Substring(1, i - 2); // skip first and last paranthesis
|
||||||
yield return new Tuple<string, int>(name, int.Parse(number));
|
yield return new Tuple<string, int>(name, int.Parse(number));
|
||||||
vertexlist = rest.Substring(i);
|
vertexlist = rest.Substring(i);
|
||||||
|
|
@ -761,7 +757,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
case "running":
|
case "running":
|
||||||
{
|
{
|
||||||
string process;
|
string process;
|
||||||
kvp.TryGetValue("id", out process); // "process" is also good
|
kvp.TryGetValue("id", out process);
|
||||||
|
if (process == null)
|
||||||
|
kvp.TryGetValue("process", out process);
|
||||||
string machine = kvp["computer"];
|
string machine = kvp["computer"];
|
||||||
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
||||||
this.jobVertices.Remap(vi, process);
|
this.jobVertices.Remap(vi, process);
|
||||||
|
|
@ -776,6 +774,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
vi.SetState(ExecutedVertexInstance.VertexState.Successful);
|
vi.SetState(ExecutedVertexInstance.VertexState.Successful);
|
||||||
vi.End = timeStamp;
|
vi.End = timeStamp;
|
||||||
vi.ExitCode = "";
|
vi.ExitCode = "";
|
||||||
|
this.UsefulCPUTime += vi.RunningTime;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
case "failed":
|
case "failed":
|
||||||
|
|
@ -784,7 +783,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
if (vi.State != ExecutedVertexInstance.VertexState.Started)
|
if (vi.State != ExecutedVertexInstance.VertexState.Started)
|
||||||
vi.SetState(ExecutedVertexInstance.VertexState.Cancelled);
|
vi.SetState(ExecutedVertexInstance.VertexState.Cancelled);
|
||||||
else
|
else
|
||||||
|
{
|
||||||
vi.SetState(ExecutedVertexInstance.VertexState.Failed);
|
vi.SetState(ExecutedVertexInstance.VertexState.Failed);
|
||||||
|
if (vi.RunningTime > TimeSpan.Zero)
|
||||||
|
this.WastedCPUTime += vi.RunningTime;
|
||||||
|
}
|
||||||
if (kvp.ContainsKey("errorstring"))
|
if (kvp.ContainsKey("errorstring"))
|
||||||
vi.AddErrorString(kvp["errorstring"]);
|
vi.AddErrorString(kvp["errorstring"]);
|
||||||
string exitcode;
|
string exitcode;
|
||||||
|
|
@ -797,32 +800,15 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
}
|
}
|
||||||
else if (kvp.ContainsKey("outputChannel"))
|
else if (kvp.ContainsKey("outputChannel"))
|
||||||
{
|
{
|
||||||
string chan = kvp["outputChannel"];
|
|
||||||
int channelNo = int.Parse(chan);
|
|
||||||
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
||||||
|
if (kvp.ContainsKey("errorstring"))
|
||||||
if (!kvp.ContainsKey("errorstatus"))
|
vi.AddErrorString(kvp["errorstring"]);
|
||||||
{
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
if (kvp.ContainsKey("errorstring"))
|
|
||||||
vi.AddErrorString(kvp["errorstring"]);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else if (kvp.ContainsKey("inputChannel"))
|
else if (kvp.ContainsKey("inputChannel"))
|
||||||
{
|
{
|
||||||
string chan = kvp["inputChannel"];
|
|
||||||
int channelNo = int.Parse(chan);
|
|
||||||
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
||||||
|
if (kvp.ContainsKey("errorstring"))
|
||||||
if (!kvp.ContainsKey("errorstatus"))
|
|
||||||
{
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
vi.AddErrorString(kvp["errorstring"]);
|
vi.AddErrorString(kvp["errorstring"]);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else if (kvp.ContainsKey("io"))
|
else if (kvp.ContainsKey("io"))
|
||||||
{
|
{
|
||||||
|
|
@ -834,6 +820,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
|
|
||||||
if (vi.InputChannels == null)
|
if (vi.InputChannels == null)
|
||||||
vi.InputChannels = new Dictionary<int, ChannelEndpointDescription>();
|
vi.InputChannels = new Dictionary<int, ChannelEndpointDescription>();
|
||||||
|
|
||||||
for (int i = 0; i < numberOfInputs; i++)
|
for (int i = 0; i < numberOfInputs; i++)
|
||||||
{
|
{
|
||||||
string uri;
|
string uri;
|
||||||
|
|
@ -870,6 +857,24 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
vi.DataRead = totalRead;
|
vi.DataRead = totalRead;
|
||||||
vi.DataWritten = totalWritten;
|
vi.DataWritten = totalWritten;
|
||||||
|
|
||||||
|
if (vi.InputChannels != null)
|
||||||
|
{
|
||||||
|
foreach (int ch in vi.InputChannels.Keys)
|
||||||
|
{
|
||||||
|
long bytes = TryGetNumeric(kvp, "rb." + ch);
|
||||||
|
vi.InputChannels[ch].Size = bytes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (vi.OutputChannels != null)
|
||||||
|
{
|
||||||
|
foreach (int ch in vi.OutputChannels.Keys)
|
||||||
|
{
|
||||||
|
long bytes = TryGetNumeric(kvp, "wb." + ch);
|
||||||
|
vi.OutputChannels[ch].Size = bytes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
this.TotalDataRead += totalRead;
|
this.TotalDataRead += totalRead;
|
||||||
this.LocalReadData += localRead;
|
this.LocalReadData += localRead;
|
||||||
this.CrossPodDataRead += tempReadCrossRack;
|
this.CrossPodDataRead += tempReadCrossRack;
|
||||||
|
|
@ -878,17 +883,26 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
else if (kvp["io"] == "running")
|
else if (kvp["io"] == "running")
|
||||||
{
|
{
|
||||||
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
ExecutedVertexInstance vi = this.jobVertices.FindVertex(number, version);
|
||||||
|
|
||||||
foreach (int ch in vi.InputChannels.Keys)
|
if (vi.InputChannels != null)
|
||||||
{
|
{
|
||||||
long bytes = TryGetNumeric(kvp, "rb." + ch);
|
foreach (int ch in vi.InputChannels.Keys)
|
||||||
vi.InputChannels[ch].Size = bytes;
|
{
|
||||||
|
long bytes = TryGetNumeric(kvp, "rb." + ch);
|
||||||
|
vi.InputChannels[ch].Size = bytes;
|
||||||
|
|
||||||
|
bytes = TryGetNumeric(kvp, "tb." + ch);
|
||||||
|
vi.InputChannels[ch].TotalSize = bytes;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (int ch in vi.OutputChannels.Keys)
|
if (vi.InputChannels != null)
|
||||||
{
|
{
|
||||||
long bytes = TryGetNumeric(kvp, "wb." + ch);
|
foreach (int ch in vi.OutputChannels.Keys)
|
||||||
vi.OutputChannels[ch].Size = bytes;
|
{
|
||||||
|
long bytes = TryGetNumeric(kvp, "wb." + ch);
|
||||||
|
vi.OutputChannels[ch].Size = bytes;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
long totalRead = TryGetNumeric(kvp, "totalRead");
|
long totalRead = TryGetNumeric(kvp, "totalRead");
|
||||||
|
|
@ -1062,7 +1076,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
else if (m.Groups[1].Value == "ices")
|
else if (m.Groups[1].Value == "ices")
|
||||||
onevertex = false;
|
onevertex = false;
|
||||||
else
|
else
|
||||||
throw new CalypsoDryadException("Can't figure out if one or many vertices");
|
throw new DryadException("Can't figure out if one or many vertices");
|
||||||
|
|
||||||
IEnumerable<Tuple<string, int>> vertexList = DryadLinqJobInfo.ParseVertices(vertices);
|
IEnumerable<Tuple<string, int>> vertexList = DryadLinqJobInfo.ParseVertices(vertices);
|
||||||
|
|
||||||
|
|
@ -1084,7 +1098,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
}
|
}
|
||||||
|
|
||||||
if (vertexcount > 1 && onevertex)
|
if (vertexcount > 1 && onevertex)
|
||||||
throw new CalypsoDryadException("Expected one vertex, found " + vertexcount);
|
throw new DryadException("Expected one vertex, found " + vertexcount);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
|
@ -1225,7 +1239,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
if (vi.State == ExecutedVertexInstance.VertexState.Started)
|
if (vi.State == ExecutedVertexInstance.VertexState.Started)
|
||||||
{
|
{
|
||||||
Console.WriteLine("Timing information while vertex is still running " + vi);
|
Console.WriteLine("Timing information while vertex is still running " + vi);
|
||||||
//throw new CalypsoClusterException("Timing information for vertex still running: " + vi);
|
//throw new ClusterException("Timing information for vertex still running: " + vi);
|
||||||
}
|
}
|
||||||
DateTime last = vi.SetTiming(createtime, m.Groups[5].Value, m.Groups[6].Value, m.Groups[7].Value, m.Groups[8].Value, m.Groups[9].Value);
|
DateTime last = vi.SetTiming(createtime, m.Groups[5].Value, m.Groups[6].Value, m.Groups[7].Value, m.Groups[8].Value, m.Groups[9].Value);
|
||||||
if (last > this.lastTimestampSeen)
|
if (last > this.lastTimestampSeen)
|
||||||
|
|
@ -1245,7 +1259,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
throw new CalypsoDryadException("Unmatched timing information line " + line);
|
throw new DryadException("Unmatched timing information line " + line);
|
||||||
}
|
}
|
||||||
else if (line.Contains("Process has failed"))
|
else if (line.Contains("Process has failed"))
|
||||||
{
|
{
|
||||||
|
|
@ -1410,7 +1424,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
return retval;
|
return retval;
|
||||||
}
|
}
|
||||||
|
|
||||||
private ISharedStreamReader cachedStdoutReader = null;
|
private ISharedStreamReader cachedStdoutReader;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Remember how many lines were parsed, and skip them on a second invocation.
|
/// Remember how many lines were parsed, and skip them on a second invocation.
|
||||||
|
|
@ -1420,10 +1434,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// Parse the stdout.txt file from the job manager.
|
/// Parse the stdout.txt file from the job manager.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="file">File to parse.</param>
|
/// <param name="file">File to parse.</param>
|
||||||
/// <param name="statusReporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="updateProgress">Delegate used to report progress.</param>
|
|
||||||
/// <returns>True if the parsing succeeds.</returns>
|
/// <returns>True if the parsing succeeds.</returns>
|
||||||
private bool ParseStdout(IClusterResidentObject file, StatusReporter statusReporter, Action<int> updateProgress)
|
private bool ParseStdout(IClusterResidentObject file, CommManager manager)
|
||||||
{
|
{
|
||||||
int currentLine = 0;
|
int currentLine = 0;
|
||||||
if (this.stdoutLinesParsed == 0)
|
if (this.stdoutLinesParsed == 0)
|
||||||
|
|
@ -1440,15 +1453,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
string message = "Scanning JM stdout " + file;
|
string message = "Scanning JM stdout " + file;
|
||||||
if (filesize >= 0)
|
if (filesize >= 0)
|
||||||
message += string.Format("({0:N0} bytes)", filesize);
|
message += string.Format("({0:N0} bytes)", filesize);
|
||||||
statusReporter(message, StatusKind.LongOp);
|
manager.Status(message, StatusKind.LongOp);
|
||||||
|
|
||||||
if (this.cachedStdoutReader == null)
|
if (this.cachedStdoutReader == null)
|
||||||
this.cachedStdoutReader = file.GetStream();
|
this.cachedStdoutReader = file.GetStream();
|
||||||
if (this.cachedStdoutReader.Exception != null)
|
if (this.cachedStdoutReader.Exception != null)
|
||||||
{
|
{
|
||||||
statusReporter("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
|
manager.Status("Exception while opening stdout " + this.cachedStdoutReader.Exception.Message, StatusKind.Error);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
while (!this.cachedStdoutReader.EndOfStream)
|
while (!this.cachedStdoutReader.EndOfStream)
|
||||||
{
|
{
|
||||||
string line = this.cachedStdoutReader.ReadLine();
|
string line = this.cachedStdoutReader.ReadLine();
|
||||||
|
|
@ -1457,6 +1471,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
{
|
{
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
|
manager.Token.ThrowIfCancellationRequested();
|
||||||
int startLine = currentLine;
|
int startLine = currentLine;
|
||||||
bool completeLine = true;
|
bool completeLine = true;
|
||||||
try
|
try
|
||||||
|
|
@ -1465,7 +1480,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
statusReporter(string.Format("Line {0}: Exception {1}", currentLine, ex.Message), StatusKind.Error);
|
manager.Status(string.Format("Line {0}: Exception {1}", currentLine, ex.Message), StatusKind.Error);
|
||||||
Console.WriteLine("Line {0}: Exception {1}", currentLine, ex);
|
Console.WriteLine("Line {0}: Exception {1}", currentLine, ex);
|
||||||
}
|
}
|
||||||
if (!completeLine)
|
if (!completeLine)
|
||||||
|
|
@ -1485,7 +1500,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
currentLine++;
|
currentLine++;
|
||||||
if (currentLine % 100 == 0 && filesize > 0)
|
if (currentLine % 100 == 0 && filesize > 0)
|
||||||
{
|
{
|
||||||
updateProgress(Math.Min(100, (int)(100 * readbytes / filesize)));
|
manager.Progress(Math.Min(100, (int)(100 * readbytes / filesize)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1500,13 +1515,16 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
// we are done with this stream
|
// we are done with this stream
|
||||||
if (this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Failed ||
|
if (this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Failed ||
|
||||||
this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Successful)
|
this.ManagerVertex.State == ExecutedVertexInstance.VertexState.Successful)
|
||||||
|
{
|
||||||
this.cachedStdoutReader.Close();
|
this.cachedStdoutReader.Close();
|
||||||
|
this.cachedStdoutReader = null; // will force reopening if refreshed
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
statusReporter("Exception while reading stdout " + e.Message, StatusKind.Error);
|
manager.Status("Exception while reading stdout " + e.Message, StatusKind.Error);
|
||||||
Trace.TraceInformation(e.ToString());
|
Trace.TraceInformation(e.ToString());
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
@ -1592,7 +1610,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
string line = sr.ReadLine();
|
string line = sr.ReadLine();
|
||||||
if (!line.Contains("DryadProfiler")) continue;
|
if (!line.Contains("DryadProfiler")) continue;
|
||||||
|
|
||||||
CosmosLogEntry le = new CosmosLogEntry(line);
|
DryadLogEntry le = new DryadLogEntry(line);
|
||||||
if (le.Subsystem != "DryadProfiler") continue;
|
if (le.Subsystem != "DryadProfiler") continue;
|
||||||
if (!le.Message.EndsWith("channel status")) continue;
|
if (!le.Message.EndsWith("channel status")) continue;
|
||||||
|
|
||||||
|
|
@ -1905,7 +1923,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Parse the query plan: cluster-specific.
|
/// Parse the query plan: cluster-specific.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected abstract void ParseQueryPlan();
|
/// <param name="manager">Communication manager.</param>
|
||||||
|
protected abstract void ParseQueryPlan(CommManager manager);
|
||||||
|
|
||||||
int fictitiousStages;
|
int fictitiousStages;
|
||||||
|
|
||||||
|
|
@ -2009,11 +2028,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// Factory: create the plan for a given job.
|
/// Factory: create the plan for a given job.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="dryadLinqJobInfo">Job to create plan for.</param>
|
/// <param name="dryadLinqJobInfo">Job to create plan for.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
|
||||||
/// <returns>The plan or null.</returns>
|
/// <returns>The plan or null.</returns>
|
||||||
public static DryadJobStaticPlan CreatePlan(DryadLinqJobInfo dryadLinqJobInfo, StatusReporter reporter)
|
/// <param name="manager">Communication manager.</param>
|
||||||
|
public static DryadJobStaticPlan CreatePlan(DryadLinqJobInfo dryadLinqJobInfo, CommManager manager)
|
||||||
{
|
{
|
||||||
reporter("Trying to build static plan", StatusKind.LongOp);
|
manager.Status("Trying to build static plan", StatusKind.LongOp);
|
||||||
ClusterConfiguration config = dryadLinqJobInfo.ClusterConfiguration;
|
ClusterConfiguration config = dryadLinqJobInfo.ClusterConfiguration;
|
||||||
IClusterResidentObject file = config.JobQueryPlan(dryadLinqJobInfo.Summary);
|
IClusterResidentObject file = config.JobQueryPlan(dryadLinqJobInfo.Summary);
|
||||||
if (config is CacheClusterConfiguration)
|
if (config is CacheClusterConfiguration)
|
||||||
|
|
@ -2025,12 +2044,12 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
{
|
{
|
||||||
retval = new DryadLinqJobStaticPlan(config, file.GetStream());
|
retval = new DryadLinqJobStaticPlan(config, file.GetStream());
|
||||||
}
|
}
|
||||||
retval.ParseQueryPlan();
|
retval.ParseQueryPlan(manager);
|
||||||
return retval;
|
return retval;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
reporter("Exception while looking for plan " + file.Exception.Message, StatusKind.Error);
|
manager.Status("Exception while looking for plan " + file.Exception.Message, StatusKind.Error);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -2056,9 +2075,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Parse an XML query plan and represent that information.
|
/// Parse an XML query plan and represent that information.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected override void ParseQueryPlan()
|
/// <param name="manager">Communicaton manager.</param>
|
||||||
|
protected override void ParseQueryPlan(CommManager manager)
|
||||||
{
|
{
|
||||||
string planString = this.planStream.ReadToEnd();
|
string planString = this.planStream.ReadToEnd(manager.Token);
|
||||||
|
|
||||||
XDocument plan = XDocument.Parse(planString);
|
XDocument plan = XDocument.Parse(planString);
|
||||||
// ReSharper disable PossibleNullReferenceException
|
// ReSharper disable PossibleNullReferenceException
|
||||||
|
|
@ -2104,7 +2124,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
info.Arity = Connection.ConnectionType.AllToAll;
|
info.Arity = Connection.ConnectionType.AllToAll;
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
throw new CalypsoDryadException("Don't know about connection of type " + connection);
|
throw new DryadException("Don't know about connection of type " + connection);
|
||||||
}
|
}
|
||||||
switch (cht)
|
switch (cht)
|
||||||
{
|
{
|
||||||
|
|
@ -2118,7 +2138,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
info.ChannelKind = Connection.ChannelType.Fifo;
|
info.ChannelKind = Connection.ChannelType.Fifo;
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
throw new CalypsoDryadException("Don't know about channel of type " + cht);
|
throw new DryadException("Don't know about channel of type " + cht);
|
||||||
}
|
}
|
||||||
this.perNodeConnectionInfo.Add(stage.Id, info);
|
this.perNodeConnectionInfo.Add(stage.Id, info);
|
||||||
}
|
}
|
||||||
|
|
@ -2247,14 +2267,15 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Parse the Algebra file.
|
/// Parse the Algebra file.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private void ParseAlgebra()
|
/// <param name="manager">Communication manager.</param>
|
||||||
|
private void ParseAlgebra(CommManager manager)
|
||||||
{
|
{
|
||||||
// TODO: this parser is not really complete, as I don't understand the semantics of all xml elements.
|
// TODO: this parser is not really complete, as I don't understand the semantics of all xml elements.
|
||||||
Dictionary<string, string> outToStage = new Dictionary<string, string>(); // map an output to a stage name. Assume that ios have unique names.
|
Dictionary<string, string> outToStage = new Dictionary<string, string>(); // map an output to a stage name. Assume that ios have unique names.
|
||||||
Dictionary<string, List<string>> inputs = new Dictionary<string, List<string>>();
|
Dictionary<string, List<string>> inputs = new Dictionary<string, List<string>>();
|
||||||
|
|
||||||
// <CsJobAlgebra> <graph> <process> ...
|
// <CsJobAlgebra> <graph> <process> ...
|
||||||
string planString = this.planStream.ReadToEnd();
|
string planString = this.planStream.ReadToEnd(manager.Token);
|
||||||
XDocument plan = XDocument.Parse(planString);
|
XDocument plan = XDocument.Parse(planString);
|
||||||
// ReSharper disable PossibleNullReferenceException
|
// ReSharper disable PossibleNullReferenceException
|
||||||
XElement graph = plan.Root.Element("graph"); // graph node, children are stages
|
XElement graph = plan.Root.Element("graph"); // graph node, children are stages
|
||||||
|
|
@ -2416,13 +2437,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Parse the vertex definition file.
|
/// Parse the vertex definition file.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private void ParseVertexDef()
|
private void ParseVertexDef(CommManager manager)
|
||||||
{
|
{
|
||||||
if (this.vertexDef.Exception != null)
|
if (this.vertexDef.Exception != null)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
// <ScopeVertices> <ScopeVertex> <operator> <input> </input> <output> </output>
|
// <ScopeVertices> <ScopeVertex> <operator> <input> </input> <output> </output>
|
||||||
string planString = this.vertexDef.ReadToEnd();
|
string planString = this.vertexDef.ReadToEnd(manager.Token);
|
||||||
XDocument vxDef = XDocument.Parse(planString);
|
XDocument vxDef = XDocument.Parse(planString);
|
||||||
|
|
||||||
XElement vertices = vxDef.Root;
|
XElement vertices = vxDef.Root;
|
||||||
|
|
@ -2469,10 +2490,11 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Parse the query plan for a Scope job.
|
/// Parse the query plan for a Scope job.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected override void ParseQueryPlan()
|
/// <param name="manager">Communication manager.</param>
|
||||||
|
protected override void ParseQueryPlan(CommManager manager)
|
||||||
{
|
{
|
||||||
this.ParseAlgebra();
|
this.ParseAlgebra(manager);
|
||||||
this.ParseVertexDef();
|
this.ParseVertexDef(manager);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2653,9 +2675,13 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string LocalPath { get; protected set; }
|
public string LocalPath { get; protected set; }
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// How big is the channel (0 if it cannot be determined, e.g. FIFO, -1 if the channel data cannot be retrieved).
|
/// The actual data read/written so far (0 if it cannot be determined, e.g. FIFO, -1 if the channel data cannot be retrieved).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long Size { get; set; }
|
public long Size { get; set; }
|
||||||
|
/// <summary>
|
||||||
|
/// How much of the channel was
|
||||||
|
/// </summary>
|
||||||
|
public long TotalSize { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// String representation of the endpoint.
|
/// String representation of the endpoint.
|
||||||
|
|
@ -2664,7 +2690,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
{
|
{
|
||||||
string uritype = this.UriType;
|
string uritype = this.UriType;
|
||||||
string localpath = this.LocalPath;
|
string localpath = this.LocalPath;
|
||||||
return string.Format("{0,4} {1,20:N0} {2}://{3}", this.Number, this.Size, uritype, localpath);
|
if (this.TotalSize == 0)
|
||||||
|
return string.Format("{0,4} {1,20:N0} {2}://{3}", this.Number, this.Size, uritype, localpath);
|
||||||
|
else
|
||||||
|
return string.Format("{0,4} {1,20:N0}/{2,20:N0} {3}://{4}", this.Number, this.Size, this.TotalSize, uritype, localpath);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -2684,7 +2713,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
|
|
||||||
int sepindex = uri.IndexOf("://");
|
int sepindex = uri.IndexOf("://");
|
||||||
if (sepindex < 0)
|
if (sepindex < 0)
|
||||||
throw new CalypsoDryadException("Channel URI " + uri + " does not contain separator ://");
|
throw new DryadException("Channel URI " + uri + " does not contain separator ://");
|
||||||
|
|
||||||
this.UriType = uri.Substring(0, sepindex);
|
this.UriType = uri.Substring(0, sepindex);
|
||||||
// some HPC URIs use the compression scheme as an "option" (not really defined for file:// uris, but...)
|
// some HPC URIs use the compression scheme as an "option" (not really defined for file:// uris, but...)
|
||||||
|
|
@ -2737,10 +2766,10 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
this.Number = number;
|
this.Number = number;
|
||||||
int sepindex = uri.IndexOf("://");
|
int sepindex = uri.IndexOf("://");
|
||||||
if (sepindex < 0)
|
if (sepindex < 0)
|
||||||
throw new CalypsoClusterException("Channel URI " + uri + " does not contain separator ://");
|
throw new ClusterException("Channel URI " + uri + " does not contain separator ://");
|
||||||
this.UriType = uri.Substring(0, sepindex);
|
this.UriType = uri.Substring(0, sepindex);
|
||||||
this.LocalPath = uri.Substring(sepindex + 3);
|
this.LocalPath = uri.Substring(sepindex + 3);
|
||||||
this.Size = size;
|
this.TotalSize = size;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2883,7 +2912,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
this.StdoutFile = job.ClusterConfiguration.ProcessStdoutFile(this.ProcessIdentifier, false, machine, job.Summary);
|
this.StdoutFile = job.ClusterConfiguration.ProcessStdoutFile(this.ProcessIdentifier, false, machine, job.Summary);
|
||||||
this.SetState(VertexState.Started);
|
this.SetState(VertexState.Started);
|
||||||
if (approxStartTime == DateTime.MinValue)
|
if (approxStartTime == DateTime.MinValue)
|
||||||
throw new CalypsoDryadException("Unexpected small start time for vertex");
|
throw new DryadException("Unexpected small start time for vertex");
|
||||||
this.LogDirectory = job.ClusterConfiguration.ProcessLogDirectory(this.ProcessIdentifier, false, machine, job.Summary);
|
this.LogDirectory = job.ClusterConfiguration.ProcessLogDirectory(this.ProcessIdentifier, false, machine, job.Summary);
|
||||||
this.LogFilesPattern = job.ClusterConfiguration.VertexLogFilesPattern(false, job.Summary);
|
this.LogFilesPattern = job.ClusterConfiguration.VertexLogFilesPattern(false, job.Summary);
|
||||||
this.UniqueID = uniqueId;
|
this.UniqueID = uniqueId;
|
||||||
|
|
@ -3049,7 +3078,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
{
|
{
|
||||||
Trace.TraceInformation("Vertex {0} which is not started is still running?", this.Name);
|
Trace.TraceInformation("Vertex {0} which is not started is still running?", this.Name);
|
||||||
return;
|
return;
|
||||||
//throw new CalypsoClusterException("Vertex which is not started is still running?");
|
//throw new ClusterException("Vertex which is not started is still running?");
|
||||||
}
|
}
|
||||||
if (this.Start > when)
|
if (this.Start > when)
|
||||||
// This can happen if the cluster clocks are not synchronized with the local machine clocks.
|
// This can happen if the cluster clocks are not synchronized with the local machine clocks.
|
||||||
|
|
@ -3088,7 +3117,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
TimeSpan total = TimeSpan.FromSeconds(totSeconds);
|
TimeSpan total = TimeSpan.FromSeconds(totSeconds);
|
||||||
DateTime totalTime = creation + total;
|
DateTime totalTime = creation + total;
|
||||||
if (totSeconds < 0)
|
if (totSeconds < 0)
|
||||||
throw new CalypsoDryadException("Negative total time for vertex " + this.Name);
|
throw new DryadException("Negative total time for vertex " + this.Name);
|
||||||
|
|
||||||
// if the vertex has no machine just ignore the times
|
// if the vertex has no machine just ignore the times
|
||||||
if (string.IsNullOrEmpty(this.Machine))
|
if (string.IsNullOrEmpty(this.Machine))
|
||||||
|
|
@ -3209,9 +3238,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <param name="uriprefix">If the channel is an output, prefix the path with this; this is null for inputs.</param>
|
/// <param name="uriprefix">If the channel is an output, prefix the path with this; this is null for inputs.</param>
|
||||||
/// <param name="skip">If true, do not return anything (still useful to advance the stream reader).</param>
|
/// <param name="skip">If true, do not return anything (still useful to advance the stream reader).</param>
|
||||||
/// <param name="fast">If true the channel sizes are not discovered; this is much faster, since no remote machines are queried for files.</param>
|
/// <param name="fast">If true the channel sizes are not discovered; this is much faster, since no remote machines are queried for files.</param>
|
||||||
/// <param name="updateProgress">Delegate used to report progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
private Dictionary<int, ChannelEndpointDescription> DiscoverOriginalInfoChannels(ISharedStreamReader sr, string uriprefix, bool skip, bool fast, CommManager manager)
|
||||||
private Dictionary<int, ChannelEndpointDescription> DiscoverOriginalInfoChannels(ISharedStreamReader sr, string uriprefix, bool skip, bool fast, StatusReporter reporter, Action<int> updateProgress)
|
|
||||||
{
|
{
|
||||||
bool isInput = uriprefix == null;
|
bool isInput = uriprefix == null;
|
||||||
|
|
||||||
|
|
@ -3231,21 +3259,18 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
string channel = sr.ReadLine();
|
string channel = sr.ReadLine();
|
||||||
if (channel == null)
|
if (channel == null)
|
||||||
{
|
{
|
||||||
if (updateProgress != null)
|
manager.Progress(100);
|
||||||
updateProgress(100);
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
if (!skip)
|
if (!skip)
|
||||||
{
|
{
|
||||||
ChannelEndpointDescription desc = new ChannelEndpointDescription(isInput, i, channel, uriprefix, fast, reporter);
|
ChannelEndpointDescription desc = new ChannelEndpointDescription(isInput, i, channel, uriprefix, fast, manager.Status);
|
||||||
channels.Add(i, desc);
|
channels.Add(i, desc);
|
||||||
if (updateProgress != null)
|
manager.Progress(i * 100 / channelCount);
|
||||||
updateProgress(i * 100 / channelCount);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (updateProgress != null)
|
manager.Progress(100);
|
||||||
updateProgress(100);
|
|
||||||
if (skip)
|
if (skip)
|
||||||
return null;
|
return null;
|
||||||
return channels;
|
return channels;
|
||||||
|
|
@ -3258,9 +3283,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <param name="inputs">If true discover the inputs.</param>
|
/// <param name="inputs">If true discover the inputs.</param>
|
||||||
/// <param name="outputs">If true discover the outputs.</param>
|
/// <param name="outputs">If true discover the outputs.</param>
|
||||||
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
|
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
|
||||||
/// <param name="progress">Delegate used to report progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
public bool DiscoverOriginalInfoChannels(bool inputs, bool outputs, bool fast, CommManager manager)
|
||||||
public bool DiscoverOriginalInfoChannels(bool inputs, bool outputs, bool fast, StatusReporter reporter, Action<int> progress)
|
|
||||||
{
|
{
|
||||||
string filename = string.Format("vertex-{0}-{1}-rerun-originalInfo.txt", this.Number, this.Version);
|
string filename = string.Format("vertex-{0}-{1}-rerun-originalInfo.txt", this.Number, this.Version);
|
||||||
bool success = true;
|
bool success = true;
|
||||||
|
|
@ -3270,7 +3294,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
// skip discovery
|
// skip discovery
|
||||||
inputs = false;
|
inputs = false;
|
||||||
ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream();
|
ISharedStreamReader sr = this.WorkDirectory.GetFile(filename).GetStream();
|
||||||
var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, reporter, progress);
|
var channels = this.DiscoverOriginalInfoChannels(sr, null, !inputs, fast, manager);
|
||||||
if (channels == null)
|
if (channels == null)
|
||||||
{
|
{
|
||||||
if (inputs)
|
if (inputs)
|
||||||
|
|
@ -3281,7 +3305,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
if (this.OutputChannels != null)
|
if (this.OutputChannels != null)
|
||||||
// skip discovery
|
// skip discovery
|
||||||
outputs = false;
|
outputs = false;
|
||||||
channels = this.DiscoverOriginalInfoChannels(sr, this.WorkDirectory.ToString(), !outputs, fast, reporter, progress);
|
channels = this.DiscoverOriginalInfoChannels(sr, this.WorkDirectory.ToString(), !outputs, fast, manager);
|
||||||
if (channels == null)
|
if (channels == null)
|
||||||
{
|
{
|
||||||
if (outputs)
|
if (outputs)
|
||||||
|
|
@ -3300,28 +3324,27 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <param name="inputs">If true discover the inputs.</param>
|
/// <param name="inputs">If true discover the inputs.</param>
|
||||||
/// <param name="outputs">If true discover the outputs.</param>
|
/// <param name="outputs">If true discover the outputs.</param>
|
||||||
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
|
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
|
||||||
/// <param name="progress">Delegate used to report progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
|
||||||
// ReSharper disable UnusedParameter.Global
|
// ReSharper disable UnusedParameter.Global
|
||||||
public bool DiscoverScopeChannels(bool inputs, bool outputs, bool fast, StatusReporter reporter, Action<int> progress)
|
public bool DiscoverScopeChannels(bool inputs, bool outputs, bool fast, CommManager manager)
|
||||||
// ReSharper restore UnusedParameter.Global
|
// ReSharper restore UnusedParameter.Global
|
||||||
{
|
{
|
||||||
// find the xml file
|
// find the xml file
|
||||||
var files = this.WorkDirectory.GetFilesAndFolders("vcmdStart*.xml").ToList();
|
var files = this.WorkDirectory.GetFilesAndFolders("vcmdStart*.xml").ToList();
|
||||||
if (files.Count != 1)
|
if (files.Count != 1)
|
||||||
{
|
{
|
||||||
reporter("Cannot locate vcmdStart*.xml file", StatusKind.Error);
|
manager.Status("Cannot locate vcmdStart*.xml file", StatusKind.Error);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
ISharedStreamReader sr = files.First().GetStream();
|
ISharedStreamReader sr = files.First().GetStream();
|
||||||
if (sr.Exception != null)
|
if (sr.Exception != null)
|
||||||
{
|
{
|
||||||
reporter("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);
|
manager.Status("Error reading vcmdStart*.xml file" + sr.Exception.Message, StatusKind.Error);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReSharper disable PossibleNullReferenceException
|
// ReSharper disable PossibleNullReferenceException
|
||||||
XDocument plan = XDocument.Parse(sr.ReadToEnd());
|
XDocument plan = XDocument.Parse(sr.ReadToEnd(manager.Token));
|
||||||
if (inputs && this.InputChannels == null)
|
if (inputs && this.InputChannels == null)
|
||||||
{
|
{
|
||||||
var channels = new Dictionary<int, ChannelEndpointDescription>();
|
var channels = new Dictionary<int, ChannelEndpointDescription>();
|
||||||
|
|
@ -3365,9 +3388,8 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
/// <param name="inputs">If true discover the inputs.</param>
|
/// <param name="inputs">If true discover the inputs.</param>
|
||||||
/// <param name="outputs">If true discover the outputs.</param>
|
/// <param name="outputs">If true discover the outputs.</param>
|
||||||
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
|
/// <param name="fast">If true do not discover the channel sizes (much faster).</param>
|
||||||
/// <param name="progress">Delegate used to report progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
public bool DiscoverChannels(bool inputs, bool outputs, bool fast, CommManager manager)
|
||||||
public bool DiscoverChannels(bool inputs, bool outputs, bool fast, StatusReporter reporter, Action<int> progress)
|
|
||||||
{
|
{
|
||||||
// check if the result is already cached
|
// check if the result is already cached
|
||||||
if ((this.InputChannels != null || !inputs) &&
|
if ((this.InputChannels != null || !inputs) &&
|
||||||
|
|
@ -3395,7 +3417,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
|
|
||||||
if (wd is UNCFile)
|
if (wd is UNCFile)
|
||||||
{
|
{
|
||||||
result = this.DiscoverOriginalInfoChannels(inputs, outputs, fast, reporter, progress);
|
result = this.DiscoverOriginalInfoChannels(inputs, outputs, fast, manager);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
|
@ -3448,9 +3470,9 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
internal void Update(string name, string guid)
|
internal void Update(string name, string guid)
|
||||||
{
|
{
|
||||||
if (this.State != VertexState.Cancelled && this.State != VertexState.Abandoned)
|
if (this.State != VertexState.Cancelled && this.State != VertexState.Abandoned)
|
||||||
throw new CalypsoDryadException("Updating a non-cancelled/abandoned vertex");
|
throw new DryadException("Updating a non-cancelled/abandoned vertex");
|
||||||
if (this.Name != name)
|
if (this.Name != name)
|
||||||
throw new CalypsoDryadException("Vertex changed name");
|
throw new DryadException("Vertex changed name");
|
||||||
this.UniqueID = guid;
|
this.UniqueID = guid;
|
||||||
this.SetState(VertexState.Created);
|
this.SetState(VertexState.Created);
|
||||||
// the stdoutfile is expected to change, so I don't invalidate the cache
|
// the stdoutfile is expected to change, so I don't invalidate the cache
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,21 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
|
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
|
||||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||||
</packages>
|
</packages>
|
||||||
|
|
@ -22,13 +22,13 @@ limitations under the License.
|
||||||
#undef USE_DSC
|
#undef USE_DSC
|
||||||
#undef USE_TIDYFS
|
#undef USE_TIDYFS
|
||||||
|
|
||||||
using Microsoft.Research.Calypso.Tools;
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
using Microsoft.Research.Tools;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
namespace Microsoft.Research.JobObjectModel
|
||||||
{
|
{
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -197,7 +197,7 @@ namespace Microsoft.Research.Calypso.JobObjectModel
|
||||||
{
|
{
|
||||||
var sourceInputs = plan.GetStageConnections(source, true).ToList();
|
var sourceInputs = plan.GetStageConnections(source, true).ToList();
|
||||||
if (sourceInputs.Count() != 1)
|
if (sourceInputs.Count() != 1)
|
||||||
throw new CalypsoDryadException("Unexpected number of inputs for stage " + source.Name);
|
throw new DryadException("Unexpected number of inputs for stage " + source.Name);
|
||||||
source = sourceInputs.First().From;
|
source = sourceInputs.First().From;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,6 @@
|
||||||
|
|
||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
# Visual Studio 2013
|
# Visual Studio 2012
|
||||||
VisualStudioVersion = 12.0.30110.0
|
|
||||||
MinimumVisualStudioVersion = 10.0.40219.1
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JobBrowser", "JobBrowser\JobBrowser.csproj", "{EDDD2E0B-A52B-4E25-9436-B874017673FF}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JobBrowser", "JobBrowser\JobBrowser.csproj", "{EDDD2E0B-A52B-4E25-9436-B874017673FF}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools", "Tools\Tools.csproj", "{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools", "Tools\Tools.csproj", "{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}"
|
||||||
|
|
@ -13,72 +11,26 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UsefulForms", "UsefulForms\
|
||||||
EndProject
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|Any CPU = Debug|Any CPU
|
|
||||||
Debug|Mixed Platforms = Debug|Mixed Platforms
|
|
||||||
Debug|x64 = Debug|x64
|
Debug|x64 = Debug|x64
|
||||||
Debug|x86 = Debug|x86
|
|
||||||
Release|Any CPU = Release|Any CPU
|
|
||||||
Release|Mixed Platforms = Release|Mixed Platforms
|
|
||||||
Release|x64 = Release|x64
|
Release|x64 = Release|x64
|
||||||
Release|x86 = Release|x86
|
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x64.ActiveCfg = Debug|x64
|
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x64.Build.0 = Debug|x64
|
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x64.Build.0 = Debug|x64
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x86.ActiveCfg = Debug|Any CPU
|
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Any CPU.Build.0 = Release|Any CPU
|
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x64.ActiveCfg = Release|x64
|
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x64.ActiveCfg = Release|x64
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x64.Build.0 = Release|x64
|
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x64.Build.0 = Release|x64
|
||||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x86.ActiveCfg = Release|Any CPU
|
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x64.ActiveCfg = Debug|x64
|
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x64.Build.0 = Debug|x64
|
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x64.Build.0 = Debug|x64
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x86.ActiveCfg = Debug|Any CPU
|
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Any CPU.Build.0 = Release|Any CPU
|
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x64.ActiveCfg = Release|x64
|
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x64.ActiveCfg = Release|x64
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x64.Build.0 = Release|x64
|
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x64.Build.0 = Release|x64
|
||||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x86.ActiveCfg = Release|Any CPU
|
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x64.ActiveCfg = Debug|x64
|
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x64.Build.0 = Debug|x64
|
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x64.Build.0 = Debug|x64
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x86.ActiveCfg = Debug|Any CPU
|
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Any CPU.Build.0 = Release|Any CPU
|
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x64.ActiveCfg = Release|x64
|
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x64.ActiveCfg = Release|x64
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x64.Build.0 = Release|x64
|
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x64.Build.0 = Release|x64
|
||||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x86.ActiveCfg = Release|Any CPU
|
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x64.ActiveCfg = Debug|x64
|
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x64.Build.0 = Debug|x64
|
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x64.Build.0 = Debug|x64
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x86.ActiveCfg = Debug|Any CPU
|
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Any CPU.Build.0 = Release|Any CPU
|
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x64.ActiveCfg = Release|x64
|
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x64.ActiveCfg = Release|x64
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x64.Build.0 = Release|x64
|
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x64.Build.0 = Release|x64
|
||||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x86.ActiveCfg = Release|Any CPU
|
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
|
|
|
||||||
|
|
@ -28,11 +28,11 @@ using System.Linq;
|
||||||
using System.Net;
|
using System.Net;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using Microsoft.Research.Calypso.JobObjectModel;
|
using Microsoft.Research.JobObjectModel;
|
||||||
using Microsoft.Research.Calypso.Tools;
|
using Microsoft.Research.Tools;
|
||||||
using Microsoft.Research.Calypso.UsefulForms;
|
using Microsoft.Research.UsefulForms;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Class to browse jobs on cluster, copy, summarize and start visualization.
|
/// Class to browse jobs on cluster, copy, summarize and start visualization.
|
||||||
|
|
@ -73,7 +73,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
private TextWriterTraceListener LogFile;
|
private TextWriterTraceListener LogFile;
|
||||||
|
|
||||||
private BackgroundWorkQueue queue;
|
private BackgroundWorkQueue queue;
|
||||||
private BackgroundWorker queueWorker;
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Jobs from the cluster.
|
/// Jobs from the cluster.
|
||||||
|
|
@ -92,8 +91,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.InitializeComponent();
|
this.InitializeComponent();
|
||||||
this.status = new StatusWriter(this.statuslabel, this.statusStrip, this.Status);
|
this.status = new StatusWriter(this.statuslabel, this.statusStrip, this.Status);
|
||||||
|
|
||||||
this.queueWorker = new BackgroundWorker();
|
BackgroundWorker queueWorker = new BackgroundWorker();
|
||||||
this.queue = new BackgroundWorkQueue(this.queueWorker);
|
this.queue = new BackgroundWorkQueue(queueWorker, null, null);
|
||||||
|
|
||||||
this.completeJobsList = new List<ClusterJobInformation>();
|
this.completeJobsList = new List<ClusterJobInformation>();
|
||||||
this.refreshTimer = new Timer();
|
this.refreshTimer = new Timer();
|
||||||
|
|
@ -235,7 +234,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.toolStripMenuItem_job.Enabled = true;
|
this.toolStripMenuItem_job.Enabled = true;
|
||||||
|
|
||||||
var item = new BackgroundWorkItem<List<ClusterJobInformation>>(
|
var item = new BackgroundWorkItem<List<ClusterJobInformation>>(
|
||||||
(s, p) => BuildClusterJobList(s, p, this.clusterStatus, this.SelectedVirtualCluster),
|
m => BuildClusterJobList(m, this.clusterStatus, this.SelectedVirtualCluster),
|
||||||
this.JobListRetrieved,
|
this.JobListRetrieved,
|
||||||
"getJobs");
|
"getJobs");
|
||||||
this.Queue(item);
|
this.Queue(item);
|
||||||
|
|
@ -250,6 +249,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
|
|
||||||
private void JobListRetrieved(bool cancelled, List<ClusterJobInformation> jobs)
|
private void JobListRetrieved(bool cancelled, List<ClusterJobInformation> jobs)
|
||||||
{
|
{
|
||||||
|
if (cancelled) return;
|
||||||
|
|
||||||
this.filteredDataGridView.DataGridView.ClearSelection();
|
this.filteredDataGridView.DataGridView.ClearSelection();
|
||||||
this.completeJobsList = jobs;
|
this.completeJobsList = jobs;
|
||||||
this.clusterJobs.SetItems(this.completeJobsList);
|
this.clusterJobs.SetItems(this.completeJobsList);
|
||||||
|
|
@ -274,12 +275,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// Talk to the web server and build the list of clustr jobs; used it to populate the upper panel.
|
/// Talk to the web server and build the list of clustr jobs; used it to populate the upper panel.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="virtualCluster">Virtual cluster selected; defined only for Scope clusters.</param>
|
/// <param name="virtualCluster">Virtual cluster selected; defined only for Scope clusters.</param>
|
||||||
/// <param name="progress">Reports progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reporter">Reports status.</param>
|
|
||||||
/// <param name="status">Cluster to scan.</param>
|
/// <param name="status">Cluster to scan.</param>
|
||||||
private static List<ClusterJobInformation> BuildClusterJobList(StatusReporter reporter, Action<int> progress, ClusterStatus status, string virtualCluster)
|
private static List<ClusterJobInformation> BuildClusterJobList(CommManager manager, ClusterStatus status, string virtualCluster)
|
||||||
{
|
{
|
||||||
return status.GetClusterJobList(virtualCluster, reporter, progress).ToList();
|
return status.GetClusterJobList(virtualCluster, manager).ToList();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -311,7 +311,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
if (js == null)
|
if (js == null)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
DryadLinqJobInfo job = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.clusterStatus.Config, js, false, this.Status, delegate {});
|
// TODO: this should run in the background
|
||||||
|
CommManager manager = new CommManager(this.Status, delegate { }, new System.Threading.CancellationTokenSource().Token);
|
||||||
|
DryadLinqJobInfo job = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.clusterStatus.Config, js, false, manager);
|
||||||
if (job != null)
|
if (job != null)
|
||||||
{
|
{
|
||||||
JobBrowser browser = new JobBrowser(job);
|
JobBrowser browser = new JobBrowser(job);
|
||||||
|
|
@ -366,7 +368,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
IEnumerable<ClusterJobInformation> ti = this.SelectedJobs();
|
IEnumerable<ClusterJobInformation> ti = this.SelectedJobs();
|
||||||
this.Status("Starting job browser...", StatusKind.LongOp);
|
this.Status("Starting job browser...", StatusKind.LongOp);
|
||||||
IEnumerable<DryadLinqJobSummary> jobs = ti.Select(t => t.DiscoverDryadLinqJob(this.clusterStatus, this.Status)).ToList();
|
IEnumerable<DryadLinqJobSummary> jobs = ti.Select(t => t.DiscoverDryadLinqJob(this.clusterStatus, this.Status)).ToList();
|
||||||
IEnumerable<DryadLinqJobInfo> detailed = jobs.Select(j => DryadLinqJobInfo.CreateDryadLinqJobInfo(this.clusterStatus.Config, j, false, this.Status, delegate { }));
|
|
||||||
|
CommManager manager = new CommManager(this.Status, delegate { }, new System.Threading.CancellationTokenSource().Token);
|
||||||
|
IEnumerable<DryadLinqJobInfo> detailed = jobs.Select(j => DryadLinqJobInfo.CreateDryadLinqJobInfo(this.clusterStatus.Config, j, false, manager));
|
||||||
foreach (DryadLinqJobInfo j in detailed)
|
foreach (DryadLinqJobInfo j in detailed)
|
||||||
{
|
{
|
||||||
if (j == null) continue;
|
if (j == null) continue;
|
||||||
|
|
@ -442,7 +446,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
IEnumerable<DryadLinqJobSummary> jobs = todo.Select(j => j.DiscoverDryadLinqJob(this.clusterStatus, this.Status)).Where(j => j != null);
|
IEnumerable<DryadLinqJobSummary> jobs = todo.Select(j => j.DiscoverDryadLinqJob(this.clusterStatus, this.Status)).Where(j => j != null);
|
||||||
|
|
||||||
var item = new BackgroundWorkItem<bool>(
|
var item = new BackgroundWorkItem<bool>(
|
||||||
(s, p) => ClusterWork.CancelJobs(jobs, this.clusterStatus, s, p),
|
m => ClusterWork.CancelJobs(jobs, this.clusterStatus, m),
|
||||||
(c, b) => { },
|
(c, b) => { },
|
||||||
"cancel");
|
"cancel");
|
||||||
this.Queue(item);
|
this.Queue(item);
|
||||||
|
|
@ -504,8 +508,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.autoRefreshToolStripMenuItem.Checked = this.formSettings.AutoRefresh;
|
this.autoRefreshToolStripMenuItem.Checked = this.formSettings.AutoRefresh;
|
||||||
|
|
||||||
this.AddClusterNameToMenu("<add>");
|
this.AddClusterNameToMenu("<add>");
|
||||||
|
this.AddClusterNameToMenu("<scan>");
|
||||||
|
|
||||||
ClusterConfiguration.ReconstructKnownCluster(this.formSettings.KnownClusters);
|
ClusterConfiguration.ReconstructKnownCluster(this.formSettings.KnownClusters);
|
||||||
|
|
||||||
|
int found = 0;
|
||||||
IEnumerable<string> clusters = ClusterConfiguration.GetKnownClusterNames();
|
IEnumerable<string> clusters = ClusterConfiguration.GetKnownClusterNames();
|
||||||
foreach (string c in clusters)
|
foreach (string c in clusters)
|
||||||
{
|
{
|
||||||
|
|
@ -515,7 +522,12 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
{
|
{
|
||||||
(config as CacheClusterConfiguration).StartCaching();
|
(config as CacheClusterConfiguration).StartCaching();
|
||||||
}
|
}
|
||||||
|
found++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (found == 0)
|
||||||
|
// try to find them by scanning
|
||||||
|
this.ScanClusters();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -539,6 +551,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
newItem.Click += this.AddNewCluster;
|
newItem.Click += this.AddNewCluster;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (clusterName == "<scan>")
|
||||||
|
{
|
||||||
|
newItem.Click += this.ScanClusters;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
var selItem = newItem.DropDownItems.Add("Select");
|
var selItem = newItem.DropDownItems.Add("Select");
|
||||||
var delItem = newItem.DropDownItems.Add("Delete");
|
var delItem = newItem.DropDownItems.Add("Delete");
|
||||||
|
|
@ -548,6 +565,31 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
editItem.Click += editItem_Click;
|
editItem.Click += editItem_Click;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Scan the clusters we are subscribed to and add them to the list of known clusters.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="sender">Unused.</param>
|
||||||
|
/// <param name="e">Unused.</param>
|
||||||
|
private void ScanClusters(object sender, EventArgs e)
|
||||||
|
{
|
||||||
|
this.ScanClusters();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Scan the clusters we are subscribed to and add them to the list of known clusters.
|
||||||
|
/// </summary>
|
||||||
|
private void ScanClusters()
|
||||||
|
{
|
||||||
|
this.Status("Scanning for known clusters", StatusKind.LongOp);
|
||||||
|
foreach (var conf in ClusterConfiguration.EnumerateSubscribedClusters())
|
||||||
|
{
|
||||||
|
ClusterConfiguration.AddKnownCluster(conf);
|
||||||
|
this.AddClusterNameToMenu(conf.Name);
|
||||||
|
this.Status("Adding cluster " + conf.Name, StatusKind.OK);
|
||||||
|
}
|
||||||
|
this.Status("Scan completed", StatusKind.OK);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Edit a cluster.
|
/// Edit a cluster.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|
@ -684,7 +726,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
IEnumerable<DryadLinqJobSummary> jobs = todo.Select(j => j.DiscoverDryadLinqJob(this.clusterStatus, this.Status)).Where(j => j != null);
|
IEnumerable<DryadLinqJobSummary> jobs = todo.Select(j => j.DiscoverDryadLinqJob(this.clusterStatus, this.Status)).Where(j => j != null);
|
||||||
|
|
||||||
var item = new BackgroundWorkItem<List<DiagnosisLog>>(
|
var item = new BackgroundWorkItem<List<DiagnosisLog>>(
|
||||||
(s, p) => ClusterWork.DiagnoseJobs(jobs, this.clusterStatus.Config, s, p),
|
m => ClusterWork.DiagnoseJobs(jobs, this.clusterStatus.Config, m),
|
||||||
DiagnosisResult.ShowDiagnosisResult,
|
DiagnosisResult.ShowDiagnosisResult,
|
||||||
"cancel");
|
"cancel");
|
||||||
this.Queue(item);
|
this.Queue(item);
|
||||||
|
|
@ -895,18 +937,18 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="jobs">Jobs to cancel.</param>
|
/// <param name="jobs">Jobs to cancel.</param>
|
||||||
/// <param name="cluster">Cluster where the jobs are running.</param>
|
/// <param name="cluster">Cluster where the jobs are running.</param>
|
||||||
/// <param name="statusReporter">Delegate used to report errors.</param>
|
|
||||||
/// <returns>True if all cancellations succeed.</returns>
|
/// <returns>True if all cancellations succeed.</returns>
|
||||||
/// <param name="updateProgress">Delegate used to report progress.</param>
|
/// <param name="manager">Communicatoni manager.</param>
|
||||||
// ReSharper disable once UnusedParameter.Global
|
// ReSharper disable once UnusedParameter.Global
|
||||||
public static bool CancelJobs(IEnumerable<DryadLinqJobSummary> jobs, ClusterStatus cluster, StatusReporter statusReporter, Action<int> updateProgress)
|
public static bool CancelJobs(IEnumerable<DryadLinqJobSummary> jobs, ClusterStatus cluster, CommManager manager)
|
||||||
{
|
{
|
||||||
bool done = true;
|
bool done = true;
|
||||||
foreach (DryadLinqJobSummary job in jobs)
|
foreach (DryadLinqJobSummary job in jobs)
|
||||||
{
|
{
|
||||||
|
manager.Token.ThrowIfCancellationRequested();
|
||||||
if (job.Status != ClusterJobInformation.ClusterJobStatus.Running)
|
if (job.Status != ClusterJobInformation.ClusterJobStatus.Running)
|
||||||
{
|
{
|
||||||
statusReporter("Job " + job.Name + " does not appear to be running; will still try to cancel", StatusKind.Error);
|
manager.Status("Job " + job.Name + " does not appear to be running; will still try to cancel", StatusKind.Error);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool success;
|
bool success;
|
||||||
|
|
@ -923,9 +965,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
}
|
}
|
||||||
|
|
||||||
if (success)
|
if (success)
|
||||||
statusReporter("Job " + job.Name + " cancelled", StatusKind.OK);
|
manager.Status("Job " + job.Name + " cancelled", StatusKind.OK);
|
||||||
else
|
else
|
||||||
statusReporter("Cancellation of " + job.Name + " failed " + reason, StatusKind.Error);
|
manager.Status("Cancellation of " + job.Name + " failed " + reason, StatusKind.Error);
|
||||||
done &= success;
|
done &= success;
|
||||||
}
|
}
|
||||||
return done;
|
return done;
|
||||||
|
|
@ -936,9 +978,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="jobs">Jobs to diagnose.</param>
|
/// <param name="jobs">Jobs to diagnose.</param>
|
||||||
/// <param name="config">Cluster configuration.</param>
|
/// <param name="config">Cluster configuration.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communicatino manager.</param>
|
||||||
/// <param name="updateProgress">Delegate used to report progress.</param>
|
public static List<DiagnosisLog> DiagnoseJobs(IEnumerable<DryadLinqJobSummary> jobs, ClusterConfiguration config, CommManager manager)
|
||||||
public static List<DiagnosisLog> DiagnoseJobs(IEnumerable<DryadLinqJobSummary> jobs, ClusterConfiguration config, StatusReporter reporter, Action<int> updateProgress)
|
|
||||||
{
|
{
|
||||||
var dryadLinqJobSummaries = jobs as DryadLinqJobSummary[] ?? jobs.ToArray();
|
var dryadLinqJobSummaries = jobs as DryadLinqJobSummary[] ?? jobs.ToArray();
|
||||||
int jobCount = dryadLinqJobSummaries.Count();
|
int jobCount = dryadLinqJobSummaries.Count();
|
||||||
|
|
@ -949,15 +990,16 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
{
|
{
|
||||||
if (summary == null) continue;
|
if (summary == null) continue;
|
||||||
|
|
||||||
JobFailureDiagnosis diagnosis = JobFailureDiagnosis.CreateJobFailureDiagnosis(config, summary, reporter, updateProgress);
|
manager.Token.ThrowIfCancellationRequested();
|
||||||
reporter("Diagnosing " + summary.ShortName(), StatusKind.LongOp);
|
JobFailureDiagnosis diagnosis = JobFailureDiagnosis.CreateJobFailureDiagnosis(config, summary, manager);
|
||||||
|
manager.Status("Diagnosing " + summary.ShortName(), StatusKind.LongOp);
|
||||||
DiagnosisLog log = diagnosis.Diagnose();
|
DiagnosisLog log = diagnosis.Diagnose();
|
||||||
result.Add(log);
|
result.Add(log);
|
||||||
|
|
||||||
done++;
|
done++;
|
||||||
updateProgress(done * 100 / jobCount);
|
manager.Progress(done * 100 / jobCount);
|
||||||
}
|
}
|
||||||
reporter("Diagnosis complete", StatusKind.OK);
|
manager.Status("Diagnosis complete", StatusKind.OK);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,9 +18,9 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
using Microsoft.Research.Calypso.Tools;
|
using Microsoft.Research.Tools;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
partial class ClusterBrowser
|
partial class ClusterBrowser
|
||||||
{
|
{
|
||||||
|
|
@ -56,10 +56,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.openInJobBrowserToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.openInJobBrowserToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.diagnoseToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
|
this.diagnoseToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.terminateToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
|
this.terminateToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
|
this.cancelToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.statusStrip = new System.Windows.Forms.StatusStrip();
|
this.statusStrip = new System.Windows.Forms.StatusStrip();
|
||||||
this.statuslabel = new System.Windows.Forms.ToolStripStatusLabel();
|
this.statuslabel = new System.Windows.Forms.ToolStripStatusLabel();
|
||||||
this.toolStripStatusLabel_backgroundWork = new System.Windows.Forms.ToolStripStatusLabel();
|
|
||||||
this.toolStripStatusLabel_currentWork = new System.Windows.Forms.ToolStripStatusLabel();
|
|
||||||
this.toolStripProgressBar = new System.Windows.Forms.ToolStripProgressBar();
|
this.toolStripProgressBar = new System.Windows.Forms.ToolStripProgressBar();
|
||||||
this.flowLayoutPanel_header = new System.Windows.Forms.FlowLayoutPanel();
|
this.flowLayoutPanel_header = new System.Windows.Forms.FlowLayoutPanel();
|
||||||
this.label_vc = new System.Windows.Forms.Label();
|
this.label_vc = new System.Windows.Forms.Label();
|
||||||
|
|
@ -78,7 +77,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.logFileToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.logFileToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.autoRefreshToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.autoRefreshToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.clusterToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.clusterToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.filteredDataGridView = new Microsoft.Research.Calypso.Tools.FilteredDataGridView();
|
this.filteredDataGridView = new FilteredDataGridView();
|
||||||
this.contextMenuStrip_job.SuspendLayout();
|
this.contextMenuStrip_job.SuspendLayout();
|
||||||
this.statusStrip.SuspendLayout();
|
this.statusStrip.SuspendLayout();
|
||||||
this.flowLayoutPanel_header.SuspendLayout();
|
this.flowLayoutPanel_header.SuspendLayout();
|
||||||
|
|
@ -92,7 +91,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.diagnoseToolStripMenuItem1,
|
this.diagnoseToolStripMenuItem1,
|
||||||
this.terminateToolStripMenuItem1});
|
this.terminateToolStripMenuItem1});
|
||||||
this.contextMenuStrip_job.Name = "contextMenuStrip_job";
|
this.contextMenuStrip_job.Name = "contextMenuStrip_job";
|
||||||
this.contextMenuStrip_job.Size = new System.Drawing.Size(182, 92);
|
this.contextMenuStrip_job.Size = new System.Drawing.Size(182, 70);
|
||||||
//
|
//
|
||||||
// openInJobBrowserToolStripMenuItem
|
// openInJobBrowserToolStripMenuItem
|
||||||
//
|
//
|
||||||
|
|
@ -114,15 +113,19 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.terminateToolStripMenuItem1.Name = "terminateToolStripMenuItem1";
|
this.terminateToolStripMenuItem1.Name = "terminateToolStripMenuItem1";
|
||||||
this.terminateToolStripMenuItem1.Size = new System.Drawing.Size(181, 22);
|
this.terminateToolStripMenuItem1.Size = new System.Drawing.Size(181, 22);
|
||||||
this.terminateToolStripMenuItem1.Text = "Terminate";
|
this.terminateToolStripMenuItem1.Text = "Terminate";
|
||||||
this.terminateToolStripMenuItem1.Visible = false;
|
|
||||||
this.terminateToolStripMenuItem1.Click += new System.EventHandler(this.terminateToolStripMenuItem_Click);
|
this.terminateToolStripMenuItem1.Click += new System.EventHandler(this.terminateToolStripMenuItem_Click);
|
||||||
//
|
//
|
||||||
|
// cancelToolStripMenuItem1
|
||||||
|
//
|
||||||
|
this.cancelToolStripMenuItem1.Name = "cancelToolStripMenuItem1";
|
||||||
|
this.cancelToolStripMenuItem1.Size = new System.Drawing.Size(186, 22);
|
||||||
|
this.cancelToolStripMenuItem1.Text = "Cancel current work";
|
||||||
|
this.cancelToolStripMenuItem1.Click += new System.EventHandler(this.cancelToolStripMenuItem_Click);
|
||||||
|
//
|
||||||
// statusStrip
|
// statusStrip
|
||||||
//
|
//
|
||||||
this.statusStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] {
|
this.statusStrip.Items.AddRange(new System.Windows.Forms.ToolStripItem[] {
|
||||||
this.statuslabel,
|
this.statuslabel,
|
||||||
this.toolStripStatusLabel_backgroundWork,
|
|
||||||
this.toolStripStatusLabel_currentWork,
|
|
||||||
this.toolStripProgressBar});
|
this.toolStripProgressBar});
|
||||||
this.statusStrip.Location = new System.Drawing.Point(0, 431);
|
this.statusStrip.Location = new System.Drawing.Point(0, 431);
|
||||||
this.statusStrip.Name = "statusStrip";
|
this.statusStrip.Name = "statusStrip";
|
||||||
|
|
@ -134,32 +137,16 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
//
|
//
|
||||||
this.statuslabel.Name = "statuslabel";
|
this.statuslabel.Name = "statuslabel";
|
||||||
this.statuslabel.Overflow = System.Windows.Forms.ToolStripItemOverflow.Never;
|
this.statuslabel.Overflow = System.Windows.Forms.ToolStripItemOverflow.Never;
|
||||||
this.statuslabel.Size = new System.Drawing.Size(833, 17);
|
this.statuslabel.Size = new System.Drawing.Size(1026, 17);
|
||||||
this.statuslabel.Spring = true;
|
this.statuslabel.Spring = true;
|
||||||
this.statuslabel.Text = "Status displayed here";
|
this.statuslabel.Text = "Status displayed here";
|
||||||
this.statuslabel.TextAlign = System.Drawing.ContentAlignment.MiddleLeft;
|
this.statuslabel.TextAlign = System.Drawing.ContentAlignment.MiddleLeft;
|
||||||
//
|
//
|
||||||
// toolStripStatusLabel_backgroundWork
|
|
||||||
//
|
|
||||||
this.toolStripStatusLabel_backgroundWork.BorderStyle = System.Windows.Forms.Border3DStyle.SunkenOuter;
|
|
||||||
this.toolStripStatusLabel_backgroundWork.Name = "toolStripStatusLabel_backgroundWork";
|
|
||||||
this.toolStripStatusLabel_backgroundWork.Overflow = System.Windows.Forms.ToolStripItemOverflow.Never;
|
|
||||||
this.toolStripStatusLabel_backgroundWork.Size = new System.Drawing.Size(109, 17);
|
|
||||||
this.toolStripStatusLabel_backgroundWork.Text = "0 pending activities";
|
|
||||||
//
|
|
||||||
// toolStripStatusLabel_currentWork
|
|
||||||
//
|
|
||||||
this.toolStripStatusLabel_currentWork.BorderStyle = System.Windows.Forms.Border3DStyle.SunkenOuter;
|
|
||||||
this.toolStripStatusLabel_currentWork.Name = "toolStripStatusLabel_currentWork";
|
|
||||||
this.toolStripStatusLabel_currentWork.Overflow = System.Windows.Forms.ToolStripItemOverflow.Never;
|
|
||||||
this.toolStripStatusLabel_currentWork.Size = new System.Drawing.Size(84, 17);
|
|
||||||
this.toolStripStatusLabel_currentWork.Text = "Doing nothing";
|
|
||||||
//
|
|
||||||
// toolStripProgressBar
|
// toolStripProgressBar
|
||||||
//
|
//
|
||||||
this.toolStripProgressBar.Name = "toolStripProgressBar";
|
this.toolStripProgressBar.Name = "toolStripProgressBar";
|
||||||
this.toolStripProgressBar.Size = new System.Drawing.Size(100, 16);
|
this.toolStripProgressBar.Size = new System.Drawing.Size(100, 16);
|
||||||
//
|
//
|
||||||
// flowLayoutPanel_header
|
// flowLayoutPanel_header
|
||||||
//
|
//
|
||||||
this.flowLayoutPanel_header.AutoSize = true;
|
this.flowLayoutPanel_header.AutoSize = true;
|
||||||
|
|
@ -219,6 +206,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
// jobToolStripMenuItem_file
|
// jobToolStripMenuItem_file
|
||||||
//
|
//
|
||||||
this.jobToolStripMenuItem_file.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
|
this.jobToolStripMenuItem_file.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
|
||||||
|
this.cancelToolStripMenuItem1,
|
||||||
this.newWindowToolStripMenuItem,
|
this.newWindowToolStripMenuItem,
|
||||||
this.refreshToolStripMenuItem1,
|
this.refreshToolStripMenuItem1,
|
||||||
this.exitToolStripMenuItem1});
|
this.exitToolStripMenuItem1});
|
||||||
|
|
@ -264,7 +252,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.toolStripMenuItem_job.Size = new System.Drawing.Size(37, 20);
|
this.toolStripMenuItem_job.Size = new System.Drawing.Size(37, 20);
|
||||||
this.toolStripMenuItem_job.Text = "&Job";
|
this.toolStripMenuItem_job.Text = "&Job";
|
||||||
this.toolStripMenuItem_job.ToolTipText = "View job informaotion in detail.";
|
this.toolStripMenuItem_job.ToolTipText = "View job informaotion in detail.";
|
||||||
this.toolStripMenuItem_job.Visible = false;
|
|
||||||
//
|
//
|
||||||
// jobBrowserToolStripMenuItem
|
// jobBrowserToolStripMenuItem
|
||||||
//
|
//
|
||||||
|
|
@ -279,6 +266,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.diagnoseToolStripMenuItem.Size = new System.Drawing.Size(163, 22);
|
this.diagnoseToolStripMenuItem.Size = new System.Drawing.Size(163, 22);
|
||||||
this.diagnoseToolStripMenuItem.Text = "Diagnose";
|
this.diagnoseToolStripMenuItem.Text = "Diagnose";
|
||||||
this.diagnoseToolStripMenuItem.ToolTipText = "Attempt to diagnose job failures.";
|
this.diagnoseToolStripMenuItem.ToolTipText = "Attempt to diagnose job failures.";
|
||||||
|
this.diagnoseToolStripMenuItem.Visible = false;
|
||||||
this.diagnoseToolStripMenuItem.Click += new System.EventHandler(this.diagnoseToolStripMenuItem_Click);
|
this.diagnoseToolStripMenuItem.Click += new System.EventHandler(this.diagnoseToolStripMenuItem_Click);
|
||||||
//
|
//
|
||||||
// terminateToolStripMenuItem
|
// terminateToolStripMenuItem
|
||||||
|
|
@ -295,6 +283,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.openFromURLToolStripMenuItem.Size = new System.Drawing.Size(163, 22);
|
this.openFromURLToolStripMenuItem.Size = new System.Drawing.Size(163, 22);
|
||||||
this.openFromURLToolStripMenuItem.Text = "Open job URL...";
|
this.openFromURLToolStripMenuItem.Text = "Open job URL...";
|
||||||
this.openFromURLToolStripMenuItem.ToolTipText = "Open the job given a URL.";
|
this.openFromURLToolStripMenuItem.ToolTipText = "Open the job given a URL.";
|
||||||
|
this.openFromURLToolStripMenuItem.Visible = false;
|
||||||
this.openFromURLToolStripMenuItem.Click += new System.EventHandler(this.openFromURLToolStripMenuItem_Click);
|
this.openFromURLToolStripMenuItem.Click += new System.EventHandler(this.openFromURLToolStripMenuItem_Click);
|
||||||
//
|
//
|
||||||
// settingsToolStripMenuItem
|
// settingsToolStripMenuItem
|
||||||
|
|
@ -309,7 +298,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
// logFileToolStripMenuItem
|
// logFileToolStripMenuItem
|
||||||
//
|
//
|
||||||
this.logFileToolStripMenuItem.Name = "logFileToolStripMenuItem";
|
this.logFileToolStripMenuItem.Name = "logFileToolStripMenuItem";
|
||||||
this.logFileToolStripMenuItem.Size = new System.Drawing.Size(139, 22);
|
this.logFileToolStripMenuItem.Size = new System.Drawing.Size(152, 22);
|
||||||
this.logFileToolStripMenuItem.Text = "Log file";
|
this.logFileToolStripMenuItem.Text = "Log file";
|
||||||
this.logFileToolStripMenuItem.ToolTipText = "When enabled logs errors in the selected file.";
|
this.logFileToolStripMenuItem.ToolTipText = "When enabled logs errors in the selected file.";
|
||||||
this.logFileToolStripMenuItem.Click += new System.EventHandler(this.logFileToolStripMenuItem_Click);
|
this.logFileToolStripMenuItem.Click += new System.EventHandler(this.logFileToolStripMenuItem_Click);
|
||||||
|
|
@ -317,7 +306,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
// autoRefreshToolStripMenuItem
|
// autoRefreshToolStripMenuItem
|
||||||
//
|
//
|
||||||
this.autoRefreshToolStripMenuItem.Name = "autoRefreshToolStripMenuItem";
|
this.autoRefreshToolStripMenuItem.Name = "autoRefreshToolStripMenuItem";
|
||||||
this.autoRefreshToolStripMenuItem.Size = new System.Drawing.Size(139, 22);
|
this.autoRefreshToolStripMenuItem.Size = new System.Drawing.Size(152, 22);
|
||||||
this.autoRefreshToolStripMenuItem.Text = "Auto refresh";
|
this.autoRefreshToolStripMenuItem.Text = "Auto refresh";
|
||||||
this.autoRefreshToolStripMenuItem.Click += new System.EventHandler(this.autoRefreshToolStripMenuItem_Click);
|
this.autoRefreshToolStripMenuItem.Click += new System.EventHandler(this.autoRefreshToolStripMenuItem_Click);
|
||||||
//
|
//
|
||||||
|
|
@ -326,15 +315,15 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.clusterToolStripMenuItem.Name = "clusterToolStripMenuItem";
|
this.clusterToolStripMenuItem.Name = "clusterToolStripMenuItem";
|
||||||
this.clusterToolStripMenuItem.Size = new System.Drawing.Size(56, 20);
|
this.clusterToolStripMenuItem.Size = new System.Drawing.Size(56, 20);
|
||||||
this.clusterToolStripMenuItem.Text = "Cluster";
|
this.clusterToolStripMenuItem.Text = "Cluster";
|
||||||
|
//
|
||||||
// filteredDataGridView
|
// filteredDataGridView
|
||||||
//
|
//
|
||||||
this.filteredDataGridView.BorderStyle = System.Windows.Forms.BorderStyle.FixedSingle;
|
this.filteredDataGridView.BorderStyle = System.Windows.Forms.BorderStyle.FixedSingle;
|
||||||
this.filteredDataGridView.ContextMenuStrip = this.contextMenuStrip_job;
|
this.filteredDataGridView.ContextMenuStrip = this.contextMenuStrip_job;
|
||||||
this.filteredDataGridView.Dock = System.Windows.Forms.DockStyle.Fill;
|
this.filteredDataGridView.Dock = System.Windows.Forms.DockStyle.Fill;
|
||||||
this.filteredDataGridView.Location = new System.Drawing.Point(0, 56);
|
this.filteredDataGridView.Location = new System.Drawing.Point(0, 24);
|
||||||
this.filteredDataGridView.Name = "filteredDataGridView";
|
this.filteredDataGridView.Name = "filteredDataGridView";
|
||||||
this.filteredDataGridView.Size = new System.Drawing.Size(1143, 375);
|
this.filteredDataGridView.Size = new System.Drawing.Size(1143, 429);
|
||||||
this.filteredDataGridView.TabIndex = 15;
|
this.filteredDataGridView.TabIndex = 15;
|
||||||
this.filteredDataGridView.CellFormatting += new System.Windows.Forms.DataGridViewCellFormattingEventHandler(this.filteredDataGridView_CellFormatting);
|
this.filteredDataGridView.CellFormatting += new System.Windows.Forms.DataGridViewCellFormattingEventHandler(this.filteredDataGridView_CellFormatting);
|
||||||
this.filteredDataGridView.CellMouseDoubleClick += new System.Windows.Forms.DataGridViewCellMouseEventHandler(this.filteredDataGridView_CellMouseDoubleClick);
|
this.filteredDataGridView.CellMouseDoubleClick += new System.Windows.Forms.DataGridViewCellMouseEventHandler(this.filteredDataGridView_CellMouseDoubleClick);
|
||||||
|
|
@ -365,6 +354,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void cancelToolStripMenuItem_Click(object sender, System.EventArgs e)
|
||||||
|
{
|
||||||
|
this.queue.CancelCurrentWork();
|
||||||
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
private System.Windows.Forms.StatusStrip statusStrip;
|
private System.Windows.Forms.StatusStrip statusStrip;
|
||||||
|
|
@ -378,13 +372,12 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
private System.Windows.Forms.ToolStripMenuItem refreshToolStripMenuItem1;
|
private System.Windows.Forms.ToolStripMenuItem refreshToolStripMenuItem1;
|
||||||
private System.Windows.Forms.ToolStripMenuItem openFromURLToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem openFromURLToolStripMenuItem;
|
||||||
private System.Windows.Forms.ToolStripMenuItem terminateToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem terminateToolStripMenuItem;
|
||||||
private System.Windows.Forms.ToolStripStatusLabel toolStripStatusLabel_backgroundWork;
|
|
||||||
private System.Windows.Forms.ToolStripStatusLabel toolStripStatusLabel_currentWork;
|
|
||||||
private System.Windows.Forms.ToolStripMenuItem diagnoseToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem diagnoseToolStripMenuItem;
|
||||||
private System.Windows.Forms.ContextMenuStrip contextMenuStrip_job;
|
private System.Windows.Forms.ContextMenuStrip contextMenuStrip_job;
|
||||||
private System.Windows.Forms.ToolStripMenuItem openInJobBrowserToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem openInJobBrowserToolStripMenuItem;
|
||||||
private System.Windows.Forms.ToolStripMenuItem diagnoseToolStripMenuItem1;
|
private System.Windows.Forms.ToolStripMenuItem diagnoseToolStripMenuItem1;
|
||||||
private System.Windows.Forms.ToolStripMenuItem terminateToolStripMenuItem1;
|
private System.Windows.Forms.ToolStripMenuItem terminateToolStripMenuItem1;
|
||||||
|
private System.Windows.Forms.ToolStripMenuItem cancelToolStripMenuItem1;
|
||||||
private System.Windows.Forms.ToolStripProgressBar toolStripProgressBar;
|
private System.Windows.Forms.ToolStripProgressBar toolStripProgressBar;
|
||||||
private FilteredDataGridView filteredDataGridView;
|
private FilteredDataGridView filteredDataGridView;
|
||||||
private System.Windows.Forms.ToolStripMenuItem newWindowToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem newWindowToolStripMenuItem;
|
||||||
|
|
|
||||||
|
|
@ -123,9 +123,6 @@
|
||||||
<metadata name="statusStrip.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
<metadata name="statusStrip.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||||
<value>17, 17</value>
|
<value>17, 17</value>
|
||||||
</metadata>
|
</metadata>
|
||||||
<metadata name="backgroundWorker.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
|
||||||
<value>253, 17</value>
|
|
||||||
</metadata>
|
|
||||||
<metadata name="menuStrip.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
<metadata name="menuStrip.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||||
<value>419, 16</value>
|
<value>419, 16</value>
|
||||||
</metadata>
|
</metadata>
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
namespace Microsoft.Research.JobObjectModel
|
||||||
{
|
{
|
||||||
partial class ClusterConfigEditor
|
partial class ClusterConfigEditor
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -23,9 +23,9 @@ using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Drawing;
|
using System.Drawing;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
using Microsoft.Research.Calypso.Tools;
|
using Microsoft.Research.Tools;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
namespace Microsoft.Research.JobObjectModel
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Editor for cluster configuration.
|
/// Editor for cluster configuration.
|
||||||
|
|
|
||||||
|
|
@ -19,15 +19,15 @@ limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
using Microsoft.Research.Calypso.JobObjectModel;
|
|
||||||
using Microsoft.Research.Calypso.Tools;
|
|
||||||
using System;
|
using System;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
|
using Microsoft.Research.JobObjectModel;
|
||||||
|
using Microsoft.Research.Tools;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The result of a decision (ternary booleans?)
|
/// The result of a decision (ternary booleans?)
|
||||||
|
|
@ -236,30 +236,24 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public DryadLinqJobInfo Job { get; protected set; }
|
public DryadLinqJobInfo Job { get; protected set; }
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Delegate used to report erorrs.
|
/// Communication manager.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public StatusReporter Reporter { get; protected set; }
|
public CommManager Manager { get; protected set; }
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Plan of the job.
|
/// Plan of the job.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public DryadJobStaticPlan StaticPlan { get; protected set; }
|
public DryadJobStaticPlan StaticPlan { get; protected set; }
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Delegate used to report progress.
|
|
||||||
/// </summary>
|
|
||||||
public Action<int> ProgressReporter { get; protected set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Create a FailureDiagnosis object.
|
/// Create a FailureDiagnosis object.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="job">Job being diagnosed.</param>
|
/// <param name="job">Job being diagnosed.</param>
|
||||||
/// <param name="plan">Static plan of the job.</param>
|
/// <param name="plan">Static plan of the job.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="progressReporter">Delegate used to report progress.</param>
|
protected FailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, CommManager manager)
|
||||||
protected FailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, StatusReporter reporter, Action<int> progressReporter)
|
|
||||||
{
|
{
|
||||||
this.Job = job;
|
this.Job = job;
|
||||||
this.StaticPlan = plan;
|
this.StaticPlan = plan;
|
||||||
this.Reporter = reporter;
|
this.Manager = manager;
|
||||||
this.ProgressReporter = progressReporter;
|
|
||||||
this.Summary = job.Summary;
|
this.Summary = job.Summary;
|
||||||
this.cluster = job.ClusterConfiguration;
|
this.cluster = job.ClusterConfiguration;
|
||||||
}
|
}
|
||||||
|
|
@ -267,19 +261,18 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Try to find the job information from cluster and summary.
|
/// Try to find the job information from cluster and summary.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="status">Delegate used to report status.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="progress">Delegate used to report progress.</param>
|
protected void FindJobInfo(CommManager manager)
|
||||||
protected void FindJobInfo(StatusReporter status, Action<int> progress)
|
|
||||||
{
|
{
|
||||||
DryadLinqJobInfo jobinfo = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.cluster, this.Summary, true, status, progress);
|
DryadLinqJobInfo jobinfo = DryadLinqJobInfo.CreateDryadLinqJobInfo(this.cluster, this.Summary, true, manager);
|
||||||
if (jobinfo == null)
|
if (jobinfo == null)
|
||||||
{
|
{
|
||||||
status("Cannot collect information for " + Summary.ShortName() + " to diagnose", StatusKind.Error);
|
manager.Status("Cannot collect information for " + Summary.ShortName() + " to diagnose", StatusKind.Error);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.Job = jobinfo;
|
this.Job = jobinfo;
|
||||||
this.StaticPlan = JobObjectModel.DryadJobStaticPlan.CreatePlan(jobinfo, status);
|
this.StaticPlan = JobObjectModel.DryadJobStaticPlan.CreatePlan(jobinfo, manager);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -287,15 +280,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="config">Cluster where job resides.</param>
|
/// <param name="config">Cluster where job resides.</param>
|
||||||
/// <param name="summary">Job summary.</param>
|
/// <param name="summary">Job summary.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="progressReporter">Delegate used to report progress.</param>
|
protected FailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, CommManager manager)
|
||||||
protected FailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, StatusReporter reporter, Action<int> progressReporter)
|
|
||||||
{
|
{
|
||||||
this.cluster = config;
|
this.cluster = config;
|
||||||
this.Summary = summary;
|
this.Summary = summary;
|
||||||
this.Reporter = reporter;
|
this.Manager = manager;
|
||||||
this.ProgressReporter = progressReporter;
|
this.FindJobInfo(manager);
|
||||||
this.FindJobInfo(reporter, progressReporter);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -327,11 +318,10 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="vertex">Vertex to diagnose.</param>
|
/// <param name="vertex">Vertex to diagnose.</param>
|
||||||
/// <param name="job">Job containing the vertex.</param>
|
/// <param name="job">Job containing the vertex.</param>
|
||||||
/// <param name="progressReporter">Delegate used to report progress.</param>
|
|
||||||
/// <param name="reporter">Delegate used to report status.</param>
|
|
||||||
/// <param name="plan">Plan of the executed job.</param>
|
/// <param name="plan">Plan of the executed job.</param>
|
||||||
protected VertexFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, ExecutedVertexInstance vertex, StatusReporter reporter, Action<int> progressReporter)
|
/// <param name="manager">Communication manager.</param>
|
||||||
: base(job, plan, reporter, progressReporter)
|
protected VertexFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, ExecutedVertexInstance vertex, CommManager manager)
|
||||||
|
: base(job, plan, manager)
|
||||||
{
|
{
|
||||||
this.Job = job;
|
this.Job = job;
|
||||||
this.Vertex = vertex;
|
this.Vertex = vertex;
|
||||||
|
|
@ -344,15 +334,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="vertex">Vertex to diagnose.</param>
|
/// <param name="vertex">Vertex to diagnose.</param>
|
||||||
/// <param name="job">Job containing the vertex.</param>
|
/// <param name="job">Job containing the vertex.</param>
|
||||||
/// <param name="progressReporter">Delegate used to report progress.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="reporter">Delegate used to report status.</param>
|
|
||||||
/// <returns>A subclass of VertexFailureDiagnosis.</returns>
|
/// <returns>A subclass of VertexFailureDiagnosis.</returns>
|
||||||
/// <param name="plan">Plan of the executed job.</param>
|
/// <param name="plan">Plan of the executed job.</param>
|
||||||
public static VertexFailureDiagnosis CreateVertexFailureDiagnosis(DryadLinqJobInfo job,
|
public static VertexFailureDiagnosis CreateVertexFailureDiagnosis(DryadLinqJobInfo job,
|
||||||
DryadJobStaticPlan plan,
|
DryadJobStaticPlan plan,
|
||||||
ExecutedVertexInstance vertex,
|
ExecutedVertexInstance vertex,
|
||||||
StatusReporter reporter,
|
CommManager manager)
|
||||||
Action<int> progressReporter)
|
|
||||||
{
|
{
|
||||||
ClusterConfiguration config = job.ClusterConfiguration;
|
ClusterConfiguration config = job.ClusterConfiguration;
|
||||||
if (config is CacheClusterConfiguration)
|
if (config is CacheClusterConfiguration)
|
||||||
|
|
@ -381,7 +369,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
DiagnosisLog log = new DiagnosisLog(this.Job, this.Summary);
|
DiagnosisLog log = new DiagnosisLog(this.Job, this.Summary);
|
||||||
log.AddMessage(new DiagnosisMessage(DiagnosisMessage.Importance.Final, "Diagnostic for " + this.VertexName, "Vertex state is " + this.Vertex.State));
|
log.AddMessage(new DiagnosisMessage(DiagnosisMessage.Importance.Final, "Diagnostic for " + this.VertexName, "Vertex state is " + this.Vertex.State));
|
||||||
this.Diagnose(log);
|
this.Diagnose(log);
|
||||||
this.Reporter("Vertex diagnosis complete", StatusKind.OK);
|
this.Manager.Status("Vertex diagnosis complete", StatusKind.OK);
|
||||||
return log;
|
return log;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -409,7 +397,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// Detect whether the vertex had problems reading a particular channel.
|
/// Detect whether the vertex had problems reading a particular channel.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The channel that cannot be read, or null if that's not the problem.</returns>
|
/// <returns>The channel that cannot be read, or null if that's not the problem.</returns>
|
||||||
public virtual ChannelEndpointDescription ChannelReadFailure()
|
/// <param name="manager">Communication manager.</param>
|
||||||
|
public virtual ChannelEndpointDescription ChannelReadFailure(CommManager manager)
|
||||||
{
|
{
|
||||||
List<string> stack = this.StackTrace().ToList();
|
List<string> stack = this.StackTrace().ToList();
|
||||||
if (stack.Count == 0)
|
if (stack.Count == 0)
|
||||||
|
|
@ -424,7 +413,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
bool success = int.TryParse(m.Groups[3].Value, out channelNo);
|
bool success = int.TryParse(m.Groups[3].Value, out channelNo);
|
||||||
if (!success)
|
if (!success)
|
||||||
return null;
|
return null;
|
||||||
this.Vertex.DiscoverChannels(true, false, true, this.Reporter, null);
|
this.Vertex.DiscoverChannels(true, false, true, manager);
|
||||||
var channels = this.Vertex.InputChannels;
|
var channels = this.Vertex.InputChannels;
|
||||||
if (channels == null)
|
if (channels == null)
|
||||||
return null;
|
return null;
|
||||||
|
|
@ -604,11 +593,10 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// Create a class to diagnose the problems of a job.
|
/// Create a class to diagnose the problems of a job.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="job">Job to diagnose.</param>
|
/// <param name="job">Job to diagnose.</param>
|
||||||
/// <param name="progress">Delegate used to report progress.</param>
|
|
||||||
/// <param name="status">Delegate used to report status.</param>
|
|
||||||
/// <param name="plan">Plan of the diagnosed job.</param>
|
/// <param name="plan">Plan of the diagnosed job.</param>
|
||||||
protected JobFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, StatusReporter status, Action<int> progress)
|
/// <param name="manager">Communication manager.</param>
|
||||||
: base(job, plan, status, progress)
|
protected JobFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, CommManager manager)
|
||||||
|
: base(job, plan, manager)
|
||||||
{
|
{
|
||||||
this.diagnosisLog = new DiagnosisLog(job, job.Summary);
|
this.diagnosisLog = new DiagnosisLog(job, job.Summary);
|
||||||
this.jobManager = this.Job.ManagerVertex;
|
this.jobManager = this.Job.ManagerVertex;
|
||||||
|
|
@ -617,12 +605,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Create a class to diagnose the problems of a job.
|
/// Create a class to diagnose the problems of a job.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="progress">Delegate used to report progress.</param>
|
|
||||||
/// <param name="status">Delegate used to report status.</param>
|
|
||||||
/// <param name="config">Cluster where job resides.</param>
|
/// <param name="config">Cluster where job resides.</param>
|
||||||
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="summary">Job summary.</param>
|
/// <param name="summary">Job summary.</param>
|
||||||
protected JobFailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, StatusReporter status, Action<int> progress)
|
protected JobFailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, CommManager manager)
|
||||||
: base(config, summary, status, progress)
|
: base(config, summary, manager)
|
||||||
{
|
{
|
||||||
this.diagnosisLog = new DiagnosisLog(this.Job, summary);
|
this.diagnosisLog = new DiagnosisLog(this.Job, summary);
|
||||||
if (this.Job != null)
|
if (this.Job != null)
|
||||||
|
|
@ -825,11 +812,10 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// Create a suitable Job Failure diagnosis object for the job being analyzed.
|
/// Create a suitable Job Failure diagnosis object for the job being analyzed.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="job">Job to diagnose.</param>
|
/// <param name="job">Job to diagnose.</param>
|
||||||
/// <param name="status">Delegate used to report errors.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <param name="progress">Delegate used to report progress.</param>
|
|
||||||
/// <returns>A subclass of JobFailureDiagnosis with the type appropriate for the job.</returns>
|
/// <returns>A subclass of JobFailureDiagnosis with the type appropriate for the job.</returns>
|
||||||
/// <param name="plan">Plan of the job being diagnosed.</param>
|
/// <param name="plan">Plan of the job being diagnosed.</param>
|
||||||
public static JobFailureDiagnosis CreateJobFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, StatusReporter status, Action<int> progress)
|
public static JobFailureDiagnosis CreateJobFailureDiagnosis(DryadLinqJobInfo job, DryadJobStaticPlan plan, CommManager manager)
|
||||||
{
|
{
|
||||||
ClusterConfiguration config = job.ClusterConfiguration;
|
ClusterConfiguration config = job.ClusterConfiguration;
|
||||||
if (config is CacheClusterConfiguration)
|
if (config is CacheClusterConfiguration)
|
||||||
|
|
@ -843,11 +829,10 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// Create a suitable Job Failure diagnosis object for the job being analyzed.
|
/// Create a suitable Job Failure diagnosis object for the job being analyzed.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="summary">Job to diagnose.</param>
|
/// <param name="summary">Job to diagnose.</param>
|
||||||
/// <param name="status">Delegate used to report errors.</param>
|
|
||||||
/// <param name="progress">Delegate used to report progress.</param>
|
|
||||||
/// <param name="config">Cluster where job resides.</param>
|
/// <param name="config">Cluster where job resides.</param>
|
||||||
|
/// <param name="manager">Communication manager.</param>
|
||||||
/// <returns>A subclass of JobFailureDiagnosis with the type appropriate for the job.</returns>
|
/// <returns>A subclass of JobFailureDiagnosis with the type appropriate for the job.</returns>
|
||||||
public static JobFailureDiagnosis CreateJobFailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, StatusReporter status, Action<int> progress)
|
public static JobFailureDiagnosis CreateJobFailureDiagnosis(ClusterConfiguration config, DryadLinqJobSummary summary, CommManager manager)
|
||||||
{
|
{
|
||||||
if (config is CacheClusterConfiguration)
|
if (config is CacheClusterConfiguration)
|
||||||
config = (config as CacheClusterConfiguration).ActualConfig(summary);
|
config = (config as CacheClusterConfiguration).ActualConfig(summary);
|
||||||
|
|
@ -860,7 +845,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// This is incomplete: e.g., it does not work for tidyfs streams.
|
/// This is incomplete: e.g., it does not work for tidyfs streams.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>Yes if there were correlated failures.</returns>
|
/// <returns>Yes if there were correlated failures.</returns>
|
||||||
protected Decision LookForCorrelatedReadFailures()
|
/// <param name="manager">Communication manager.</param>
|
||||||
|
protected Decision LookForCorrelatedReadFailures(CommManager manager)
|
||||||
{
|
{
|
||||||
// if we have more than this many failures we start to worry
|
// if we have more than this many failures we start to worry
|
||||||
const int maxFailures = 5;
|
const int maxFailures = 5;
|
||||||
|
|
@ -876,13 +862,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
int verticesDone = 0;
|
int verticesDone = 0;
|
||||||
foreach (ExecutedVertexInstance v in failures)
|
foreach (ExecutedVertexInstance v in failures)
|
||||||
{
|
{
|
||||||
var crf = VertexFailureDiagnosis.CreateVertexFailureDiagnosis(this.Job, this.StaticPlan, v, null, null).ChannelReadFailure();
|
var crf = VertexFailureDiagnosis.CreateVertexFailureDiagnosis(this.Job, this.StaticPlan, v, manager).ChannelReadFailure(manager);
|
||||||
if (crf != null)
|
if (crf != null)
|
||||||
{
|
{
|
||||||
channelsFailed.Add(crf);
|
channelsFailed.Add(crf);
|
||||||
}
|
}
|
||||||
verticesDone++;
|
verticesDone++;
|
||||||
this.ProgressReporter(verticesDone * 100 / totalFailures);
|
manager.Progress(verticesDone * 100 / totalFailures);
|
||||||
}
|
}
|
||||||
if (channelsFailed.Count() < maxFailures)
|
if (channelsFailed.Count() < maxFailures)
|
||||||
return Decision.No;
|
return Decision.No;
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
using System;
|
using System;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -20,11 +20,11 @@ limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using Microsoft.Research.Calypso.JobObjectModel;
|
|
||||||
using System;
|
using System;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
|
using Microsoft.Research.JobObjectModel;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Display the diagnosis results.
|
/// Display the diagnosis results.
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@ using System;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
static class Program
|
static class Program
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
partial class JobBrowser
|
partial class JobBrowser
|
||||||
{
|
{
|
||||||
|
|
@ -65,13 +65,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.menu = new System.Windows.Forms.MenuStrip();
|
this.menu = new System.Windows.Forms.MenuStrip();
|
||||||
this.jobToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.jobToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.refreshToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.refreshToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.collectDataToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
|
||||||
this.hideCancelledVerticesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.hideCancelledVerticesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.exportToCSVToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.exportToCSVToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.toolStripMenuItem_terminate = new System.Windows.Forms.ToolStripMenuItem();
|
this.toolStripMenuItem_terminate = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.packageCachedFilesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.packageCachedFilesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.diagnoseToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.diagnoseToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.closeToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.closeToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
|
this.cancelCurrentWorkToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.stageToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.stageToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.cacheLogsForAllVerticesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.cacheLogsForAllVerticesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.vertexToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.vertexToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
|
|
@ -80,7 +80,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.profileLocallyCPUSamplingToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.profileLocallyCPUSamplingToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.profileLocallyMemorySamplingToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.profileLocallyMemorySamplingToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.jMStdoutMentionsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.jMStdoutMentionsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.jMLogsMentionsToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
|
|
||||||
this.diagnoseToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
|
this.diagnoseToolStripMenuItem1 = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.viewToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.viewToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.loadFileInEditorToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.loadFileInEditorToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
|
|
@ -124,7 +123,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.menuItem_stageVertexProfileLocallyCPUSampling = new System.Windows.Forms.ToolStripMenuItem();
|
this.menuItem_stageVertexProfileLocallyCPUSampling = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.menuItem_stageVertexProfileLocallyMemorySampling = new System.Windows.Forms.ToolStripMenuItem();
|
this.menuItem_stageVertexProfileLocallyMemorySampling = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.jMStdoutLinesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
this.jMStdoutLinesToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.jMLogsMentionsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();
|
|
||||||
this.diagnoseToolStripMenuItem2 = new System.Windows.Forms.ToolStripMenuItem();
|
this.diagnoseToolStripMenuItem2 = new System.Windows.Forms.ToolStripMenuItem();
|
||||||
this.splitContainer_vertexData = new System.Windows.Forms.SplitContainer();
|
this.splitContainer_vertexData = new System.Windows.Forms.SplitContainer();
|
||||||
this.dataGridView_vertexHeader = new System.Windows.Forms.DataGridView();
|
this.dataGridView_vertexHeader = new System.Windows.Forms.DataGridView();
|
||||||
|
|
@ -147,7 +145,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.toolStripStatusLabel_currentWork = new System.Windows.Forms.ToolStripStatusLabel();
|
this.toolStripStatusLabel_currentWork = new System.Windows.Forms.ToolStripStatusLabel();
|
||||||
this.toolStripStatusLabel_backgroundWork = new System.Windows.Forms.ToolStripStatusLabel();
|
this.toolStripStatusLabel_backgroundWork = new System.Windows.Forms.ToolStripStatusLabel();
|
||||||
this.toolStripProgressBar = new System.Windows.Forms.ToolStripProgressBar();
|
this.toolStripProgressBar = new System.Windows.Forms.ToolStripProgressBar();
|
||||||
this.backgroundWorker = new System.ComponentModel.BackgroundWorker();
|
|
||||||
this.menu.SuspendLayout();
|
this.menu.SuspendLayout();
|
||||||
((System.ComponentModel.ISupportInitialize)(this.splitContainer_jobAndRest)).BeginInit();
|
((System.ComponentModel.ISupportInitialize)(this.splitContainer_jobAndRest)).BeginInit();
|
||||||
this.splitContainer_jobAndRest.Panel1.SuspendLayout();
|
this.splitContainer_jobAndRest.Panel1.SuspendLayout();
|
||||||
|
|
@ -204,13 +201,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
//
|
//
|
||||||
this.jobToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
|
this.jobToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
|
||||||
this.refreshToolStripMenuItem,
|
this.refreshToolStripMenuItem,
|
||||||
this.collectDataToolStripMenuItem,
|
|
||||||
this.hideCancelledVerticesToolStripMenuItem,
|
this.hideCancelledVerticesToolStripMenuItem,
|
||||||
this.exportToCSVToolStripMenuItem,
|
this.exportToCSVToolStripMenuItem,
|
||||||
this.toolStripMenuItem_terminate,
|
this.toolStripMenuItem_terminate,
|
||||||
this.packageCachedFilesToolStripMenuItem,
|
this.packageCachedFilesToolStripMenuItem,
|
||||||
this.diagnoseToolStripMenuItem,
|
this.diagnoseToolStripMenuItem,
|
||||||
this.closeToolStripMenuItem});
|
this.closeToolStripMenuItem,
|
||||||
|
this.cancelCurrentWorkToolStripMenuItem});
|
||||||
this.jobToolStripMenuItem.Name = "jobToolStripMenuItem";
|
this.jobToolStripMenuItem.Name = "jobToolStripMenuItem";
|
||||||
this.jobToolStripMenuItem.Size = new System.Drawing.Size(37, 20);
|
this.jobToolStripMenuItem.Size = new System.Drawing.Size(37, 20);
|
||||||
this.jobToolStripMenuItem.Text = "Job";
|
this.jobToolStripMenuItem.Text = "Job";
|
||||||
|
|
@ -248,7 +245,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.toolStripMenuItem_terminate.Size = new System.Drawing.Size(195, 22);
|
this.toolStripMenuItem_terminate.Size = new System.Drawing.Size(195, 22);
|
||||||
this.toolStripMenuItem_terminate.Text = "Terminate job";
|
this.toolStripMenuItem_terminate.Text = "Terminate job";
|
||||||
this.toolStripMenuItem_terminate.ToolTipText = "Requests the cluster to terminate the job execution.";
|
this.toolStripMenuItem_terminate.ToolTipText = "Requests the cluster to terminate the job execution.";
|
||||||
this.toolStripMenuItem_terminate.Visible = false;
|
|
||||||
this.toolStripMenuItem_terminate.Click += new System.EventHandler(this.toolStripMenuItem_terminate_Click);
|
this.toolStripMenuItem_terminate.Click += new System.EventHandler(this.toolStripMenuItem_terminate_Click);
|
||||||
//
|
//
|
||||||
// packageCachedFilesToolStripMenuItem
|
// packageCachedFilesToolStripMenuItem
|
||||||
|
|
@ -279,6 +275,13 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.closeToolStripMenuItem.ToolTipText = "Save the settings and close the window.";
|
this.closeToolStripMenuItem.ToolTipText = "Save the settings and close the window.";
|
||||||
this.closeToolStripMenuItem.Click += new System.EventHandler(this.closeToolStripMenuItem_Click);
|
this.closeToolStripMenuItem.Click += new System.EventHandler(this.closeToolStripMenuItem_Click);
|
||||||
//
|
//
|
||||||
|
// cancelCurrentWorkToolStripMenuItem
|
||||||
|
//
|
||||||
|
this.cancelCurrentWorkToolStripMenuItem.Name = "cancelCurrentWorkToolStripMenuItem";
|
||||||
|
this.cancelCurrentWorkToolStripMenuItem.Size = new System.Drawing.Size(195, 22);
|
||||||
|
this.cancelCurrentWorkToolStripMenuItem.Text = "Cancel current work";
|
||||||
|
this.cancelCurrentWorkToolStripMenuItem.Click += new System.EventHandler(this.cancelCurrentWorkToolStripMenuItem_Click);
|
||||||
|
//
|
||||||
// stageToolStripMenuItem
|
// stageToolStripMenuItem
|
||||||
//
|
//
|
||||||
this.stageToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
|
this.stageToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] {
|
||||||
|
|
@ -304,7 +307,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.profileLocallyCPUSamplingToolStripMenuItem,
|
this.profileLocallyCPUSamplingToolStripMenuItem,
|
||||||
this.profileLocallyMemorySamplingToolStripMenuItem,
|
this.profileLocallyMemorySamplingToolStripMenuItem,
|
||||||
this.jMStdoutMentionsToolStripMenuItem,
|
this.jMStdoutMentionsToolStripMenuItem,
|
||||||
this.jMLogsMentionsToolStripMenuItem1,
|
|
||||||
this.diagnoseToolStripMenuItem1});
|
this.diagnoseToolStripMenuItem1});
|
||||||
this.vertexToolStripMenuItem.Enabled = false;
|
this.vertexToolStripMenuItem.Enabled = false;
|
||||||
this.vertexToolStripMenuItem.Name = "vertexToolStripMenuItem";
|
this.vertexToolStripMenuItem.Name = "vertexToolStripMenuItem";
|
||||||
|
|
@ -908,10 +910,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.menuItem_stageVertexProfileLocallyCPUSampling,
|
this.menuItem_stageVertexProfileLocallyCPUSampling,
|
||||||
this.menuItem_stageVertexProfileLocallyMemorySampling,
|
this.menuItem_stageVertexProfileLocallyMemorySampling,
|
||||||
this.jMStdoutLinesToolStripMenuItem,
|
this.jMStdoutLinesToolStripMenuItem,
|
||||||
this.jMLogsMentionsToolStripMenuItem,
|
|
||||||
this.diagnoseToolStripMenuItem2});
|
this.diagnoseToolStripMenuItem2});
|
||||||
this.contextMenu_stageVertex.Name = "vertexContextMenuStrip";
|
this.contextMenu_stageVertex.Name = "vertexContextMenuStrip";
|
||||||
this.contextMenu_stageVertex.Size = new System.Drawing.Size(256, 158);
|
this.contextMenu_stageVertex.Size = new System.Drawing.Size(256, 136);
|
||||||
//
|
//
|
||||||
// menuItem_stageVertexLocalDebugManaged
|
// menuItem_stageVertexLocalDebugManaged
|
||||||
//
|
//
|
||||||
|
|
@ -1252,11 +1253,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.toolStripProgressBar.Name = "toolStripProgressBar";
|
this.toolStripProgressBar.Name = "toolStripProgressBar";
|
||||||
this.toolStripProgressBar.Size = new System.Drawing.Size(100, 16);
|
this.toolStripProgressBar.Size = new System.Drawing.Size(100, 16);
|
||||||
//
|
//
|
||||||
// backgroundWorker
|
|
||||||
//
|
|
||||||
this.backgroundWorker.DoWork += new System.ComponentModel.DoWorkEventHandler(this.backgroundWorker_DoWork);
|
|
||||||
this.backgroundWorker.RunWorkerCompleted += new System.ComponentModel.RunWorkerCompletedEventHandler(this.backgroundWorker_RunWorkerCompleted);
|
|
||||||
//
|
|
||||||
// JobBrowser
|
// JobBrowser
|
||||||
//
|
//
|
||||||
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
|
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
|
||||||
|
|
@ -1352,7 +1348,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
private System.Windows.Forms.TextBox textBox_stageCode;
|
private System.Windows.Forms.TextBox textBox_stageCode;
|
||||||
private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel3;
|
private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel3;
|
||||||
private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel4;
|
private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel4;
|
||||||
private System.Windows.Forms.ToolStripMenuItem collectDataToolStripMenuItem;
|
|
||||||
private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel5;
|
private System.Windows.Forms.FlowLayoutPanel flowLayoutPanel5;
|
||||||
private System.Windows.Forms.Label label1;
|
private System.Windows.Forms.Label label1;
|
||||||
private System.Windows.Forms.TextBox textBox_find;
|
private System.Windows.Forms.TextBox textBox_find;
|
||||||
|
|
@ -1362,7 +1357,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
private System.Windows.Forms.Label label_matches;
|
private System.Windows.Forms.Label label_matches;
|
||||||
private System.Windows.Forms.CheckBox checkBox_refresh;
|
private System.Windows.Forms.CheckBox checkBox_refresh;
|
||||||
private System.Windows.Forms.Button button_filter;
|
private System.Windows.Forms.Button button_filter;
|
||||||
private System.ComponentModel.BackgroundWorker backgroundWorker;
|
|
||||||
private System.Windows.Forms.ComboBox comboBox_vertexInformation;
|
private System.Windows.Forms.ComboBox comboBox_vertexInformation;
|
||||||
private System.Windows.Forms.Label label_comboVertex;
|
private System.Windows.Forms.Label label_comboVertex;
|
||||||
private System.Windows.Forms.ToolStripMenuItem viewToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem viewToolStripMenuItem;
|
||||||
|
|
@ -1380,9 +1374,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
private System.Windows.Forms.ToolStripMenuItem menuItem_stageVertexLocalDebugUnmanaged;
|
private System.Windows.Forms.ToolStripMenuItem menuItem_stageVertexLocalDebugUnmanaged;
|
||||||
private System.Windows.Forms.ToolStripMenuItem debugLocallyUnmanagedToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem debugLocallyUnmanagedToolStripMenuItem;
|
||||||
private System.Windows.Forms.ToolStripMenuItem jMStdoutMentionsToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem jMStdoutMentionsToolStripMenuItem;
|
||||||
private System.Windows.Forms.ToolStripMenuItem jMLogsMentionsToolStripMenuItem1;
|
|
||||||
private System.Windows.Forms.ToolStripMenuItem jMStdoutLinesToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem jMStdoutLinesToolStripMenuItem;
|
||||||
private System.Windows.Forms.ToolStripMenuItem jMLogsMentionsToolStripMenuItem;
|
|
||||||
private System.Windows.Forms.TextBox textBox_stageFilter;
|
private System.Windows.Forms.TextBox textBox_stageFilter;
|
||||||
private System.Windows.Forms.Button button_stageFilter;
|
private System.Windows.Forms.Button button_stageFilter;
|
||||||
private System.Windows.Forms.Button button_clearStageFilter;
|
private System.Windows.Forms.Button button_clearStageFilter;
|
||||||
|
|
@ -1412,5 +1404,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
private System.Windows.Forms.ToolStripMenuItem cacheAllLogsToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem cacheAllLogsToolStripMenuItem;
|
||||||
private System.Windows.Forms.ToolStripMenuItem exportToCSVToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem exportToCSVToolStripMenuItem;
|
||||||
private System.Windows.Forms.ToolStripMenuItem cacheLogsForAllVerticesToolStripMenuItem;
|
private System.Windows.Forms.ToolStripMenuItem cacheLogsForAllVerticesToolStripMenuItem;
|
||||||
|
private System.Windows.Forms.ToolStripMenuItem cancelCurrentWorkToolStripMenuItem;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -34,10 +34,10 @@ using System.Threading;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
using Microsoft.Msagl.GraphViewerGdi;
|
using Microsoft.Msagl.GraphViewerGdi;
|
||||||
using Microsoft.Msagl.Splines;
|
using Microsoft.Msagl.Splines;
|
||||||
using Microsoft.Research.Calypso.JobObjectModel;
|
using Microsoft.Research.JobObjectModel;
|
||||||
using Microsoft.Research.Calypso.Tools;
|
using Microsoft.Research.Tools;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A form to display information about a DryadLinq job.
|
/// A form to display information about a DryadLinq job.
|
||||||
|
|
@ -66,40 +66,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private BackgroundWorkQueue queue;
|
private BackgroundWorkQueue queue;
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Activity to perform by the backgroundWorker.
|
|
||||||
/// </summary>
|
|
||||||
class BackgroundWorkInfo
|
|
||||||
{
|
|
||||||
public enum WorkKind
|
|
||||||
{
|
|
||||||
LoadJobInfo,
|
|
||||||
};
|
|
||||||
|
|
||||||
public WorkKind Work; // kind of operation to perform in the background
|
|
||||||
public bool Success; // if true the work succeeded
|
|
||||||
public DateTime workStartTime; // when work is started
|
|
||||||
|
|
||||||
public override string ToString()
|
|
||||||
{
|
|
||||||
return this.Work.ToString();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Add yourself to the list of pending work.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="list">List of pending work.</param>
|
|
||||||
internal void AddTo(List<BackgroundWorkInfo> list)
|
|
||||||
{
|
|
||||||
list.Add(this);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// List of work activities to perform.
|
|
||||||
/// </summary>
|
|
||||||
readonly List<BackgroundWorkInfo> pendingWork;
|
|
||||||
|
|
||||||
// window regions starting from left-top in order going down
|
// window regions starting from left-top in order going down
|
||||||
#region JOB_HEADER
|
#region JOB_HEADER
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -352,10 +319,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.InitializeComponent();
|
this.InitializeComponent();
|
||||||
|
|
||||||
this.queueWorker = new BackgroundWorker();
|
this.queueWorker = new BackgroundWorker();
|
||||||
this.queue = new BackgroundWorkQueue(this.queueWorker);
|
this.queue = new BackgroundWorkQueue(this.queueWorker, this.toolStripStatusLabel_currentWork, this.toolStripStatusLabel_backgroundWork);
|
||||||
|
|
||||||
this.WarnedAboutDebugging = false;
|
this.WarnedAboutDebugging = false;
|
||||||
this.pendingWork = new List<BackgroundWorkInfo>();
|
|
||||||
this.status = new StatusWriter(this.toolStripStatusLabel, this.statusStrip, this.Status);
|
this.status = new StatusWriter(this.toolStripStatusLabel, this.statusStrip, this.Status);
|
||||||
|
|
||||||
this.refreshTimer = new System.Windows.Forms.Timer();
|
this.refreshTimer = new System.Windows.Forms.Timer();
|
||||||
|
|
@ -558,16 +524,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.RefreshDisplay();
|
this.RefreshDisplay();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Load information about the current job.
|
|
||||||
/// </summary>
|
|
||||||
private void LoadJobDetails()
|
|
||||||
{
|
|
||||||
BackgroundWorkInfo work = new BackgroundWorkInfo();
|
|
||||||
work.Work = BackgroundWorkInfo.WorkKind.LoadJobInfo;
|
|
||||||
this.StartBackgroundWork(work);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Loading the job information has completed.
|
/// Loading the job information has completed.
|
||||||
/// <param name="timeToLoad">Time it took to load the job information.</param>
|
/// <param name="timeToLoad">Time it took to load the job information.</param>
|
||||||
|
|
@ -857,7 +813,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
case ExecutedVertexInstance.VertexState.Failed:
|
case ExecutedVertexInstance.VertexState.Failed:
|
||||||
return Color.Tomato;
|
return Color.Tomato;
|
||||||
default:
|
default:
|
||||||
throw new CalypsoDryadException("Unexpected vertex state " + state);
|
throw new DryadException("Unexpected vertex state " + state);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -986,22 +942,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
yield return new Tuple<double, Color>(unknown, VertexStateColor(ExecutedVertexInstance.VertexState.Unknown));
|
yield return new Tuple<double, Color>(unknown, VertexStateColor(ExecutedVertexInstance.VertexState.Unknown));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static DryadJobStaticPlan CreatePlan(DryadLinqJobInfo job, StatusReporter status)
|
|
||||||
{
|
|
||||||
status("Constructing static plan", StatusKind.LongOp);
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var result = JobObjectModel.DryadJobStaticPlan.CreatePlan(job, status);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
status("Exception during building of static plan: " + ex.Message, StatusKind.Error);
|
|
||||||
Trace.TraceInformation(ex.ToString());
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Refresh and redisplay the query plan.
|
/// Refresh and redisplay the query plan.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|
@ -1010,7 +950,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.richTextBox_file.Text = "";
|
this.richTextBox_file.Text = "";
|
||||||
|
|
||||||
var item = new BackgroundWorkItem<DryadJobStaticPlan>(
|
var item = new BackgroundWorkItem<DryadJobStaticPlan>(
|
||||||
(s, p) => JobObjectModel.DryadJobStaticPlan.CreatePlan(this.Job, this.Status),
|
m => JobObjectModel.DryadJobStaticPlan.CreatePlan(this.Job, m),
|
||||||
this.PlanComputed,
|
this.PlanComputed,
|
||||||
"refresh plan");
|
"refresh plan");
|
||||||
this.Queue(item);
|
this.Queue(item);
|
||||||
|
|
@ -1455,7 +1395,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.EnableStageFiltering(true);
|
this.EnableStageFiltering(true);
|
||||||
this.stageHeaderData.RaiseListChangedEvents = false;
|
this.stageHeaderData.RaiseListChangedEvents = false;
|
||||||
this.currentStage = stage;
|
this.currentStage = stage;
|
||||||
this.currentTable = null;
|
this.currentTable = null;
|
||||||
|
|
||||||
// stageData is populated by the selectionChanged event handler for the stageHeader
|
// stageData is populated by the selectionChanged event handler for the stageHeader
|
||||||
if (this.ShowingStageOrTable != KindOfStageShown.Stage)
|
if (this.ShowingStageOrTable != KindOfStageShown.Stage)
|
||||||
|
|
@ -1677,10 +1617,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="path">Cluster object whose contents is read.</param>
|
/// <param name="path">Cluster object whose contents is read.</param>
|
||||||
/// <param name="pattern">Pattern to filter contents, for folders.</param>
|
/// <param name="pattern">Pattern to filter contents, for folders.</param>
|
||||||
/// <param name="status">Used to report status.</param>
|
|
||||||
/// <returns>The file contents.</returns>
|
/// <returns>The file contents.</returns>
|
||||||
/// <param name="progress">Progress reporter.</param>
|
/// <param name="manager">Communication manager.</param>
|
||||||
private static FileContents GetContents(StatusReporter status, Action<int> progress, IClusterResidentObject path, string pattern)
|
private static FileContents GetContents(CommManager manager, IClusterResidentObject path, string pattern)
|
||||||
{
|
{
|
||||||
if (path == null)
|
if (path == null)
|
||||||
{
|
{
|
||||||
|
|
@ -1705,6 +1644,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
int displayed = 0;
|
int displayed = 0;
|
||||||
foreach (IClusterResidentObject d in dirs)
|
foreach (IClusterResidentObject d in dirs)
|
||||||
{
|
{
|
||||||
|
manager.Token.ThrowIfCancellationRequested();
|
||||||
if (d.Exception != null)
|
if (d.Exception != null)
|
||||||
{
|
{
|
||||||
error += " [Error " + d.Exception.Message + "]";
|
error += " [Error " + d.Exception.Message + "]";
|
||||||
|
|
@ -1730,7 +1670,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
status("Extracting contents of " + path, StatusKind.LongOp);
|
manager.Status("Extracting contents of " + path, StatusKind.LongOp);
|
||||||
ISharedStreamReader sr = path.GetStream();
|
ISharedStreamReader sr = path.GetStream();
|
||||||
if (sr.Exception != null)
|
if (sr.Exception != null)
|
||||||
{
|
{
|
||||||
|
|
@ -1741,7 +1681,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
{
|
{
|
||||||
if (path.Size == 0)
|
if (path.Size == 0)
|
||||||
error += "[empty]";
|
error += "[empty]";
|
||||||
var contents = sr.ReadToEnd();
|
var contents = sr.ReadToEnd(manager.Token);
|
||||||
return new FileContents(contents, error, linkCache);
|
return new FileContents(contents, error, linkCache);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1756,7 +1696,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
private void DisplayContents1(IClusterResidentObject path, string pattern)
|
private void DisplayContents1(IClusterResidentObject path, string pattern)
|
||||||
{
|
{
|
||||||
var item = new BackgroundWorkItem<FileContents>(
|
var item = new BackgroundWorkItem<FileContents>(
|
||||||
(s, p) => GetContents(s, p, path, pattern),
|
m => GetContents(m, path, pattern),
|
||||||
this.ShowContents,
|
this.ShowContents,
|
||||||
"Read file");
|
"Read file");
|
||||||
this.Queue(item);
|
this.Queue(item);
|
||||||
|
|
@ -1892,7 +1832,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
{
|
{
|
||||||
this.label_title.Text = "Inputs";
|
this.label_title.Text = "Inputs";
|
||||||
this.Status("Discovering vertex channel information", StatusKind.LongOp);
|
this.Status("Discovering vertex channel information", StatusKind.LongOp);
|
||||||
bool found = this.currentVertex.DiscoverChannels(true, false, false, this.Status, this.UpdateProgress);
|
// TODO: this should run in the background
|
||||||
|
CommManager manager = new CommManager(this.Status, this.UpdateProgress, new CancellationTokenSource().Token);
|
||||||
|
bool found = this.currentVertex.DiscoverChannels(true, false, false, manager);
|
||||||
if (found)
|
if (found)
|
||||||
{
|
{
|
||||||
this.richTextBox_file.SuspendLayout();
|
this.richTextBox_file.SuspendLayout();
|
||||||
|
|
@ -1922,7 +1864,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
{
|
{
|
||||||
this.label_title.Text = "Outputs";
|
this.label_title.Text = "Outputs";
|
||||||
this.Status("Discovering vertex channel information", StatusKind.LongOp);
|
this.Status("Discovering vertex channel information", StatusKind.LongOp);
|
||||||
bool found = this.currentVertex.DiscoverChannels(false, true, false, this.Status, this.UpdateProgress);
|
// TODO: this should run in the background
|
||||||
|
CommManager manager = new CommManager(this.Status, this.UpdateProgress, new CancellationTokenSource().Token);
|
||||||
|
bool found = this.currentVertex.DiscoverChannels(false, true, false, manager);
|
||||||
if (found)
|
if (found)
|
||||||
{
|
{
|
||||||
this.richTextBox_file.SuspendLayout();
|
this.richTextBox_file.SuspendLayout();
|
||||||
|
|
@ -2319,7 +2263,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
this.Status("Refreshing...", StatusKind.LongOp);
|
this.Status("Refreshing...", StatusKind.LongOp);
|
||||||
this.Job.InvalidateCaches();
|
this.Job.InvalidateCaches();
|
||||||
this.stageColorMap = null; // force recomputation
|
this.stageColorMap = null; // force recomputation
|
||||||
this.LoadJobDetails();
|
this.RefreshJob();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -2340,8 +2284,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
private void JobBrowser_FormClosing(object sender, FormClosingEventArgs e)
|
private void JobBrowser_FormClosing(object sender, FormClosingEventArgs e)
|
||||||
{
|
{
|
||||||
this.refreshTimer.Stop();
|
this.refreshTimer.Stop();
|
||||||
this.pendingWork.Clear();
|
this.queue.Stop();
|
||||||
|
|
||||||
this.formSettings.WarnedAboutDebugging = this.WarnedAboutDebugging;
|
this.formSettings.WarnedAboutDebugging = this.WarnedAboutDebugging;
|
||||||
this.formSettings.WarnedAboutProfiling = this.WarnedAboutProfiling;
|
this.formSettings.WarnedAboutProfiling = this.WarnedAboutProfiling;
|
||||||
this.formSettings.Location = this.Location;
|
this.formSettings.Location = this.Location;
|
||||||
|
|
@ -2496,107 +2439,37 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// <param name="e">Unused.</param>
|
/// <param name="e">Unused.</param>
|
||||||
private void JobBrowser_Shown(object sender, EventArgs e)
|
private void JobBrowser_Shown(object sender, EventArgs e)
|
||||||
{
|
{
|
||||||
this.LoadJobDetails();
|
this.RefreshJob();
|
||||||
}
|
|
||||||
|
|
||||||
#region BACKGROUND_WORK
|
|
||||||
/// <summary>
|
|
||||||
/// Start performing a piece of background work.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="work">Work to perform.</param>
|
|
||||||
private void StartBackgroundWork(BackgroundWorkInfo work)
|
|
||||||
{
|
|
||||||
if (this.backgroundWorker.IsBusy)
|
|
||||||
{
|
|
||||||
work.AddTo(this.pendingWork);
|
|
||||||
this.toolStripStatusLabel_backgroundWork.Text = this.pendingWork.Count() + " tasks pending.";
|
|
||||||
this.Status("Queued task for execution", StatusKind.OK);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
backgroundWorker.RunWorkerAsync(work);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Show the work currently being done.
|
/// Refresh the job details.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="msg">Message to display (work description).</param>
|
private void RefreshJob()
|
||||||
private void ShowCurrentWork(string msg)
|
|
||||||
{
|
{
|
||||||
if (this.InvokeRequired)
|
DryadLinqJobInfo job = this.Job;
|
||||||
this.Invoke(new Action<string>(this.ShowCurrentWork), msg);
|
DateTime start = DateTime.Now;
|
||||||
else
|
var item = new BackgroundWorkItem<TimeSpan>(
|
||||||
this.toolStripStatusLabel_currentWork.Text = msg;
|
m =>
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Perform some background work.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="sender">Unused.</param>
|
|
||||||
/// <param name="e">Event describing the work to perform.</param>
|
|
||||||
private void backgroundWorker_DoWork(object sender, DoWorkEventArgs e)
|
|
||||||
{
|
|
||||||
BackgroundWorkInfo work = (BackgroundWorkInfo)e.Argument;
|
|
||||||
work.workStartTime = DateTime.Now;
|
|
||||||
this.ShowCurrentWork("Doing " + work + ".");
|
|
||||||
|
|
||||||
switch (work.Work)
|
|
||||||
{
|
|
||||||
case BackgroundWorkInfo.WorkKind.LoadJobInfo:
|
|
||||||
{
|
|
||||||
work.Success = this.Job.CollectEssentialInformation(this.Status, this.UpdateProgress);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
e.Result = work;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Background work has terminated.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="sender">Unused.</param>
|
|
||||||
/// <param name="e">Event describing the result.</param>
|
|
||||||
private void backgroundWorker_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
|
|
||||||
{
|
|
||||||
this.ShowCurrentWork("Doing nothing.");
|
|
||||||
if (e.Cancelled)
|
|
||||||
{
|
|
||||||
this.Status("Background work was cancelled", StatusKind.OK);
|
|
||||||
goto end;
|
|
||||||
}
|
|
||||||
else if (e.Error != null)
|
|
||||||
{
|
|
||||||
this.Status("Exception during background work: " + e.Error.Message, StatusKind.Error);
|
|
||||||
Trace.TraceInformation(e.ToString());
|
|
||||||
goto end;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (e.Result == null)
|
|
||||||
{
|
|
||||||
Trace.TraceInformation("Null result from background work!");
|
|
||||||
// I don't know why this happens
|
|
||||||
goto end;
|
|
||||||
}
|
|
||||||
BackgroundWorkInfo result = (BackgroundWorkInfo)e.Result;
|
|
||||||
if (result.Success)
|
|
||||||
{
|
|
||||||
switch (result.Work)
|
|
||||||
{
|
{
|
||||||
case BackgroundWorkInfo.WorkKind.LoadJobInfo:
|
job.CollectEssentialInformation(m);
|
||||||
this.LoadJobCompleted(DateTime.Now - result.workStartTime);
|
return DateTime.Now - start;
|
||||||
break;
|
},
|
||||||
}
|
this.JobInfoLoaded,
|
||||||
this.Status("Completed " + result, StatusKind.OK);
|
"refreshJob");
|
||||||
}
|
this.Queue(item);
|
||||||
else
|
}
|
||||||
{
|
|
||||||
// failed in background work
|
|
||||||
goto end;
|
|
||||||
}
|
|
||||||
|
|
||||||
// do not overwrite the error message if the job did not succeed
|
/// <summary>
|
||||||
|
/// Called after a job has been loaded.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="cancelled">If true the loading has been cancelled.</param>
|
||||||
|
/// <param name="loadTime">Time to load job.</param>
|
||||||
|
private void JobInfoLoaded(bool cancelled, TimeSpan loadTime)
|
||||||
|
{
|
||||||
|
if (cancelled) return;
|
||||||
|
|
||||||
// refresh the stage view too
|
this.LoadJobCompleted(loadTime);
|
||||||
string s = this.currentStage != null ? this.currentStage.Name : null;
|
string s = this.currentStage != null ? this.currentStage.Name : null;
|
||||||
if (this.doingStartup && string.IsNullOrEmpty(s))
|
if (this.doingStartup && string.IsNullOrEmpty(s))
|
||||||
{
|
{
|
||||||
|
|
@ -2613,17 +2486,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
{
|
{
|
||||||
this.SetTable(this.currentTable.Refresh(this.Job, this.Status, !this.hideCancelledVerticesToolStripMenuItem.Checked));
|
this.SetTable(this.currentTable.Refresh(this.Job, this.Status, !this.hideCancelledVerticesToolStripMenuItem.Checked));
|
||||||
}
|
}
|
||||||
|
|
||||||
end:
|
|
||||||
if (this.pendingWork.Any())
|
|
||||||
{
|
|
||||||
BackgroundWorkInfo work = this.pendingWork[0];
|
|
||||||
this.pendingWork.RemoveAt(0);
|
|
||||||
this.toolStripStatusLabel_backgroundWork.Text = this.pendingWork.Count() + " tasks pending.";
|
|
||||||
this.StartBackgroundWork(work);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region MOUSE_DYNAMIC_VIEWS
|
#region MOUSE_DYNAMIC_VIEWS
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -2879,7 +2742,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
lv.Show();
|
lv.Show();
|
||||||
|
|
||||||
var item = new BackgroundWorkItem<bool>(
|
var item = new BackgroundWorkItem<bool>(
|
||||||
(s, p) => ScanJMStdout(this.currentVertex, this.Job.ManagerVertex.StdoutFile, lv),
|
m => ScanJMStdout(this.currentVertex, this.Job.ManagerVertex.StdoutFile, lv),
|
||||||
(c, b) => { },
|
(c, b) => { },
|
||||||
"findStdout");
|
"findStdout");
|
||||||
this.Queue(item);
|
this.Queue(item);
|
||||||
|
|
@ -2936,7 +2799,8 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// <param name="e">Unused.</param>
|
/// <param name="e">Unused.</param>
|
||||||
private void diagnoseToolStripMenuItem_Click(object sender, EventArgs e)
|
private void diagnoseToolStripMenuItem_Click(object sender, EventArgs e)
|
||||||
{
|
{
|
||||||
JobFailureDiagnosis diagnosis = JobFailureDiagnosis.CreateJobFailureDiagnosis(this.Job, this.staticPlan, this.Status, this.UpdateProgress);
|
CommManager manager = new CommManager(this.Status, this.UpdateProgress, new CancellationToken());
|
||||||
|
JobFailureDiagnosis diagnosis = JobFailureDiagnosis.CreateJobFailureDiagnosis(this.Job, this.staticPlan, manager);
|
||||||
DiagnosisLog log = diagnosis.Diagnose();
|
DiagnosisLog log = diagnosis.Diagnose();
|
||||||
this.DisplayDiagnosis(log);
|
this.DisplayDiagnosis(log);
|
||||||
}
|
}
|
||||||
|
|
@ -3328,7 +3192,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
ClusterStatus clusterStatus = this.Job.ClusterConfiguration.CreateClusterStatus();
|
ClusterStatus clusterStatus = this.Job.ClusterConfiguration.CreateClusterStatus();
|
||||||
|
|
||||||
var item = new BackgroundWorkItem<bool>(
|
var item = new BackgroundWorkItem<bool>(
|
||||||
(s, p) => ClusterWork.CancelJobs(job, clusterStatus, s, p),
|
m => ClusterWork.CancelJobs(job, clusterStatus, m),
|
||||||
(c, b) => { },
|
(c, b) => { },
|
||||||
"cancel");
|
"cancel");
|
||||||
this.Queue(item);
|
this.Queue(item);
|
||||||
|
|
@ -3344,7 +3208,9 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
if (this.currentVertex == null)
|
if (this.currentVertex == null)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
VertexFailureDiagnosis vfd = VertexFailureDiagnosis.CreateVertexFailureDiagnosis(this.Job, this.staticPlan, this.currentVertex, this.Status, this.UpdateProgress);
|
// TODO: this should run in the background
|
||||||
|
CommManager manager = new CommManager(this.Status, this.UpdateProgress, new CancellationToken());
|
||||||
|
VertexFailureDiagnosis vfd = VertexFailureDiagnosis.CreateVertexFailureDiagnosis(this.Job, this.staticPlan, this.currentVertex, manager);
|
||||||
DiagnosisLog log = vfd.Diagnose();
|
DiagnosisLog log = vfd.Diagnose();
|
||||||
this.DisplayDiagnosis(log);
|
this.DisplayDiagnosis(log);
|
||||||
}
|
}
|
||||||
|
|
@ -3387,16 +3253,6 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// <param name="e">Unused.</param>
|
/// <param name="e">Unused.</param>
|
||||||
private void dataGridView_Scroll(object sender, ScrollEventArgs e)
|
private void dataGridView_Scroll(object sender, ScrollEventArgs e)
|
||||||
{
|
{
|
||||||
if (e.Type == ScrollEventType.EndScroll)
|
|
||||||
{
|
|
||||||
VScrollBar scrollbar = sender as VScrollBar;
|
|
||||||
if (scrollbar == null)
|
|
||||||
return;
|
|
||||||
DataGridView view = scrollbar.Parent as DataGridView;
|
|
||||||
if (view == null)
|
|
||||||
return;
|
|
||||||
view.AutoResizeColumns(DataGridViewAutoSizeColumnsMode.DisplayedCells);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -3427,9 +3283,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// <param name="e">Unused.</param>
|
/// <param name="e">Unused.</param>
|
||||||
private void cacheAllLogsToolStripMenuItem_Click(object sender, EventArgs e)
|
private void cacheAllLogsToolStripMenuItem_Click(object sender, EventArgs e)
|
||||||
{
|
{
|
||||||
BackgroundWorkInfo work = new BackgroundWorkInfo();
|
this.RefreshJob();
|
||||||
work.Work = BackgroundWorkInfo.WorkKind.LoadJobInfo;
|
|
||||||
this.StartBackgroundWork(work);
|
|
||||||
|
|
||||||
IClusterResidentObject folder = this.richtextBoxShownFile;
|
IClusterResidentObject folder = this.richtextBoxShownFile;
|
||||||
if (folder == null || folder.Exception != null || !folder.RepresentsAFolder)
|
if (folder == null || folder.Exception != null || !folder.RepresentsAFolder)
|
||||||
|
|
@ -3518,7 +3372,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
List<ExecutedVertexInstance> vertices = this.stageData.ToList();
|
List<ExecutedVertexInstance> vertices = this.stageData.ToList();
|
||||||
|
|
||||||
var item = new BackgroundWorkItem<bool>(
|
var item = new BackgroundWorkItem<bool>(
|
||||||
(s, p) => CacheAllVertices(this.Job.ClusterConfiguration, this.Job.Summary, vertices, s, p),
|
m => CacheAllVertices(this.Job.ClusterConfiguration, this.Job.Summary, vertices, m),
|
||||||
(c, b) => { },
|
(c, b) => { },
|
||||||
"cacheAll");
|
"cacheAll");
|
||||||
this.Queue(item);
|
this.Queue(item);
|
||||||
|
|
@ -3528,23 +3382,27 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// Cache the vertices in the list; executed on the background thread.
|
/// Cache the vertices in the list; executed on the background thread.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>True: success.</returns>
|
/// <returns>True: success.</returns>
|
||||||
|
/// <param name="manager">Communication manager.</param>
|
||||||
|
/// <param name="config">Cluster configuration.</param>
|
||||||
|
/// <param name="summary">Job to cache.</param>
|
||||||
|
/// <param name="vertices">Vertices to cache.</param>
|
||||||
private static bool CacheAllVertices(
|
private static bool CacheAllVertices(
|
||||||
ClusterConfiguration config, DryadLinqJobSummary summary, List<ExecutedVertexInstance> vertices,
|
ClusterConfiguration config, DryadLinqJobSummary summary, List<ExecutedVertexInstance> vertices,
|
||||||
StatusReporter status, Action<int> progress)
|
CommManager manager)
|
||||||
{
|
{
|
||||||
int done = 0;
|
int done = 0;
|
||||||
int todo = vertices.Count;
|
int todo = vertices.Count;
|
||||||
int files = 0;
|
int files = 0;
|
||||||
status("Caching data for " + todo + " vertices", StatusKind.LongOp);
|
manager.Status("Caching data for " + todo + " vertices", StatusKind.LongOp);
|
||||||
foreach (ExecutedVertexInstance v in vertices)
|
foreach (ExecutedVertexInstance v in vertices)
|
||||||
{
|
{
|
||||||
files += CacheVertexInfo(config, summary, v);
|
files += CacheVertexInfo(config, summary, v);
|
||||||
done++;
|
done++;
|
||||||
progress(done / todo);
|
manager.Progress(done / todo);
|
||||||
}
|
}
|
||||||
|
|
||||||
progress(100);
|
manager.Progress(100);
|
||||||
status("Cached " + files + " files", StatusKind.OK);
|
manager.Status("Cached " + files + " files", StatusKind.OK);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3576,6 +3434,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
}
|
}
|
||||||
|
|
||||||
ISharedStreamReader reader = file.GetStream();
|
ISharedStreamReader reader = file.GetStream();
|
||||||
|
// ReSharper disable once UnusedVariable
|
||||||
foreach (string line in reader.ReadAllLines())
|
foreach (string line in reader.ReadAllLines())
|
||||||
{
|
{
|
||||||
// discard; causes caching
|
// discard; causes caching
|
||||||
|
|
@ -3584,6 +3443,11 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
}
|
}
|
||||||
return cached;
|
return cached;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void cancelCurrentWorkToolStripMenuItem_Click(object sender, EventArgs e)
|
||||||
|
{
|
||||||
|
this.queue.CancelCurrentWork();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||||
|
|
@ -8,8 +9,8 @@
|
||||||
<ProjectGuid>{EDDD2E0B-A52B-4E25-9436-B874017673FF}</ProjectGuid>
|
<ProjectGuid>{EDDD2E0B-A52B-4E25-9436-B874017673FF}</ProjectGuid>
|
||||||
<OutputType>Exe</OutputType>
|
<OutputType>Exe</OutputType>
|
||||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||||
<RootNamespace>DryadAnalysis</RootNamespace>
|
<RootNamespace>JobBrowser</RootNamespace>
|
||||||
<AssemblyName>JobBrowser</AssemblyName>
|
<AssemblyName>DryadLinqBrowser</AssemblyName>
|
||||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||||
<FileAlignment>512</FileAlignment>
|
<FileAlignment>512</FileAlignment>
|
||||||
<StartupObject>
|
<StartupObject>
|
||||||
|
|
@ -22,6 +23,7 @@
|
||||||
<UpgradeBackupLocation>
|
<UpgradeBackupLocation>
|
||||||
</UpgradeBackupLocation>
|
</UpgradeBackupLocation>
|
||||||
<OldToolsVersion>3.5</OldToolsVersion>
|
<OldToolsVersion>3.5</OldToolsVersion>
|
||||||
|
<TargetFrameworkProfile />
|
||||||
<PublishUrl>publish\</PublishUrl>
|
<PublishUrl>publish\</PublishUrl>
|
||||||
<Install>true</Install>
|
<Install>true</Install>
|
||||||
<InstallFrom>Disk</InstallFrom>
|
<InstallFrom>Disk</InstallFrom>
|
||||||
|
|
@ -36,29 +38,6 @@
|
||||||
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
|
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
|
||||||
<UseApplicationTrust>false</UseApplicationTrust>
|
<UseApplicationTrust>false</UseApplicationTrust>
|
||||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||||
<TargetFrameworkProfile />
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
|
||||||
<DebugSymbols>true</DebugSymbols>
|
|
||||||
<DebugType>full</DebugType>
|
|
||||||
<Optimize>false</Optimize>
|
|
||||||
<OutputPath>bin\Debug\</OutputPath>
|
|
||||||
<DefineConstants>TRACE;DEBUG</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
<DocumentationFile>bin\Debug\JobBrowser.xml</DocumentationFile>
|
|
||||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
|
||||||
<Prefer32Bit>false</Prefer32Bit>
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
|
||||||
<DebugType>pdbonly</DebugType>
|
|
||||||
<Optimize>true</Optimize>
|
|
||||||
<OutputPath>bin\Release\</OutputPath>
|
|
||||||
<DefineConstants>TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
|
||||||
<Prefer32Bit>false</Prefer32Bit>
|
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||||
<DebugSymbols>true</DebugSymbols>
|
<DebugSymbols>true</DebugSymbols>
|
||||||
|
|
@ -79,7 +58,48 @@
|
||||||
<ErrorReport>prompt</ErrorReport>
|
<ErrorReport>prompt</ErrorReport>
|
||||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
<PropertyGroup>
|
||||||
|
<RunPostBuildEvent>OnBuildSuccess</RunPostBuildEvent>
|
||||||
|
</PropertyGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
<Reference Include="EnvDTE, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="EnvDTE80, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE80.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="EnvDTE90, Version=9.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE90.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Hpc.Scheduler, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\Daphne\Dependences\Microsoft.Hpc.Scheduler.dll</HintPath>
|
||||||
|
<Private>True</Private>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Hpc.Scheduler.Properties, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\..\Daphne\Dependences\Microsoft.Hpc.Scheduler.Properties.dll</HintPath>
|
||||||
|
<Private>True</Private>
|
||||||
|
</Reference>
|
||||||
<Reference Include="Microsoft.Msagl, Version=3.0.0.0, Culture=neutral, PublicKeyToken=7f84f738e1fc245c, processorArchitecture=MSIL">
|
<Reference Include="Microsoft.Msagl, Version=3.0.0.0, Culture=neutral, PublicKeyToken=7f84f738e1fc245c, processorArchitecture=MSIL">
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
|
||||||
|
|
@ -92,13 +112,71 @@
|
||||||
<SpecificVersion>False</SpecificVersion>
|
<SpecificVersion>False</SpecificVersion>
|
||||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
|
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
|
||||||
</Reference>
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.2.0, Culture=neutral, processorArchitecture=AMD64">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks">
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions">
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
|
||||||
|
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
|
||||||
|
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||||
|
<SpecificVersion>False</SpecificVersion>
|
||||||
|
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
<Reference Include="System" />
|
<Reference Include="System" />
|
||||||
<Reference Include="System.Core">
|
<Reference Include="System.Core">
|
||||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||||
</Reference>
|
</Reference>
|
||||||
<Reference Include="System.Data.Linq" />
|
<Reference Include="System.Data.Linq" />
|
||||||
|
<Reference Include="System.Net" />
|
||||||
|
<Reference Include="System.Net.Http" />
|
||||||
|
<Reference Include="System.Net.Http.Extensions">
|
||||||
|
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.Primitives">
|
||||||
|
<HintPath>..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
<Reference Include="System.Net.Http.WebRequest" />
|
||||||
<Reference Include="System.Runtime.Serialization" />
|
<Reference Include="System.Runtime.Serialization" />
|
||||||
<Reference Include="System.ServiceModel" />
|
<Reference Include="System.ServiceModel" />
|
||||||
|
<Reference Include="System.Web" />
|
||||||
<Reference Include="System.Windows.Forms" />
|
<Reference Include="System.Windows.Forms" />
|
||||||
<Reference Include="System.Xml.Linq">
|
<Reference Include="System.Xml.Linq">
|
||||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||||
|
|
@ -186,11 +264,11 @@
|
||||||
<Name>JobObjectModel</Name>
|
<Name>JobObjectModel</Name>
|
||||||
</ProjectReference>
|
</ProjectReference>
|
||||||
<ProjectReference Include="..\Tools\Tools.csproj">
|
<ProjectReference Include="..\Tools\Tools.csproj">
|
||||||
<Project>{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}</Project>
|
<Project>{20b91aaf-aad4-47df-9f1d-494de6e066f9}</Project>
|
||||||
<Name>Tools</Name>
|
<Name>Tools</Name>
|
||||||
</ProjectReference>
|
</ProjectReference>
|
||||||
<ProjectReference Include="..\UsefulForms\UsefulForms.csproj">
|
<ProjectReference Include="..\UsefulForms\UsefulForms.csproj">
|
||||||
<Project>{27635A68-ADFC-4192-9262-B4E6ECDDCE09}</Project>
|
<Project>{27635a68-adfc-4192-9262-b4e6ecddce09}</Project>
|
||||||
<Name>UsefulForms</Name>
|
<Name>UsefulForms</Name>
|
||||||
</ProjectReference>
|
</ProjectReference>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
@ -234,6 +312,19 @@
|
||||||
</BootstrapperPackage>
|
</BootstrapperPackage>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||||
|
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
|
||||||
|
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
|
||||||
|
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
|
||||||
|
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
|
||||||
|
</Target>
|
||||||
|
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||||
|
<PropertyGroup>
|
||||||
|
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||||
|
</PropertyGroup>
|
||||||
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||||
|
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||||
|
</Target>
|
||||||
|
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||||
Other similar extension points exist, see Microsoft.Common.targets.
|
Other similar extension points exist, see Microsoft.Common.targets.
|
||||||
<Target Name="BeforeBuild">
|
<Target Name="BeforeBuild">
|
||||||
|
|
@ -241,4 +332,4 @@
|
||||||
<Target Name="AfterBuild">
|
<Target Name="AfterBuild">
|
||||||
</Target>
|
</Target>
|
||||||
-->
|
-->
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -126,9 +126,6 @@
|
||||||
<metadata name="statusStrip.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
<metadata name="statusStrip.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||||
<value>17, 17</value>
|
<value>17, 17</value>
|
||||||
</metadata>
|
</metadata>
|
||||||
<metadata name="backgroundWorker.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
|
||||||
<value>209, 17</value>
|
|
||||||
</metadata>
|
|
||||||
<metadata name="$this.TrayHeight" type="System.Int32, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
|
<metadata name="$this.TrayHeight" type="System.Int32, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
|
||||||
<value>42</value>
|
<value>42</value>
|
||||||
</metadata>
|
</metadata>
|
||||||
|
|
|
||||||
|
|
@ -24,13 +24,13 @@ using System.Collections.Generic;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
using Microsoft.Research.Calypso.JobObjectModel;
|
using Microsoft.Research.JobObjectModel;
|
||||||
using Microsoft.Research.Calypso.Tools;
|
using Microsoft.Research.Tools;
|
||||||
using Microsoft.Research.Calypso.UsefulForms;
|
using Microsoft.Research.UsefulForms;
|
||||||
using Microsoft.Win32;
|
using Microsoft.Win32;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Class that encapsulates the steps required to debug a vertex locally.
|
/// Class that encapsulates the steps required to debug a vertex locally.
|
||||||
|
|
|
||||||
|
|
@ -18,9 +18,9 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
using Microsoft.Research.Calypso.Tools;
|
using Microsoft.Research.Tools;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
partial class LogViewer
|
partial class LogViewer
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -19,15 +19,15 @@ limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
using Microsoft.Research.Calypso.JobObjectModel;
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
using Microsoft.Research.Calypso.Tools;
|
using Microsoft.Research.JobObjectModel;
|
||||||
|
using Microsoft.Research.Tools;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.DryadAnalysis
|
namespace Microsoft.Research.DryadAnalysis
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A log viewer displays fragments of logs or other text files.
|
/// A log viewer displays fragments of logs or other text files.
|
||||||
|
|
@ -37,7 +37,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
bool canceled;
|
bool canceled;
|
||||||
|
|
||||||
DGVData<TextFileLine> shownText;
|
DGVData<TextFileLine> shownText;
|
||||||
DGVData<PositionedCosmosLogEntry> shownLogLines;
|
DGVData<PositionedDryadLogEntry> shownLogLines;
|
||||||
StatusWriter status;
|
StatusWriter status;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -77,7 +77,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
this.shownLogLines = new DGVData<PositionedCosmosLogEntry>();
|
this.shownLogLines = new DGVData<PositionedDryadLogEntry>();
|
||||||
this.filteredDataGridView.SetDataSource(this.shownLogLines);
|
this.filteredDataGridView.SetDataSource(this.shownLogLines);
|
||||||
foreach (string s in new string[] { "Malformed", "IsError", "OriginalLogLine", "File", "LineNo" })
|
foreach (string s in new string[] { "Malformed", "IsError", "OriginalLogLine", "File", "LineNo" })
|
||||||
{
|
{
|
||||||
|
|
@ -132,7 +132,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
long bytes = 0;
|
long bytes = 0;
|
||||||
|
|
||||||
List<TextFileLine> toAddText = new List<TextFileLine>();
|
List<TextFileLine> toAddText = new List<TextFileLine>();
|
||||||
List<PositionedCosmosLogEntry> toAddLog = new List<PositionedCosmosLogEntry>();
|
List<PositionedDryadLogEntry> toAddLog = new List<PositionedDryadLogEntry>();
|
||||||
while (!sr.EndOfStream)
|
while (!sr.EndOfStream)
|
||||||
{
|
{
|
||||||
string line = sr.ReadLine();
|
string line = sr.ReadLine();
|
||||||
|
|
@ -141,7 +141,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
toAddText.Add(new TextFileLine(lineno, line));
|
toAddText.Add(new TextFileLine(lineno, line));
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
PositionedCosmosLogEntry cle = new PositionedCosmosLogEntry(filename, lineno, line);
|
PositionedDryadLogEntry cle = new PositionedDryadLogEntry(filename, lineno, line);
|
||||||
if (cle.Malformed)
|
if (cle.Malformed)
|
||||||
{
|
{
|
||||||
Trace.TraceInformation("Malformed log entry: " + cle.OriginalLogLine);
|
Trace.TraceInformation("Malformed log entry: " + cle.OriginalLogLine);
|
||||||
|
|
@ -204,7 +204,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
PositionedCosmosLogEntry cle = new PositionedCosmosLogEntry(file, lineno, text);
|
PositionedDryadLogEntry cle = new PositionedDryadLogEntry(file, lineno, text);
|
||||||
if (cle.Malformed)
|
if (cle.Malformed)
|
||||||
return;
|
return;
|
||||||
this.shownLogLines.AddItem(cle);
|
this.shownLogLines.AddItem(cle);
|
||||||
|
|
@ -274,7 +274,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
var rows = this.filteredDataGridView.DataGridView.SelectedRows;
|
var rows = this.filteredDataGridView.DataGridView.SelectedRows;
|
||||||
for (int i = 0; i < rows.Count; i++)
|
for (int i = 0; i < rows.Count; i++)
|
||||||
{
|
{
|
||||||
PositionedCosmosLogEntry entry = ((PositionedCosmosLogEntry)rows[i].DataBoundItem);
|
PositionedDryadLogEntry entry = ((PositionedDryadLogEntry)rows[i].DataBoundItem);
|
||||||
position += entry.File + ":" + entry.LineNo + Environment.NewLine;
|
position += entry.File + ":" + entry.LineNo + Environment.NewLine;
|
||||||
}
|
}
|
||||||
MessageBox.Show(position, "File containing log entries");
|
MessageBox.Show(position, "File containing log entries");
|
||||||
|
|
@ -319,7 +319,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Cosmos log entry with position information.
|
/// Cosmos log entry with position information.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class PositionedCosmosLogEntry : CosmosLogEntry
|
public class PositionedDryadLogEntry : DryadLogEntry
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// File containing the log entry.
|
/// File containing the log entry.
|
||||||
|
|
@ -336,7 +336,7 @@ namespace Microsoft.Research.Calypso.DryadAnalysis
|
||||||
/// <param name="file">File containing the log entry.</param>
|
/// <param name="file">File containing the log entry.</param>
|
||||||
/// <param name="lineno">Line number.</param>
|
/// <param name="lineno">Line number.</param>
|
||||||
/// <param name="line">Line contents.</param>
|
/// <param name="line">Line contents.</param>
|
||||||
public PositionedCosmosLogEntry(string file, long lineno, string line)
|
public PositionedDryadLogEntry(string file, long lineno, string line)
|
||||||
: base(line)
|
: base(line)
|
||||||
{
|
{
|
||||||
this.File = file;
|
this.File = file;
|
||||||
|
|
|
||||||
|
|
@ -25,12 +25,12 @@ using System.Runtime.InteropServices;
|
||||||
// General Information about an assembly is controlled through the following
|
// General Information about an assembly is controlled through the following
|
||||||
// set of attributes. Change these attribute values to modify the information
|
// set of attributes. Change these attribute values to modify the information
|
||||||
// associated with an assembly.
|
// associated with an assembly.
|
||||||
[assembly: AssemblyTitle("DryadAnalysis")]
|
[assembly: AssemblyTitle("DryadLinqBrowser")]
|
||||||
[assembly: AssemblyDescription("")]
|
[assembly: AssemblyDescription("")]
|
||||||
[assembly: AssemblyConfiguration("")]
|
[assembly: AssemblyConfiguration("")]
|
||||||
[assembly: AssemblyCompany("Microsoft Corporation")]
|
[assembly: AssemblyCompany("MSIT")]
|
||||||
[assembly: AssemblyProduct("DryadAnalysis")]
|
[assembly: AssemblyProduct("DryadLinqBrowser")]
|
||||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
|
[assembly: AssemblyCopyright("Copyright © Microsoft")]
|
||||||
[assembly: AssemblyTrademark("")]
|
[assembly: AssemblyTrademark("")]
|
||||||
[assembly: AssemblyCulture("")]
|
[assembly: AssemblyCulture("")]
|
||||||
|
|
||||||
|
|
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
|
||||||
// You can specify all the values or you can default the Build and Revision Numbers
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
// by using the '*' as shown below:
|
// by using the '*' as shown below:
|
||||||
// [assembly: AssemblyVersion("1.0.*")]
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
[assembly: AssemblyVersion("0.1.0.0")]
|
[assembly: AssemblyVersion("1.0.1.0")]
|
||||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
[assembly: AssemblyFileVersion("1.0.1.0")]
|
||||||
|
|
|
||||||
|
|
@ -19,13 +19,13 @@ limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
using Microsoft.Research.Calypso.Tools;
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Drawing;
|
using System.Drawing;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
using Microsoft.Research.Tools;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.JobObjectModel
|
namespace Microsoft.Research.JobObjectModel
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Information about the dynamic execution schedule of a job.
|
/// Information about the dynamic execution schedule of a job.
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,22 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
|
<package id="Microsoft.Bcl" version="1.1.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Async" version="1.0.166" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.Net.Http" version="2.2.19" targetFramework="net45" />
|
||||||
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
|
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
|
||||||
</packages>
|
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common" version="1.0.1" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management" version="1.0.0" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||||
|
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.0.0" targetFramework="net45" />
|
||||||
|
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||||
|
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||||
|
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||||
|
</packages>
|
||||||
|
|
@ -1,48 +0,0 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
|
||||||
<package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
|
|
||||||
<metadata>
|
|
||||||
<id>Microsoft.Research.Calypso</id>
|
|
||||||
<version>0.7.0-beta024</version>
|
|
||||||
<title>Calypso Job Browser for DryadLINQ</title>
|
|
||||||
<authors>msrsvc</authors>
|
|
||||||
<owners>msrsvc,Microsoft</owners>
|
|
||||||
<licenseUrl>http://www.apache.org/licenses/LICENSE-2.0</licenseUrl>
|
|
||||||
<projectUrl>http://research.microsoft.com/en-us/um/siliconvalley/projects/BigDataDev/</projectUrl>
|
|
||||||
<requireLicenseAcceptance>true</requireLicenseAcceptance>
|
|
||||||
<description>The DryadLINQ job browser is a graphical user interface which provides a unified view of a large part of the distributed state of a DryadLINQ job. In this document we provide a brief overview of the main capabilities of the DryadLINQ job browser.</description>
|
|
||||||
<copyright>© Microsoft Corporation. All rights reserved.</copyright>
|
|
||||||
<language>en-US</language>
|
|
||||||
<tags>DryadLINQ Dryad Azure</tags>
|
|
||||||
<dependencies>
|
|
||||||
<group targetFramework=".NETFramework4.5">
|
|
||||||
<dependency id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" />
|
|
||||||
<dependency id="Microsoft.Research.Peloponnese" version="0.7.0-beta024" />
|
|
||||||
</group>
|
|
||||||
</dependencies>
|
|
||||||
<frameworkAssemblies>
|
|
||||||
<frameworkAssembly assemblyName="System" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Core" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Data" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Data.DataSetExtensions" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Data.Linq" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Deployment" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Drawing" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Runtime.Serialization" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.ServiceModel" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Windows.Forms" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Xml" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Xml.Linq" targetFramework=".NETFramework4.5" />
|
|
||||||
</frameworkAssemblies>
|
|
||||||
</metadata>
|
|
||||||
<files>
|
|
||||||
<file src="Calypso\bin\x64\Release\DistributedDataCollection.dll" target="lib\DistributedDataCollection.dll" />
|
|
||||||
<file src="Calypso\bin\x64\Release\DistributedDataCollection.pdb" target="lib\DistributedDataCollection.pdb" />
|
|
||||||
<file src="Calypso\bin\x64\Release\JobBrowser.exe" target="lib\JobBrowser.exe" />
|
|
||||||
<file src="Calypso\bin\x64\Release\JobBrowser.exe.config" target="lib\JobBrowser.exe.config" />
|
|
||||||
<file src="Calypso\bin\x64\Release\JobBrowser.pdb" target="lib\JobBrowser.pdb" />
|
|
||||||
<file src="Calypso\bin\x64\Release\Microsoft.Research.Calypso.Tools.dll" target="lib\Microsoft.Research.Calypso.Tools.dll" />
|
|
||||||
<file src="Calypso\bin\x64\Release\Microsoft.Research.Calypso.Tools.pdb" target="lib\Microsoft.Research.Calypso.Tools.pdb" />
|
|
||||||
<file src="Calypso\bin\x64\Release\Microsoft.Research.Calypso.UsefulForms.dll" target="lib\Microsoft.Research.Calypso.UsefulForms.dll" />
|
|
||||||
<file src="Calypso\bin\x64\Release\Microsoft.Research.Calypso.UsefulForms.pdb" target="lib\Microsoft.Research.Calypso.UsefulForms.pdb" />
|
|
||||||
</files>
|
|
||||||
</package>
|
|
||||||
|
|
@ -24,7 +24,7 @@ using System.Collections.Generic;
|
||||||
using System.Drawing;
|
using System.Drawing;
|
||||||
using System.Drawing.Drawing2D;
|
using System.Drawing.Drawing2D;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.Tools
|
namespace Microsoft.Research.Tools
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Deals with the plane representation of a graph.
|
/// Deals with the plane representation of a graph.
|
||||||
|
|
|
||||||
|
|
@ -28,9 +28,9 @@ using System.Runtime.InteropServices;
|
||||||
[assembly: AssemblyTitle("Tools")]
|
[assembly: AssemblyTitle("Tools")]
|
||||||
[assembly: AssemblyDescription("")]
|
[assembly: AssemblyDescription("")]
|
||||||
[assembly: AssemblyConfiguration("")]
|
[assembly: AssemblyConfiguration("")]
|
||||||
[assembly: AssemblyCompany("Microsoft Corporation")]
|
[assembly: AssemblyCompany("MSIT")]
|
||||||
[assembly: AssemblyProduct("Tools")]
|
[assembly: AssemblyProduct("Tools")]
|
||||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
|
[assembly: AssemblyCopyright("Copyright © MSIT 2008")]
|
||||||
[assembly: AssemblyTrademark("")]
|
[assembly: AssemblyTrademark("")]
|
||||||
[assembly: AssemblyCulture("")]
|
[assembly: AssemblyCulture("")]
|
||||||
|
|
||||||
|
|
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
|
||||||
// You can specify all the values or you can default the Build and Revision Numbers
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
// by using the '*' as shown below:
|
// by using the '*' as shown below:
|
||||||
// [assembly: AssemblyVersion("1.0.*")]
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
[assembly: AssemblyVersion("0.1.0.0")]
|
[assembly: AssemblyVersion("1.0.0.0")]
|
||||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
<OutputType>Library</OutputType>
|
<OutputType>Library</OutputType>
|
||||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||||
<RootNamespace>Microsoft.Research.Artemis.Tools</RootNamespace>
|
<RootNamespace>Microsoft.Research.Artemis.Tools</RootNamespace>
|
||||||
<AssemblyName>Microsoft.Research.DryadLinq.JobBrowser.Tools</AssemblyName>
|
<AssemblyName>Microsoft.Research.Calypso.Tools</AssemblyName>
|
||||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||||
<FileAlignment>512</FileAlignment>
|
<FileAlignment>512</FileAlignment>
|
||||||
<FileUpgradeFlags>
|
<FileUpgradeFlags>
|
||||||
|
|
@ -33,28 +33,6 @@
|
||||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||||
<TargetFrameworkProfile />
|
<TargetFrameworkProfile />
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
|
||||||
<DebugSymbols>true</DebugSymbols>
|
|
||||||
<DebugType>full</DebugType>
|
|
||||||
<Optimize>false</Optimize>
|
|
||||||
<OutputPath>bin\Debug\</OutputPath>
|
|
||||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
<DocumentationFile>bin\Debug\Microsoft.Research.DryadLinq.JobBrowser.Tools.xml</DocumentationFile>
|
|
||||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
|
||||||
<Prefer32Bit>false</Prefer32Bit>
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
|
||||||
<DebugType>pdbonly</DebugType>
|
|
||||||
<Optimize>true</Optimize>
|
|
||||||
<OutputPath>bin\Release\</OutputPath>
|
|
||||||
<DefineConstants>TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
|
||||||
<Prefer32Bit>false</Prefer32Bit>
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||||
<DebugSymbols>true</DebugSymbols>
|
<DebugSymbols>true</DebugSymbols>
|
||||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||||
|
|
@ -84,6 +62,7 @@
|
||||||
<Reference Include="System.Windows.Forms" />
|
<Reference Include="System.Windows.Forms" />
|
||||||
<Reference Include="System.Data" />
|
<Reference Include="System.Data" />
|
||||||
<Reference Include="System.Xml" />
|
<Reference Include="System.Xml" />
|
||||||
|
<Reference Include="System.Xml.Linq" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Compile Include="DrawingSurface.cs" />
|
<Compile Include="DrawingSurface.cs" />
|
||||||
|
|
@ -117,4 +96,4 @@
|
||||||
<Target Name="AfterBuild">
|
<Target Name="AfterBuild">
|
||||||
</Target>
|
</Target>
|
||||||
-->
|
-->
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -19,7 +19,7 @@ limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.Tools
|
namespace Microsoft.Research.Tools
|
||||||
{
|
{
|
||||||
using System;
|
using System;
|
||||||
using System.Drawing;
|
using System.Drawing;
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.Tools
|
namespace Microsoft.Research.Tools
|
||||||
{
|
{
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ using System.Net;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using System.Runtime.InteropServices;
|
using System.Runtime.InteropServices;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
using System.Windows.Forms;
|
||||||
using System.Xml;
|
using System.Xml;
|
||||||
using System.Xml.Serialization;
|
using System.Xml.Serialization;
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
|
|
@ -38,7 +39,7 @@ using System.Threading;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
|
|
||||||
// Implement here generally-useful tools.
|
// Implement here generally-useful tools.
|
||||||
namespace Microsoft.Research.Calypso.Tools
|
namespace Microsoft.Research.Tools
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// An error handling function.
|
/// An error handling function.
|
||||||
|
|
@ -66,6 +67,38 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
LongOp,
|
LongOp,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Communication management with background activities.
|
||||||
|
/// </summary>
|
||||||
|
public struct CommManager
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Used to report status.
|
||||||
|
/// </summary>
|
||||||
|
public StatusReporter Status;
|
||||||
|
/// <summary>
|
||||||
|
/// Used to report progress.
|
||||||
|
/// </summary>
|
||||||
|
public Action<int> Progress;
|
||||||
|
/// <summary>
|
||||||
|
/// Used to cancel activities.
|
||||||
|
/// </summary>
|
||||||
|
public CancellationToken Token;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Create a communication manager.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="status">Status to report errors.</param>
|
||||||
|
/// <param name="progress">Action to report progress.</param>
|
||||||
|
/// <param name="token">Token to cancel computations.</param>
|
||||||
|
public CommManager(StatusReporter status, Action<int> progress, CancellationToken token)
|
||||||
|
{
|
||||||
|
this.Status = status;
|
||||||
|
this.Progress = progress;
|
||||||
|
this.Token = token;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Untyped version of work item.
|
/// Untyped version of work item.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|
@ -100,6 +133,10 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
/// <param name="ex">Exception that occurred during background work (or null).</param>
|
/// <param name="ex">Exception that occurred during background work (or null).</param>
|
||||||
/// </summary>
|
/// </summary>
|
||||||
void RunContinuation(Exception ex);
|
void RunContinuation(Exception ex);
|
||||||
|
/// <summary>
|
||||||
|
/// Can be used to cancel this work item.
|
||||||
|
/// </summary>
|
||||||
|
CancellationTokenSource TokenSource { get; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -111,7 +148,7 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Computation to invoke. If the computation is not cancelled the result is passed as the second argument to the continuation.
|
/// Computation to invoke. If the computation is not cancelled the result is passed as the second argument to the continuation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public Func<StatusReporter, Action<int>, T> Computation { get; protected set; }
|
public Func<CommManager, T> Computation { get; protected set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Function to call when the work is completed. The first argument is 'true' if the computation was not cancelled. The second argument is the result of the computation.
|
/// Function to call when the work is completed. The first argument is 'true' if the computation was not cancelled. The second argument is the result of the computation.
|
||||||
|
|
@ -141,6 +178,10 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
/// Queue containing item.
|
/// Queue containing item.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private BackgroundWorkQueue queue;
|
private BackgroundWorkQueue queue;
|
||||||
|
/// <summary>
|
||||||
|
/// Source for cancellation token.
|
||||||
|
/// </summary>
|
||||||
|
public CancellationTokenSource TokenSource { get; protected set; }
|
||||||
|
|
||||||
// ReSharper disable ConvertToConstant.Local
|
// ReSharper disable ConvertToConstant.Local
|
||||||
bool TraceAsync =
|
bool TraceAsync =
|
||||||
|
|
@ -159,7 +200,7 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
/// <param name="computation">Computation to perform on a background thread. Ideally this should always be a static method.</param>
|
/// <param name="computation">Computation to perform on a background thread. Ideally this should always be a static method.</param>
|
||||||
/// <param name="continuation">Continuation to invoke on the foreground thread when work is done.</param>
|
/// <param name="continuation">Continuation to invoke on the foreground thread when work is done.</param>
|
||||||
/// <param name="description">Description of the background work.</param>
|
/// <param name="description">Description of the background work.</param>
|
||||||
public BackgroundWorkItem(Func<StatusReporter, Action<int>, T> computation, Action<bool, T> continuation, string description)
|
public BackgroundWorkItem(Func<CommManager, T> computation, Action<bool, T> continuation, string description)
|
||||||
{
|
{
|
||||||
this.Description = description;
|
this.Description = description;
|
||||||
this.Computation = computation;
|
this.Computation = computation;
|
||||||
|
|
@ -167,23 +208,22 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
this.reporter = null;
|
this.reporter = null;
|
||||||
this.queue = null;
|
this.queue = null;
|
||||||
this.Id = crtid++;
|
this.Id = crtid++;
|
||||||
|
this.TokenSource = new CancellationTokenSource();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Perform the background work.
|
/// Perform the background work.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="queue">Worker which does the work.</param>
|
/// <param name="q">Worker which does the work.</param>
|
||||||
/// <param name="reporter">Delegate used to report errors.</param>
|
/// <param name="rep">Delegate used to report errors.</param>
|
||||||
/// <param name="progressReporter">Delegate used to report progress.</param>
|
/// <param name="progressReporter">Delegate used to report progress.</param>
|
||||||
/// <param name="cancel">If true for an item, cancel it.</param>
|
/// <param name="cancel">If true for an item, cancel it.</param>
|
||||||
// ReSharper disable ParameterHidesMember
|
public void Queue(BackgroundWorkQueue q, StatusReporter rep, Action<int> progressReporter, Func<IBackgroundWorkItem, bool> cancel)
|
||||||
public void Queue(BackgroundWorkQueue queue, StatusReporter reporter, Action<int> progressReporter, Func<IBackgroundWorkItem, bool> cancel)
|
|
||||||
// ReSharper restore ParameterHidesMember
|
|
||||||
{
|
{
|
||||||
if (TraceAsync)
|
if (TraceAsync)
|
||||||
Console.WriteLine("{0} Queueing {1}", Utilities.PreciseTime, this.Description);
|
Console.WriteLine("{0} Queueing {1}", Utilities.PreciseTime, this.Description);
|
||||||
this.queue = queue;
|
this.queue = q;
|
||||||
this.reporter = reporter;
|
this.reporter = rep;
|
||||||
this.progress = progressReporter;
|
this.progress = progressReporter;
|
||||||
this.queue.CancelMatching(cancel);
|
this.queue.CancelMatching(cancel);
|
||||||
this.queue.Enqueue(this);
|
this.queue.Enqueue(this);
|
||||||
|
|
@ -200,7 +240,8 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
Console.WriteLine("{0} Running function {1}", Utilities.PreciseTime, this.Description);
|
Console.WriteLine("{0} Running function {1}", Utilities.PreciseTime, this.Description);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
this.Result = this.Computation(this.reporter, this.progress);
|
CommManager manager = new CommManager(this.reporter, this.progress, this.TokenSource.Token);
|
||||||
|
this.Result = this.Computation(manager);
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
|
|
@ -235,6 +276,7 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
if (TraceAsync)
|
if (TraceAsync)
|
||||||
Console.WriteLine("{1}/{0}: Cancelling", this.Description, this.Id);
|
Console.WriteLine("{1}/{0}: Cancelling", this.Description, this.Id);
|
||||||
this.Cancelled = true;
|
this.Cancelled = true;
|
||||||
|
this.TokenSource.Cancel();
|
||||||
this.queue.CancelMe(this);
|
this.queue.CancelMe(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -266,12 +308,18 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
/// </summary>
|
/// </summary>
|
||||||
IBackgroundWorkItem current;
|
IBackgroundWorkItem current;
|
||||||
|
|
||||||
|
private ToolStripStatusLabel currentItemLabel, queueSizeLabel;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Create a background work queue servicing a specified worker.
|
/// Create a background work queue servicing a specified worker.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="worker">Worker to use.</param>
|
/// <param name="worker">Worker to use.</param>
|
||||||
public BackgroundWorkQueue(BackgroundWorker worker)
|
/// <param name="current">Label where the current work is displayed.</param>
|
||||||
|
/// <param name="queue">Label where the queue size is displayed.</param>
|
||||||
|
public BackgroundWorkQueue(BackgroundWorker worker, ToolStripStatusLabel current, ToolStripStatusLabel queue)
|
||||||
{
|
{
|
||||||
|
this.currentItemLabel = current;
|
||||||
|
this.queueSizeLabel = queue;
|
||||||
if (worker == null)
|
if (worker == null)
|
||||||
throw new ArgumentNullException("worker");
|
throw new ArgumentNullException("worker");
|
||||||
this.BackgroundWorker = worker;
|
this.BackgroundWorker = worker;
|
||||||
|
|
@ -280,6 +328,7 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
this.BackgroundWorker.DoWork += this.worker_DoWork;
|
this.BackgroundWorker.DoWork += this.worker_DoWork;
|
||||||
this.queue = new List<IBackgroundWorkItem>();
|
this.queue = new List<IBackgroundWorkItem>();
|
||||||
this.current = null;
|
this.current = null;
|
||||||
|
this.stopped = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
@ -289,7 +338,7 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
/// <param name="e">Unused.</param>
|
/// <param name="e">Unused.</param>
|
||||||
void worker_DoWork(object sender, DoWorkEventArgs e)
|
void worker_DoWork(object sender, DoWorkEventArgs e)
|
||||||
{
|
{
|
||||||
if (this.current == null)
|
if (this.stopped || this.current == null)
|
||||||
return;
|
return;
|
||||||
#if DEBUG_WORKQUEUE
|
#if DEBUG_WORKQUEUE
|
||||||
#endif
|
#endif
|
||||||
|
|
@ -306,6 +355,8 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
e.Cancel = true;
|
e.Cancel = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private bool stopped;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Called when the worker is completed.
|
/// Called when the worker is completed.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|
@ -319,6 +370,8 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
#endif
|
#endif
|
||||||
IBackgroundWorkItem crt = this.current;
|
IBackgroundWorkItem crt = this.current;
|
||||||
this.current = null;
|
this.current = null;
|
||||||
|
if (this.currentItemLabel != null)
|
||||||
|
this.currentItemLabel.Text = "";
|
||||||
crt.RunContinuation(e.Error);
|
crt.RunContinuation(e.Error);
|
||||||
}
|
}
|
||||||
this.Kick();
|
this.Kick();
|
||||||
|
|
@ -348,7 +401,11 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
if (this.current != null)
|
if (this.current != null)
|
||||||
throw new Exception("current is not null");
|
throw new Exception("current is not null");
|
||||||
this.current = this.queue[0];
|
this.current = this.queue[0];
|
||||||
|
if (this.currentItemLabel != null)
|
||||||
|
this.currentItemLabel.Text = "Doing " + this.current.Description;
|
||||||
this.queue.RemoveAt(0);
|
this.queue.RemoveAt(0);
|
||||||
|
if (this.queueSizeLabel != null)
|
||||||
|
this.queueSizeLabel.Text = "Pending " + this.queue.Count + " items";
|
||||||
this.Start();
|
this.Start();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -411,9 +468,22 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Stop the queue.
|
||||||
|
/// </summary>
|
||||||
public void Stop()
|
public void Stop()
|
||||||
{
|
{
|
||||||
// TODO
|
this.stopped = true;
|
||||||
|
this.CancelCurrentWork();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Cancel the currently running work.
|
||||||
|
/// </summary>
|
||||||
|
public void CancelCurrentWork()
|
||||||
|
{
|
||||||
|
if (this.current == null) return;
|
||||||
|
this.current.Cancel();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3409,7 +3479,8 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
/// Read the stream to the end from the current position.
|
/// Read the stream to the end from the current position.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The contents of the stream.</returns>
|
/// <returns>The contents of the stream.</returns>
|
||||||
string ReadToEnd();
|
/// <param name="token">Can be used to cancel the reading.</param>
|
||||||
|
string ReadToEnd(CancellationToken token);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Read all the lines remaining in the stream.
|
/// Read all the lines remaining in the stream.
|
||||||
|
|
@ -3465,11 +3536,15 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
/// Read the whole stream to the end.
|
/// Read the whole stream to the end.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>A string containing the whole contents of the stream.</returns>
|
/// <returns>A string containing the whole contents of the stream.</returns>
|
||||||
public virtual string ReadToEnd()
|
/// <param name="token">Can be used to cancel the reading.</param>
|
||||||
|
public virtual string ReadToEnd(CancellationToken token)
|
||||||
{
|
{
|
||||||
StringBuilder result = new StringBuilder();
|
StringBuilder result = new StringBuilder();
|
||||||
foreach (string s in this.ReadAllLines())
|
foreach (string s in this.ReadAllLines())
|
||||||
|
{
|
||||||
|
token.ThrowIfCancellationRequested();
|
||||||
result.AppendLine(s);
|
result.AppendLine(s);
|
||||||
|
}
|
||||||
return result.ToString();
|
return result.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -3612,11 +3687,13 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
/// Read the stream to the end from the current position.
|
/// Read the stream to the end from the current position.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The contents of the stream.</returns>
|
/// <returns>The contents of the stream.</returns>
|
||||||
public override string ReadToEnd()
|
/// <param name="token">Can be used to cancel the reading.</param>
|
||||||
|
public override string ReadToEnd(CancellationToken token)
|
||||||
{
|
{
|
||||||
string result = this.actualReader.ReadToEnd();
|
string result = this.actualReader.ReadToEnd();
|
||||||
if (this.cacheWriter != null)
|
if (this.cacheWriter != null)
|
||||||
{
|
{
|
||||||
|
token.ThrowIfCancellationRequested();
|
||||||
this.cacheWriter.Write(result);
|
this.cacheWriter.Write(result);
|
||||||
this.cacheWriter.Close();
|
this.cacheWriter.Close();
|
||||||
if (this.onClose != null)
|
if (this.onClose != null)
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
namespace Microsoft.Research.Calypso.Tools
|
namespace Microsoft.Research.Tools
|
||||||
{
|
{
|
||||||
partial class FilteredDataGridView
|
partial class FilteredDataGridView
|
||||||
{
|
{
|
||||||
|
|
@ -33,7 +33,7 @@ namespace Microsoft.Research.Calypso.Tools
|
||||||
/// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
|
/// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
|
||||||
protected override void Dispose(bool disposing)
|
protected override void Dispose(bool disposing)
|
||||||
{
|
{
|
||||||
if (disposing && (components != null))
|
if (disposing && (components != null))
|
||||||
{
|
{
|
||||||
components.Dispose();
|
components.Dispose();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@ using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.Tools
|
namespace Microsoft.Research.Tools
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A data grid that filters the contents.
|
/// A data grid that filters the contents.
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
namespace Microsoft.Research.Calypso.UsefulForms
|
namespace Microsoft.Research.UsefulForms
|
||||||
{
|
{
|
||||||
partial class PasswordDialog
|
partial class PasswordDialog
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,7 @@ limitations under the License.
|
||||||
|
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.UsefulForms
|
namespace Microsoft.Research.UsefulForms
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Prompt user for password.
|
/// Prompt user for password.
|
||||||
|
|
|
||||||
|
|
@ -28,9 +28,9 @@ using System.Runtime.InteropServices;
|
||||||
[assembly: AssemblyTitle("UsefulForms")]
|
[assembly: AssemblyTitle("UsefulForms")]
|
||||||
[assembly: AssemblyDescription("")]
|
[assembly: AssemblyDescription("")]
|
||||||
[assembly: AssemblyConfiguration("")]
|
[assembly: AssemblyConfiguration("")]
|
||||||
[assembly: AssemblyCompany("Microsoft Corporation")]
|
[assembly: AssemblyCompany("Microsoft")]
|
||||||
[assembly: AssemblyProduct("UsefulForms")]
|
[assembly: AssemblyProduct("UsefulForms")]
|
||||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
|
[assembly: AssemblyCopyright("Copyright © Microsoft 2010")]
|
||||||
[assembly: AssemblyTrademark("")]
|
[assembly: AssemblyTrademark("")]
|
||||||
[assembly: AssemblyCulture("")]
|
[assembly: AssemblyCulture("")]
|
||||||
|
|
||||||
|
|
@ -52,5 +52,5 @@ using System.Runtime.InteropServices;
|
||||||
// You can specify all the values or you can default the Build and Revision Numbers
|
// You can specify all the values or you can default the Build and Revision Numbers
|
||||||
// by using the '*' as shown below:
|
// by using the '*' as shown below:
|
||||||
// [assembly: AssemblyVersion("1.0.*")]
|
// [assembly: AssemblyVersion("1.0.*")]
|
||||||
[assembly: AssemblyVersion("0.1.0.0")]
|
[assembly: AssemblyVersion("1.0.0.0")]
|
||||||
[assembly: AssemblyFileVersion("0.1.0.0")]
|
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@ using System.Drawing;
|
||||||
using System;
|
using System;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.Tools
|
namespace Microsoft.Research.Tools
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Delegate used for invoking status messages from accross threads.
|
/// Delegate used for invoking status messages from accross threads.
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
namespace Microsoft.Research.Calypso.UsefulForms
|
namespace Microsoft.Research.UsefulForms
|
||||||
{
|
{
|
||||||
partial class CustomDialog
|
partial class CustomDialog
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.UsefulForms
|
namespace Microsoft.Research.UsefulForms
|
||||||
{
|
{
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||||
|
|
@ -9,7 +9,7 @@
|
||||||
<OutputType>Library</OutputType>
|
<OutputType>Library</OutputType>
|
||||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||||
<RootNamespace>UsefulForms</RootNamespace>
|
<RootNamespace>UsefulForms</RootNamespace>
|
||||||
<AssemblyName>Microsoft.Research.DryadLinq.JobBrowser.UsefulForms</AssemblyName>
|
<AssemblyName>Microsoft.Research.Calypso.UsefulForms</AssemblyName>
|
||||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||||
<FileAlignment>512</FileAlignment>
|
<FileAlignment>512</FileAlignment>
|
||||||
<FileUpgradeFlags>
|
<FileUpgradeFlags>
|
||||||
|
|
@ -33,27 +33,6 @@
|
||||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||||
<TargetFrameworkProfile />
|
<TargetFrameworkProfile />
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
|
||||||
<DebugSymbols>true</DebugSymbols>
|
|
||||||
<DebugType>full</DebugType>
|
|
||||||
<Optimize>false</Optimize>
|
|
||||||
<OutputPath>bin\Debug\</OutputPath>
|
|
||||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
|
||||||
<Prefer32Bit>false</Prefer32Bit>
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
|
||||||
<DebugType>pdbonly</DebugType>
|
|
||||||
<Optimize>true</Optimize>
|
|
||||||
<OutputPath>bin\Release\</OutputPath>
|
|
||||||
<DefineConstants>TRACE</DefineConstants>
|
|
||||||
<ErrorReport>prompt</ErrorReport>
|
|
||||||
<WarningLevel>4</WarningLevel>
|
|
||||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
|
||||||
<Prefer32Bit>false</Prefer32Bit>
|
|
||||||
</PropertyGroup>
|
|
||||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||||
<DebugSymbols>true</DebugSymbols>
|
<DebugSymbols>true</DebugSymbols>
|
||||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||||
|
|
@ -72,6 +51,9 @@
|
||||||
<ErrorReport>prompt</ErrorReport>
|
<ErrorReport>prompt</ErrorReport>
|
||||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
<PropertyGroup>
|
||||||
|
<RunPostBuildEvent>OnBuildSuccess</RunPostBuildEvent>
|
||||||
|
</PropertyGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Reference Include="System" />
|
<Reference Include="System" />
|
||||||
<Reference Include="System.Core">
|
<Reference Include="System.Core">
|
||||||
|
|
@ -152,4 +134,4 @@
|
||||||
<Target Name="AfterBuild">
|
<Target Name="AfterBuild">
|
||||||
</Target>
|
</Target>
|
||||||
-->
|
-->
|
||||||
</Project>
|
</Project>
|
||||||
|
|
@ -18,7 +18,7 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
||||||
limitations under the License.
|
limitations under the License.
|
||||||
|
|
||||||
*/
|
*/
|
||||||
namespace Microsoft.Research.Calypso.UsefulForms
|
namespace Microsoft.Research.UsefulForms
|
||||||
{
|
{
|
||||||
partial class WarningBox
|
partial class WarningBox
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ limitations under the License.
|
||||||
using System;
|
using System;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
|
|
||||||
namespace Microsoft.Research.Calypso.UsefulForms
|
namespace Microsoft.Research.UsefulForms
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// A box displaying a warning, which can be turned off.
|
/// A box displaying a warning, which can be turned off.
|
||||||
|
|
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 496 KiB |
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 1.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 10 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 22 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 197 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 27 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.0 KiB |
|
|
@ -1,27 +0,0 @@
|
||||||
<?xml version="1.0"?>
|
|
||||||
<package xmlns="http://schemas.microsoft.com/packaging/2013/01/nuspec.xsd">
|
|
||||||
<metadata minClientVersion="2.5">
|
|
||||||
<id>Microsoft.Research.DryadLINQ.MSAGL</id>
|
|
||||||
<version>3.0.0.1</version>
|
|
||||||
<title>MSAGL Binary Package solely for use with DryadLINQ</title>
|
|
||||||
<authors>msrsvc</authors>
|
|
||||||
<owners>msrsvc</owners>
|
|
||||||
<licenseUrl>http://research.microsoft.com/en-us/um/siliconvalley/projects/BigDataDev/MSAGL_License.txt</licenseUrl>
|
|
||||||
<projectUrl>http://research.microsoft.com/en-us/projects/msagl/</projectUrl>
|
|
||||||
<requireLicenseAcceptance>true</requireLicenseAcceptance>
|
|
||||||
<description>This is a binary package of the Microsoft Automatic Graph Layout engine solely for use with DryadLINQ. Please see the linked license for details.</description>
|
|
||||||
<copyright>© Microsoft Corporation. All rights reserved.</copyright>
|
|
||||||
<language>en-US</language>
|
|
||||||
<frameworkAssemblies>
|
|
||||||
<frameworkAssembly assemblyName="System" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Core" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Data.DataSetExtensions" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Data.Linq" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Drawing" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Runtime.Serialization" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.ServiceModel" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Xml" targetFramework=".NETFramework4.5" />
|
|
||||||
<frameworkAssembly assemblyName="System.Xml.Linq" targetFramework=".NETFramework4.5" />
|
|
||||||
</frameworkAssemblies>
|
|
||||||
</metadata>
|
|
||||||
</package>
|
|
||||||
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue