Compare commits
18 Commits
| Author | SHA1 | Date |
|---|---|---|
misard
|
ad28579dd8 | |
|
misard
|
6980be9895 | |
|
misard
|
e72fef34c6 | |
|
Michael Isard
|
04ecff4353 | |
|
misard
|
5e9dde8ab6 | |
|
Michael Isard
|
47d0131083 | |
|
misard
|
b31fadd55e | |
|
misard
|
d3643fc5fb | |
|
misard
|
efc79b749b | |
|
Michael Isard
|
b9cda36f69 | |
Dennis Fetterly
|
5c260ffe0b | |
|
Michael Isard
|
93be3e4b66 | |
|
Michael Isard
|
ab56207d72 | |
|
Michael Isard
|
86242db082 | |
|
Michael Isard
|
5ad103cba7 | |
|
Dennis Fetterly
|
cce4d976b2 | |
|
Dennis Fetterly
|
3471622ef2 | |
|
Dennis Fetterly
|
a2a1682866 |
|
|
@ -19,3 +19,5 @@ x64/
|
|||
build/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
/Calypso/packages
|
||||
/DryadLinqTests/packages
|
||||
|
|
|
|||
|
|
@ -1,86 +0,0 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 2013
|
||||
VisualStudioVersion = 12.0.30110.0
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Calypso", "Calypso\Calypso.csproj", "{EDDD2E0B-A52B-4E25-9436-B874017673FF}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools", "Tools\Tools.csproj", "{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JobObjectModel", "JOM\JobObjectModel.csproj", "{77739535-7FAC-4487-887F-FEBA197E7572}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UsefulForms", "UsefulForms\UsefulForms.csproj", "{27635A68-ADFC-4192-9262-B4E6ECDDCE09}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Debug|Mixed Platforms = Debug|Mixed Platforms
|
||||
Debug|x64 = Debug|x64
|
||||
Debug|x86 = Debug|x86
|
||||
Release|Any CPU = Release|Any CPU
|
||||
Release|Mixed Platforms = Release|Mixed Platforms
|
||||
Release|x64 = Release|x64
|
||||
Release|x86 = Release|x86
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x64.Build.0 = Debug|x64
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x64.ActiveCfg = Release|x64
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x64.Build.0 = Release|x64
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x64.Build.0 = Debug|x64
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x64.ActiveCfg = Release|x64
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x64.Build.0 = Release|x64
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x64.Build.0 = Debug|x64
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x64.ActiveCfg = Release|x64
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x64.Build.0 = Release|x64
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x64.Build.0 = Debug|x64
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x64.ActiveCfg = Release|x64
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x64.Build.0 = Release|x64
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
|
@ -1,266 +0,0 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProductVersion>9.0.30729</ProductVersion>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<ProjectGuid>{EDDD2E0B-A52B-4E25-9436-B874017673FF}</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>DryadAnalysis</RootNamespace>
|
||||
<AssemblyName>JobBrowser</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<StartupObject>
|
||||
</StartupObject>
|
||||
<IsWebBootstrapper>false</IsWebBootstrapper>
|
||||
<ApplicationIcon>icons\DryadAnalysis.ico</ApplicationIcon>
|
||||
<ApplicationManifest>app.manifest</ApplicationManifest>
|
||||
<FileUpgradeFlags>
|
||||
</FileUpgradeFlags>
|
||||
<UpgradeBackupLocation>
|
||||
</UpgradeBackupLocation>
|
||||
<OldToolsVersion>3.5</OldToolsVersion>
|
||||
<PublishUrl>publish\</PublishUrl>
|
||||
<Install>true</Install>
|
||||
<InstallFrom>Disk</InstallFrom>
|
||||
<UpdateEnabled>false</UpdateEnabled>
|
||||
<UpdateMode>Foreground</UpdateMode>
|
||||
<UpdateInterval>7</UpdateInterval>
|
||||
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
|
||||
<UpdatePeriodically>false</UpdatePeriodically>
|
||||
<UpdateRequired>false</UpdateRequired>
|
||||
<MapFileExtensions>true</MapFileExtensions>
|
||||
<ApplicationRevision>0</ApplicationRevision>
|
||||
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
|
||||
<UseApplicationTrust>false</UseApplicationTrust>
|
||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||
<TargetFrameworkProfile />
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>TRACE;DEBUG</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<DocumentationFile>bin\Debug\JobBrowser.xml</DocumentationFile>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<Prefer32Bit>false</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<Prefer32Bit>false</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||
<DefineConstants>TRACE;DEBUG</DefineConstants>
|
||||
<DocumentationFile>bin\Debug\JobBrowser.xml</DocumentationFile>
|
||||
<DebugType>full</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
|
||||
<OutputPath>bin\x64\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<Optimize>true</Optimize>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="EnvDTE, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="EnvDTE80, Version=8.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE80.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="EnvDTE90, Version=9.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\IDE\PublicAssemblies\EnvDTE90.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hpc.Scheduler, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\Daphne\Dependences\Microsoft.Hpc.Scheduler.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hpc.Scheduler.Properties, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\Daphne\Dependences\Microsoft.Hpc.Scheduler.Properties.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl, Version=3.0.0.0, Culture=neutral, PublicKeyToken=7f84f738e1fc245c, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl.Drawing, Version=3.0.0.0, Culture=neutral, PublicKeyToken=8a3d7c21d5fa1306, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl.GraphViewerGdi, Version=3.0.0.0, Culture=neutral, PublicKeyToken=fffc27ea4058b3a1, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core">
|
||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data.Linq" />
|
||||
<Reference Include="System.Runtime.Serialization" />
|
||||
<Reference Include="System.ServiceModel" />
|
||||
<Reference Include="System.Windows.Forms" />
|
||||
<Reference Include="System.Xml.Linq">
|
||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data.DataSetExtensions">
|
||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Deployment" />
|
||||
<Reference Include="System.Drawing" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="ClusterConfigEditor.cs">
|
||||
<SubType>Form</SubType>
|
||||
</Compile>
|
||||
<Compile Include="ClusterConfigEditor.Designer.cs">
|
||||
<DependentUpon>ClusterConfigEditor.cs</DependentUpon>
|
||||
</Compile>
|
||||
<Compile Include="LocalDebuggingAndProfiling.cs" />
|
||||
<Compile Include="Diagnosis.cs" />
|
||||
<Compile Include="DiagnosisResult.cs">
|
||||
<SubType>Form</SubType>
|
||||
</Compile>
|
||||
<Compile Include="DiagnosisResult.Designer.cs">
|
||||
<DependentUpon>DiagnosisResult.cs</DependentUpon>
|
||||
</Compile>
|
||||
<Compile Include="ClusterBrowser.cs">
|
||||
<SubType>Form</SubType>
|
||||
</Compile>
|
||||
<Compile Include="ClusterBrowser.designer.cs">
|
||||
<DependentUpon>ClusterBrowser.cs</DependentUpon>
|
||||
</Compile>
|
||||
<Compile Include="DryadJobMain.cs" />
|
||||
<Compile Include="JobBrowser.cs">
|
||||
<SubType>Form</SubType>
|
||||
</Compile>
|
||||
<Compile Include="JobBrowser.Designer.cs">
|
||||
<DependentUpon>JobBrowser.cs</DependentUpon>
|
||||
</Compile>
|
||||
<Compile Include="JobSchedule.cs" />
|
||||
<Compile Include="LogViewer.cs">
|
||||
<SubType>Form</SubType>
|
||||
</Compile>
|
||||
<Compile Include="LogViewer.Designer.cs">
|
||||
<DependentUpon>LogViewer.cs</DependentUpon>
|
||||
</Compile>
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
<EmbeddedResource Include="ClusterBrowser.resx">
|
||||
<DependentUpon>ClusterBrowser.cs</DependentUpon>
|
||||
<SubType>Designer</SubType>
|
||||
</EmbeddedResource>
|
||||
<EmbeddedResource Include="ClusterConfigEditor.resx">
|
||||
<DependentUpon>ClusterConfigEditor.cs</DependentUpon>
|
||||
</EmbeddedResource>
|
||||
<EmbeddedResource Include="DiagnosisResult.resx">
|
||||
<DependentUpon>DiagnosisResult.cs</DependentUpon>
|
||||
<SubType>Designer</SubType>
|
||||
</EmbeddedResource>
|
||||
<EmbeddedResource Include="JobBrowser.resx">
|
||||
<DependentUpon>JobBrowser.cs</DependentUpon>
|
||||
<SubType>Designer</SubType>
|
||||
</EmbeddedResource>
|
||||
<EmbeddedResource Include="LogViewer.resx">
|
||||
<DependentUpon>LogViewer.cs</DependentUpon>
|
||||
<SubType>Designer</SubType>
|
||||
</EmbeddedResource>
|
||||
<EmbeddedResource Include="Properties\Resources.resx">
|
||||
<Generator>ResXFileCodeGenerator</Generator>
|
||||
<LastGenOutput>Resources.Designer.cs</LastGenOutput>
|
||||
<SubType>Designer</SubType>
|
||||
</EmbeddedResource>
|
||||
<Compile Include="Properties\Resources.Designer.cs">
|
||||
<AutoGen>True</AutoGen>
|
||||
<DependentUpon>Resources.resx</DependentUpon>
|
||||
<DesignTime>True</DesignTime>
|
||||
</Compile>
|
||||
<None Include="app.config" />
|
||||
<None Include="app.manifest" />
|
||||
<None Include="packages.config" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\JOM\JobObjectModel.csproj">
|
||||
<Project>{77739535-7FAC-4487-887F-FEBA197E7572}</Project>
|
||||
<Name>JobObjectModel</Name>
|
||||
</ProjectReference>
|
||||
<ProjectReference Include="..\Tools\Tools.csproj">
|
||||
<Project>{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}</Project>
|
||||
<Name>Tools</Name>
|
||||
</ProjectReference>
|
||||
<ProjectReference Include="..\UsefulForms\UsefulForms.csproj">
|
||||
<Project>{27635A68-ADFC-4192-9262-B4E6ECDDCE09}</Project>
|
||||
<Name>UsefulForms</Name>
|
||||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Content Include="DryadAnalysis.ico" />
|
||||
<Content Include="icons\Cluster.ico" />
|
||||
<Content Include="icons\ClusterBrowser.ico" />
|
||||
<Content Include="icons\DryadAnalysis.ico" />
|
||||
<Content Include="icons\JobBrowser.ico" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
|
||||
<Install>false</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Framework.2.0">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 2.0 %28x86%29</ProductName>
|
||||
<Install>false</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Framework.3.0">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.0 %28x86%29</ProductName>
|
||||
<Install>false</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.5</ProductName>
|
||||
<Install>true</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.5 SP1</ProductName>
|
||||
<Install>false</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>Windows Installer 3.1</ProductName>
|
||||
<Install>true</Install>
|
||||
</BootstrapperPackage>
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 11.00
|
||||
# Visual Studio 2010
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadAnalysis", "DryadAnalysis.csproj", "{EDDD2E0B-A52B-4E25-9436-B874017673FF}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ARtemis", "..\Artemis\ARtemis.csproj", "{56EFE15B-7255-4F17-830C-2798CB7E0439}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DistributedDataCollection", "..\DistributedDataCollection\DistributedDataCollection.csproj", "{77739535-7FAC-4487-887F-FEBA197E7572}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools", "..\Tools\Tools.csproj", "{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UsefulForms", "..\UsefulForms\UsefulForms.csproj", "{27635A68-ADFC-4192-9262-B4E6ECDDCE09}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadDiag", "..\DryadDiag\DryadDiag.csproj", "{4012902C-54C6-473C-BA48-1087963CF439}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Debug|Mixed Platforms = Debug|Mixed Platforms
|
||||
Debug|x86 = Debug|x86
|
||||
Release|Any CPU = Release|Any CPU
|
||||
Release|Mixed Platforms = Release|Mixed Platforms
|
||||
Release|x86 = Release|x86
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{EDDD2E0B-A52B-4E25-9436-B874017673FF}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{56EFE15B-7255-4F17-830C-2798CB7E0439}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{56EFE15B-7255-4F17-830C-2798CB7E0439}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{56EFE15B-7255-4F17-830C-2798CB7E0439}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{56EFE15B-7255-4F17-830C-2798CB7E0439}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{56EFE15B-7255-4F17-830C-2798CB7E0439}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{56EFE15B-7255-4F17-830C-2798CB7E0439}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{56EFE15B-7255-4F17-830C-2798CB7E0439}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{56EFE15B-7255-4F17-830C-2798CB7E0439}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{56EFE15B-7255-4F17-830C-2798CB7E0439}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{56EFE15B-7255-4F17-830C-2798CB7E0439}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{77739535-7FAC-4487-887F-FEBA197E7572}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|Mixed Platforms.Build.0 = Release|Any CPU
|
||||
{27635A68-ADFC-4192-9262-B4E6ECDDCE09}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{4012902C-54C6-473C-BA48-1087963CF439}.Debug|Any CPU.ActiveCfg = Debug|x86
|
||||
{4012902C-54C6-473C-BA48-1087963CF439}.Debug|Mixed Platforms.ActiveCfg = Debug|x86
|
||||
{4012902C-54C6-473C-BA48-1087963CF439}.Debug|Mixed Platforms.Build.0 = Debug|x86
|
||||
{4012902C-54C6-473C-BA48-1087963CF439}.Debug|x86.ActiveCfg = Debug|x86
|
||||
{4012902C-54C6-473C-BA48-1087963CF439}.Debug|x86.Build.0 = Debug|x86
|
||||
{4012902C-54C6-473C-BA48-1087963CF439}.Release|Any CPU.ActiveCfg = Release|x86
|
||||
{4012902C-54C6-473C-BA48-1087963CF439}.Release|Mixed Platforms.ActiveCfg = Release|x86
|
||||
{4012902C-54C6-473C-BA48-1087963CF439}.Release|Mixed Platforms.Build.0 = Release|x86
|
||||
{4012902C-54C6-473C-BA48-1087963CF439}.Release|x86.ActiveCfg = Release|x86
|
||||
{4012902C-54C6-473C-BA48-1087963CF439}.Release|x86.Build.0 = Release|x86
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -1,194 +0,0 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProductVersion>9.0.21022</ProductVersion>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<ProjectGuid>{77739535-7FAC-4487-887F-FEBA197E7572}</ProjectGuid>
|
||||
<OutputType>Library</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>DistributedDataCollection</RootNamespace>
|
||||
<AssemblyName>DistributedDataCollection</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<StartupObject>
|
||||
</StartupObject>
|
||||
<FileUpgradeFlags>
|
||||
</FileUpgradeFlags>
|
||||
<UpgradeBackupLocation>
|
||||
</UpgradeBackupLocation>
|
||||
<OldToolsVersion>3.5</OldToolsVersion>
|
||||
<PublishUrl>publish\</PublishUrl>
|
||||
<Install>true</Install>
|
||||
<InstallFrom>Disk</InstallFrom>
|
||||
<UpdateEnabled>false</UpdateEnabled>
|
||||
<UpdateMode>Foreground</UpdateMode>
|
||||
<UpdateInterval>7</UpdateInterval>
|
||||
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
|
||||
<UpdatePeriodically>false</UpdatePeriodically>
|
||||
<UpdateRequired>false</UpdateRequired>
|
||||
<MapFileExtensions>true</MapFileExtensions>
|
||||
<ApplicationRevision>0</ApplicationRevision>
|
||||
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
|
||||
<IsWebBootstrapper>false</IsWebBootstrapper>
|
||||
<UseApplicationTrust>false</UseApplicationTrust>
|
||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||
<TargetFrameworkProfile />
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<DocumentationFile>bin\Debug\DistributedDataCollection.XML</DocumentationFile>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<Prefer32Bit>false</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<Prefer32Bit>false</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<DocumentationFile>bin\Debug\DistributedDataCollection.XML</DocumentationFile>
|
||||
<DebugType>full</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
|
||||
<OutputPath>bin\x64\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<Optimize>true</Optimize>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hpc.Scheduler, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\Dependencies\Microsoft.Hpc.Scheduler.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hpc.Scheduler.Properties, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\Dependencies\Microsoft.Hpc.Scheduler.Properties.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.0-beta023\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=1.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.0-beta023\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json">
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core">
|
||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data.Services.Client" />
|
||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq">
|
||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data.DataSetExtensions">
|
||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="ClusterAccess.cs" />
|
||||
<Compile Include="ClusterConfiguration.cs" />
|
||||
<Compile Include="ClusterStatus.cs" />
|
||||
<Compile Include="Cosmos.cs" />
|
||||
<Compile Include="jobinfo.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
<Compile Include="Storage.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Tools\Tools.csproj">
|
||||
<Project>{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}</Project>
|
||||
<Name>Tools</Name>
|
||||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
|
||||
<Install>false</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.5 SP1</ProductName>
|
||||
<Install>true</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>Windows Installer 3.1</ProductName>
|
||||
<Install>true</Install>
|
||||
</BootstrapperPackage>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="packages.config" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.0-beta023\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.0-beta023\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.0-beta023" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.1" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="3.1.0.0" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<package xmlns="http://schemas.microsoft.com/packaging/2013/01/nuspec.xsd">
|
||||
<metadata minClientVersion="2.5">
|
||||
<id>Microsoft.Research.Calypso</id>
|
||||
<version>0.7.0-beta024</version>
|
||||
<title>Calypso Job Browser for DryadLINQ</title>
|
||||
<authors>msrsvc</authors>
|
||||
<owners>msrsvc,Microsoft</owners>
|
||||
<licenseUrl>http://www.apache.org/licenses/LICENSE-2.0</licenseUrl>
|
||||
<projectUrl>http://research.microsoft.com/en-us/um/siliconvalley/projects/BigDataDev/</projectUrl>
|
||||
<requireLicenseAcceptance>true</requireLicenseAcceptance>
|
||||
<description>The DryadLINQ job browser is a graphical user interface which provides a unified view of a large part of the distributed state of a DryadLINQ job. In this document we provide a brief overview of the main capabilities of the DryadLINQ job browser.</description>
|
||||
<copyright>© Microsoft Corporation. All rights reserved.</copyright>
|
||||
<language>en-US</language>
|
||||
<tags>DryadLINQ Dryad Azure</tags>
|
||||
<dependencies>
|
||||
<group targetFramework=".NETFramework4.5">
|
||||
<dependency id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" />
|
||||
<dependency id="Microsoft.Research.Peloponnese" version="0.7.0-beta024" />
|
||||
</group>
|
||||
</dependencies>
|
||||
<frameworkAssemblies>
|
||||
<frameworkAssembly assemblyName="System" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Core" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Data" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Data.DataSetExtensions" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Data.Linq" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Deployment" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Drawing" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Runtime.Serialization" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.ServiceModel" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Windows.Forms" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Xml" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Xml.Linq" targetFramework=".NETFramework4.5" />
|
||||
</frameworkAssemblies>
|
||||
</metadata>
|
||||
<files>
|
||||
<file src="Calypso\bin\x64\Release\DistributedDataCollection.dll" target="content\DistributedDataCollection.dll" />
|
||||
<file src="Calypso\bin\x64\Release\DistributedDataCollection.pdb" target="content\DistributedDataCollection.pdb" />
|
||||
|
||||
</files>
|
||||
</package>
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
Currently the source of Calypso is generated by Deploy.exe, which
|
||||
strips out some unnecessary code. The master source is in some other
|
||||
repository.
|
||||
|
||||
|
||||
|
|
@ -1,120 +0,0 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProductVersion>9.0.30729</ProductVersion>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<ProjectGuid>{20B91AAF-AAD4-47DF-9F1D-494DE6E066F9}</ProjectGuid>
|
||||
<OutputType>Library</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>Microsoft.Research.Artemis.Tools</RootNamespace>
|
||||
<AssemblyName>Microsoft.Research.Calypso.Tools</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<FileUpgradeFlags>
|
||||
</FileUpgradeFlags>
|
||||
<OldToolsVersion>3.5</OldToolsVersion>
|
||||
<UpgradeBackupLocation />
|
||||
<PublishUrl>publish\</PublishUrl>
|
||||
<Install>true</Install>
|
||||
<InstallFrom>Disk</InstallFrom>
|
||||
<UpdateEnabled>false</UpdateEnabled>
|
||||
<UpdateMode>Foreground</UpdateMode>
|
||||
<UpdateInterval>7</UpdateInterval>
|
||||
<UpdateIntervalUnits>Days</UpdateIntervalUnits>
|
||||
<UpdatePeriodically>false</UpdatePeriodically>
|
||||
<UpdateRequired>false</UpdateRequired>
|
||||
<MapFileExtensions>true</MapFileExtensions>
|
||||
<ApplicationRevision>0</ApplicationRevision>
|
||||
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
|
||||
<IsWebBootstrapper>false</IsWebBootstrapper>
|
||||
<UseApplicationTrust>false</UseApplicationTrust>
|
||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||
<TargetFrameworkProfile />
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<DocumentationFile>bin\Debug\Microsoft.Research.Calypso.Tools.xml</DocumentationFile>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<Prefer32Bit>false</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<Prefer32Bit>false</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<DocumentationFile>bin\Debug\Microsoft.Hpc.Daphe.Tools.xml</DocumentationFile>
|
||||
<DebugType>full</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
|
||||
<OutputPath>bin\x64\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<Optimize>true</Optimize>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core">
|
||||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Drawing" />
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Windows.Forms" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="DrawingSurface.cs" />
|
||||
<Compile Include="graphlayout.cs" />
|
||||
<Compile Include="partitionedTables.cs" />
|
||||
<Compile Include="tools.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Client.3.5">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.5 SP1 Client Profile</ProductName>
|
||||
<Install>false</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Net.Framework.3.5.SP1">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>.NET Framework 3.5 SP1</ProductName>
|
||||
<Install>true</Install>
|
||||
</BootstrapperPackage>
|
||||
<BootstrapperPackage Include="Microsoft.Windows.Installer.3.1">
|
||||
<Visible>False</Visible>
|
||||
<ProductName>Windows Installer 3.1</ProductName>
|
||||
<Install>true</Install>
|
||||
</BootstrapperPackage>
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
Binary file not shown.
|
|
@ -1,27 +0,0 @@
|
|||
<?xml version="1.0"?>
|
||||
<package xmlns="http://schemas.microsoft.com/packaging/2013/01/nuspec.xsd">
|
||||
<metadata minClientVersion="2.5">
|
||||
<id>Microsoft.Research.DryadLINQ.MSAGL</id>
|
||||
<version>3.0.0.1</version>
|
||||
<title>MSAGL Binary Package solely for use with DryadLINQ</title>
|
||||
<authors>msrsvc</authors>
|
||||
<owners>msrsvc</owners>
|
||||
<licenseUrl>http://research.microsoft.com/en-us/um/siliconvalley/projects/BigDataDev/MSAGL_License.txt</licenseUrl>
|
||||
<projectUrl>http://research.microsoft.com/en-us/projects/msagl/</projectUrl>
|
||||
<requireLicenseAcceptance>true</requireLicenseAcceptance>
|
||||
<description>This is a binary package of the Microsoft Automatic Graph Layout engine solely for use with DryadLINQ. Please see the linked license for details.</description>
|
||||
<copyright>© Microsoft Corporation. All rights reserved.</copyright>
|
||||
<language>en-US</language>
|
||||
<frameworkAssemblies>
|
||||
<frameworkAssembly assemblyName="System" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Core" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Data.DataSetExtensions" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Data.Linq" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Drawing" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Runtime.Serialization" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.ServiceModel" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Xml" targetFramework=".NETFramework4.5" />
|
||||
<frameworkAssembly assemblyName="System.Xml.Linq" targetFramework=".NETFramework4.5" />
|
||||
</frameworkAssemblies>
|
||||
</metadata>
|
||||
</package>
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,6 +1,7 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
|
|
@ -9,10 +10,11 @@
|
|||
<OutputType>Library</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>Microsoft.Research.Dryad.ClusterInterface</RootNamespace>
|
||||
<AssemblyName>DryadHttpClusterInterface</AssemblyName>
|
||||
<AssemblyName>Microsoft.Research.Dryad.HttpClusterInterface</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<TargetFrameworkProfile />
|
||||
<NuGetPackageImportStamp>0025dfe5</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
|
|
@ -33,60 +35,12 @@
|
|||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Data.Services.Client" />
|
||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
|
|
@ -95,24 +49,34 @@
|
|||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\SharedAssemblyInfo.cs">
|
||||
<Link>Properties\SharedAssemblyInfo.cs</Link>
|
||||
</Compile>
|
||||
<Compile Include="Interfaces.cs" />
|
||||
<Compile Include="HttpScheduler.cs" />
|
||||
<Compile Include="Process.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="app.config" />
|
||||
<None Include="packages.config" />
|
||||
<None Include="app.config">
|
||||
<SubType>Designer</SubType>
|
||||
</None>
|
||||
<None Include="packages.config">
|
||||
<SubType>Designer</SubType>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
|
|
|
|||
|
|
@ -90,10 +90,10 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
return fileServer.Uri.AbsoluteUri;
|
||||
}
|
||||
|
||||
public IProcess NewProcess(IProcessWatcher watcher, string commandLine)
|
||||
public IProcess NewProcess(IProcessWatcher watcher, string commandLine, string commandLineArguments)
|
||||
{
|
||||
ISchedulerProcess process = scheduler.NewProcess();
|
||||
return new Process(process, watcher, commandLine, logger);
|
||||
return new Process(process, watcher, commandLine, commandLineArguments, logger);
|
||||
}
|
||||
|
||||
public void ScheduleProcess(IProcess ip, List<Affinity> affinities)
|
||||
|
|
|
|||
|
|
@ -368,8 +368,9 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
/// the application on the process' lifecycle</param>
|
||||
/// <param name="commandLine">the command line to use to start the process on the remote
|
||||
/// computer</param>
|
||||
/// <param name="commandLineArguments">arguments to provide to the remote process</param>
|
||||
/// <returns>a handle to the new process</returns>
|
||||
IProcess NewProcess(IProcessWatcher watcher, string commandLine);
|
||||
IProcess NewProcess(IProcessWatcher watcher, string commandLine, string commandLineArguments);
|
||||
|
||||
/// <summary>
|
||||
/// request that a Process object, return from NewProcess, be scheduled according
|
||||
|
|
|
|||
|
|
@ -56,11 +56,6 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
/// </summary>
|
||||
private State state;
|
||||
|
||||
/// <summary>
|
||||
/// string used to start the remote process
|
||||
/// </summary>
|
||||
private string commandLine;
|
||||
|
||||
/// <summary>
|
||||
/// this is the handle that the scheduler supplies that is used to refer
|
||||
/// to the process
|
||||
|
|
@ -114,11 +109,12 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
/// <summary>
|
||||
/// construct a new object to represent the lifecycle of a process being scheduled
|
||||
/// </summary>
|
||||
public Process(ISchedulerProcess p, IProcessWatcher w, string cmd, ILogger l)
|
||||
public Process(ISchedulerProcess p, IProcessWatcher w, string cmd, string cmdLineArgs, ILogger l)
|
||||
{
|
||||
schedulerProcess = p;
|
||||
state = State.Initializing;
|
||||
commandLine = cmd;
|
||||
CommandLine = cmd;
|
||||
CommandLineArguments = cmdLineArgs;
|
||||
watcher = w;
|
||||
cancelTask = new TaskCompletionSource<bool>();
|
||||
directory = null;
|
||||
|
|
@ -137,7 +133,12 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
/// <summary>
|
||||
/// the string used to start the remote process
|
||||
/// </summary>
|
||||
public string CommandLine { get { return commandLine; } }
|
||||
public string CommandLine { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// arguments provided when starting the remote process
|
||||
/// </summary>
|
||||
public string CommandLineArguments { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// the local directory of the process at the daemon's host computer
|
||||
|
|
@ -199,8 +200,12 @@ namespace Microsoft.Research.Dryad.ClusterInterface
|
|||
|
||||
ToMatched(computer, DateTime.Now.ToFileTimeUtc());
|
||||
|
||||
StringBuilder payload = new StringBuilder();
|
||||
payload.AppendLine(CommandLine);
|
||||
payload.AppendLine(CommandLineArguments);
|
||||
|
||||
Task<string> bail = interrupt.ContinueWith((t) => "");
|
||||
Task<string> upload = PostRequest(computer, directory + "?op=create", Encoding.UTF8.GetBytes(commandLine));
|
||||
Task<string> upload = PostRequest(computer, directory + "?op=create", Encoding.UTF8.GetBytes(payload.ToString()));
|
||||
Task<string> completed = await Task.WhenAny(bail, upload);
|
||||
|
||||
if (completed == bail)
|
||||
|
|
|
|||
|
|
@ -24,32 +24,7 @@ using System.Runtime.InteropServices;
|
|||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("DryadHttpClusterInterface")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("Microsoft IT")]
|
||||
[assembly: AssemblyProduct("DryadHttpClusterInterface")]
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft IT 2013")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.HttpClusterInterface")]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("9d262d5f-f6c5-4f9c-a1cc-7568cb399bf9")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("1.0.0.0")]
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||
|
|
|
|||
|
|
@ -4,24 +4,40 @@
|
|||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
@ -1,13 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
|
||||
</packages>
|
||||
30
Dryad.sln
30
Dryad.sln
|
|
@ -1,6 +1,8 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 2012
|
||||
# Visual Studio 2013
|
||||
VisualStudioVersion = 12.0.30723.0
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DryadVertex", "DryadVertex", "{99F5E7FE-ADD4-4E9D-BA90-5D3D3409BEB3}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "VertexHost", "VertexHost", "{37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D}"
|
||||
|
|
@ -27,7 +29,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dprocess", "DryadVertex\Ver
|
|||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ManagedWrapperVertex", "DryadVertex\VertexHost\vertex\managedwrappervertex\ManagedWrapperVertex.vcxproj", "{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}"
|
||||
EndProject
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHost", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}"
|
||||
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHostBody", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}"
|
||||
ProjectSection(ProjectDependencies) = postProject
|
||||
{AA529122-F51C-48D7-A8C1-C0B24F570885} = {AA529122-F51C-48D7-A8C1-C0B24F570885}
|
||||
{482E0741-E244-4974-97D4-3A7167581E91} = {482E0741-E244-4974-97D4-3A7167581E91}
|
||||
|
|
@ -78,7 +80,18 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalScheduler", "LocalSche
|
|||
{A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6} = {A85853FB-AAAB-4F1F-AAAB-D051E8F0B2E6}
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadLinqTests", "DryadLinqTests\DryadLinqTests.csproj", "{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}"
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VertexHostMain", "DryadVertex\VertexHost\vertex\VertexHostMain\VertexHostMain.csproj", "{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "NuSpec", "NuSpec", "{35121ABA-5FD2-4E1D-9C05-033D78D824F3}"
|
||||
ProjectSection(SolutionItems) = preProject
|
||||
Microsoft.Research.Dryad.nuspec = Microsoft.Research.Dryad.nuspec
|
||||
Microsoft.Research.Dryad.targets = Microsoft.Research.Dryad.targets
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Misc", "Misc", "{D361F025-2F7F-4C1A-8FBC-4AD826CC656D}"
|
||||
ProjectSection(SolutionItems) = preProject
|
||||
SharedAssemblyInfo.cs = SharedAssemblyInfo.cs
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
|
|
@ -146,10 +159,10 @@ Global
|
|||
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Debug|x64.Build.0 = Debug|x64
|
||||
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.ActiveCfg = Release|x64
|
||||
{FA798DE2-A442-4D2B-B31A-FA9274308290}.Release|x64.Build.0 = Release|x64
|
||||
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Debug|x64.Build.0 = Debug|x64
|
||||
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Release|x64.ActiveCfg = Release|x64
|
||||
{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}.Release|x64.Build.0 = Release|x64
|
||||
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Debug|x64.Build.0 = Debug|x64
|
||||
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Release|x64.ActiveCfg = Release|x64
|
||||
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}.Release|x64.Build.0 = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
@ -162,11 +175,12 @@ Global
|
|||
{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
|
||||
{57663B94-E11B-431E-BE4B-E2C61112DEC5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
|
||||
{AA529122-F51C-48D7-A8C1-C0B24F570885} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
|
||||
{A0033286-9C5F-4113-BAA5-58A1274F95C5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
|
||||
{BDEDD3BB-C7E2-498F-A212-F99786C8E23C} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
{AB9EA66C-5811-49A7-B002-24203AEB9083} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
{3EE0920C-0607-4569-9EC3-5C12BB6EF244} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
{89C5654B-02E4-478D-A7E6-50D79F638B4F} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
{A0033286-9C5F-4113-BAA5-58A1274F95C5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
|
||||
{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352} = {DC277807-506B-4F7C-BECD-345079B91044}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
|
|
@ -10,7 +13,7 @@
|
|||
<OutputType>Exe</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>Microsoft.Research.Dryad.GraphManager</RootNamespace>
|
||||
<AssemblyName>DryadLinqGraphManager</AssemblyName>
|
||||
<AssemblyName>Microsoft.Research.Dryad.GraphManager</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<FileUpgradeFlags>
|
||||
|
|
@ -33,6 +36,7 @@
|
|||
<ApplicationVersion>1.0.0.%2a</ApplicationVersion>
|
||||
<UseApplicationTrust>false</UseApplicationTrust>
|
||||
<BootstrapperEnabled>true</BootstrapperEnabled>
|
||||
<NuGetPackageImportStamp>5c14de70</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
|
|
@ -53,52 +57,28 @@
|
|||
<CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core">
|
||||
|
|
@ -108,9 +88,12 @@
|
|||
<RequiredTargetFramework>3.5</RequiredTargetFramework>
|
||||
</Reference>
|
||||
<Reference Include="System.Data.Services.Client" />
|
||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
<HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq">
|
||||
|
|
@ -123,6 +106,9 @@
|
|||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\SharedAssemblyInfo.cs">
|
||||
<Link>Properties\SharedAssemblyInfo.cs</Link>
|
||||
</Compile>
|
||||
<Compile Include="DryadLinqApplication.cs" />
|
||||
<Compile Include="GraphBuilder.cs" />
|
||||
<Compile Include="LinqToDryadException.cs" />
|
||||
|
|
@ -168,10 +154,19 @@
|
|||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
err = input.Open(app.GetUniverse(), new Uri(info.sources[0]).AbsolutePath);
|
||||
if (!SUCCEEDED(err))
|
||||
{
|
||||
string msg = String.Format("Could not read DSC input file {0}", info.sources[0]);
|
||||
string msg = String.Format("Could not read Partitioned file input {0}", info.sources[0]);
|
||||
throw new LinqToDryadException(msg, err);
|
||||
}
|
||||
|
||||
|
|
@ -127,7 +127,9 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
|
||||
DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");
|
||||
|
||||
err = input.Open(app.GetUniverse(), info.sources[0]);
|
||||
Uri srcUri = new Uri(info.sources[0]);
|
||||
|
||||
err = input.Open(app.GetUniverse(), srcUri.GetLeftPart(UriPartial.Path), info.recordType);
|
||||
if (!SUCCEEDED(err))
|
||||
{
|
||||
string msg = String.Format("Could not read HDFS input fileset {0}: {1}", info.sources[0], input.GetError());
|
||||
|
|
@ -182,10 +184,16 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
if ( info.ioType == VertexInfo.IOType.PARTITIONEDFILE )
|
||||
{
|
||||
DrPartitionOutputStream output = new DrPartitionOutputStream();
|
||||
int err = output.Open(new Uri(info.sources[0]).AbsolutePath, info.partitionUncPath);
|
||||
Uri sourceUri = new Uri(info.sources[0]);
|
||||
string sourcePath = sourceUri.AbsolutePath;
|
||||
if (!String.IsNullOrEmpty(sourceUri.Host))
|
||||
{
|
||||
sourcePath = @"\\" + sourceUri.Host + sourcePath;
|
||||
}
|
||||
int err = output.Open(sourcePath, info.partitionUncPath);
|
||||
if (!SUCCEEDED(err))
|
||||
{
|
||||
string msg = String.Format("Could not open DSC output fileset {0}", info.sources[0]);
|
||||
string msg = String.Format("Could not open output fileset {0}", sourcePath);
|
||||
throw new LinqToDryadException(msg, err);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -73,12 +73,12 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
|
||||
public static Uri AddDfsJobDirectoryFromArgs(string arg)
|
||||
{
|
||||
string jobId = Microsoft.Research.Peloponnese.Storage.AzureUtils.ApplicationIdFromEnvironment();
|
||||
string jobDirectoryBase = Microsoft.Research.Peloponnese.Storage.AzureUtils.CmdLineDecode(arg.Substring(arg.IndexOf('=') + 1));
|
||||
string jobId = Microsoft.Research.Peloponnese.Yarn.Utils.ApplicationIdFromEnvironment();
|
||||
string jobDirectoryBase = Microsoft.Research.Peloponnese.Utils.CmdLineDecode(arg.Substring(arg.IndexOf('=') + 1));
|
||||
string jobDirectorySpecific = jobDirectoryBase.Replace("_JOBID_", jobId);
|
||||
|
||||
UriBuilder logLocation = new UriBuilder(jobDirectorySpecific);
|
||||
logLocation.Path = logLocation.Path + "calypso.log";
|
||||
logLocation.Path = logLocation.Path.TrimEnd('/') + "/calypso.log";
|
||||
|
||||
DebugHelper.AddReporter(new DrCalypsoReporter(logLocation.Uri.AbsoluteUri));
|
||||
|
||||
|
|
@ -94,7 +94,7 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
{
|
||||
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
|
||||
{
|
||||
string dfsPath = dfsDirectory.AbsolutePath + Path.GetFileName(localPath);
|
||||
string dfsPath = dfsDirectory.AbsolutePath.TrimEnd('/') + "/" + Path.GetFileName(localPath);
|
||||
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
|
||||
hdfs.UploadAll(localPath, dfsPath);
|
||||
}
|
||||
|
|
@ -102,11 +102,12 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
else if (dfsDirectory.Scheme == "azureblob")
|
||||
{
|
||||
string account, key, container, blob;
|
||||
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
|
||||
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container);
|
||||
string dfsPath = blob + Path.GetFileName(localPath);
|
||||
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
|
||||
azure.PutDfsFile(localPath, dfsPath);
|
||||
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
|
||||
var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
|
||||
string dfsPath = blob.TrimEnd('/') + "/" + Path.GetFileName(localPath);
|
||||
Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
|
||||
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsUri.AbsoluteUri);
|
||||
azure.PutDfsFile(dfsUri, localPath);
|
||||
}
|
||||
else if (dfsDirectory.Scheme == "file")
|
||||
{
|
||||
|
|
@ -138,10 +139,11 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
else if (dfsDirectory.Scheme == "azureblob")
|
||||
{
|
||||
string account, key, container, blob;
|
||||
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
|
||||
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container);
|
||||
Microsoft.Research.Peloponnese.Azure.Utils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
|
||||
var azure = new Microsoft.Research.Peloponnese.Azure.AzureDfsClient(account, key, container);
|
||||
string dfsPath = blob + dfsName;
|
||||
azure.PutDfsFile(payloadBytes, dfsPath);
|
||||
Uri dfsUri = Microsoft.Research.Peloponnese.Azure.Utils.ToAzureUri(account, container, dfsPath, null, key);
|
||||
azure.PutDfsFile(dfsUri, payloadBytes);
|
||||
}
|
||||
else if (dfsDirectory.Scheme == "file")
|
||||
{
|
||||
|
|
@ -207,7 +209,39 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
logDir = logDir.Split(',').First().Trim();
|
||||
DrLogging.Initialize(Path.Combine(logDir, "graphmanager"), false);
|
||||
}
|
||||
|
||||
string logLevel = Environment.GetEnvironmentVariable("DRYAD_LOGGING_LEVEL");
|
||||
if (logLevel == null)
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Debug);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (logLevel == "OFF")
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Off);
|
||||
}
|
||||
else if (logLevel == "CRITICAL")
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Assert);
|
||||
}
|
||||
else if (logLevel == "ERROR")
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Error);
|
||||
}
|
||||
else if (logLevel == "WARN")
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Warning);
|
||||
}
|
||||
else if (logLevel == "INFO")
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Info);
|
||||
}
|
||||
else
|
||||
{
|
||||
DrLogging.SetLoggingLevel(DrLogTypeManaged.Debug);
|
||||
}
|
||||
}
|
||||
|
||||
// Report start time to Artemis - must come after
|
||||
// DrLogging is initialized so stdout is redirected
|
||||
|
|
|
|||
|
|
@ -28,6 +28,13 @@ using Microsoft.Research.Dryad;
|
|||
|
||||
namespace Microsoft.Research.Dryad.GraphManager
|
||||
{
|
||||
internal class ConfigDependency : Microsoft.Research.Peloponnese.Shared.AssemblyDependencyAttribute
|
||||
{
|
||||
public ConfigDependency() : base("Microsoft.Research.Dryad.GraphManager.exe.config", false)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
class Program
|
||||
{
|
||||
static Uri dfsDirectory;
|
||||
|
|
|
|||
|
|
@ -22,33 +22,12 @@ using System.Reflection;
|
|||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
[assembly: Microsoft.Research.Dryad.GraphManager.ConfigDependency()]
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("DryadLinqGraphManager")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyProduct("DryadLinqGraphManager")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.GraphManager")]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("55c7f4b4-02ab-4309-b276-3c9ab5a927e0")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("1.0.0.0")]
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||
|
|
|
|||
|
|
@ -237,6 +237,12 @@ namespace Microsoft.Research.Dryad.GraphManager
|
|||
vertex.info = new VertexInfo();
|
||||
vertex.info.ioType = GetIoType(ioType);
|
||||
|
||||
XmlNode recordType = storageSet.SelectSingleNode("RecordType");
|
||||
if (recordType != null)
|
||||
{
|
||||
vertex.info.recordType = recordType.InnerXml;
|
||||
}
|
||||
|
||||
XmlNodeList storageUris = storageSet.SelectNodes("SourceURI");
|
||||
vertex.info.sources = new string[storageUris.Count];
|
||||
for (int indexStorageUri=0; indexStorageUri<storageUris.Count; indexStorageUri++)
|
||||
|
|
|
|||
|
|
@ -7,24 +7,40 @@
|
|||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.2" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -5,6 +5,14 @@
|
|||
</startup>
|
||||
<runtime>
|
||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
|
|
@ -18,12 +26,20 @@
|
|||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,469 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class ApplyAndForkTests
|
||||
{
|
||||
public static void Run(DryadLinqContext context, string matchPattern)
|
||||
{
|
||||
TestLog.Message(" **********************");
|
||||
TestLog.Message(" ApplyAndForkTests ");
|
||||
TestLog.Message(" **********************");
|
||||
|
||||
var tests = new Dictionary<string, Action>()
|
||||
{
|
||||
{"NonHomomorphicUnaryApply", () => NonHomomorphicUnaryApply(context) },
|
||||
{"HomomorphicUnaryApply", () => HomomorphicUnaryApply(context) },
|
||||
{"NonHomomorphicBinaryApply", () => NonHomomorphicBinaryApply(context) },
|
||||
{"LeftHomomorphicBinaryApply", () => LeftHomomorphicBinaryApply(context) },
|
||||
{"FullHomomorphicBinaryApply_DifferentDataSets", () => FullHomomorphicBinaryApply_DifferentDataSets(context) },
|
||||
{"FullHomomorphicBinaryApply_IdenticalDataSets", () => FullHomomorphicBinaryApply_IdenticalDataSets(context) },
|
||||
{"Aggregate_WithCombiner", () => Aggregate_WithCombiner(context) },
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
{
|
||||
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
|
||||
{
|
||||
test.Value.Invoke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static IEnumerable<int> NonHomomorphic_Unary_Func(IEnumerable<int> input)
|
||||
{
|
||||
return input;
|
||||
}
|
||||
|
||||
// [DistributiveOverConcat]
|
||||
public static IEnumerable<int> Homomorphic_Unary_Func(IEnumerable<int> input)
|
||||
{
|
||||
return input;
|
||||
}
|
||||
|
||||
public static IEnumerable<int> NonHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
|
||||
{
|
||||
return left;
|
||||
}
|
||||
|
||||
// [LeftDistributiveOverConcat]
|
||||
public static IEnumerable<int> LeftHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
|
||||
{
|
||||
return left;
|
||||
}
|
||||
|
||||
// Note: an apply function must only consume each enumerable once, and it must produce an enumerable
|
||||
// So for a simple pass-through function that does a little work, we must enumerate only once.
|
||||
// Else we get the error: "An HpcLinq channel can't be read more than once."
|
||||
// [DistributiveOverConcat]
|
||||
public static IEnumerable<int> FullHomomorphic_Binary_Func(IEnumerable<int> left, IEnumerable<int> right)
|
||||
{
|
||||
long cLeft = 0;
|
||||
foreach (int x in left)
|
||||
{
|
||||
cLeft++;
|
||||
yield return x;
|
||||
}
|
||||
|
||||
long cRight = 0;
|
||||
foreach (int x in right)
|
||||
{
|
||||
cRight++;
|
||||
yield return x;
|
||||
}
|
||||
|
||||
if (cLeft == 0)
|
||||
throw new Exception("a node received empty left-data");
|
||||
|
||||
if (cRight == 0)
|
||||
throw new Exception("a node received empty right-data");
|
||||
}
|
||||
|
||||
public static bool NonHomomorphicUnaryApply(DryadLinqContext context)
|
||||
{
|
||||
string testName = "NonHomomorphicUnaryApply";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/NonHomomorphicUnaryApply";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(x => NonHomomorphic_Unary_Func(x));
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool HomomorphicUnaryApply(DryadLinqContext context)
|
||||
{
|
||||
string testName = "HomomorphicUnaryApply";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/HomomorphicUnaryApply";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool NonHomomorphicBinaryApply(DryadLinqContext context)
|
||||
{
|
||||
string testName = "NonHomomorphicBinaryApply";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/NonHomomorphicBinaryApply";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.Apply(pt1, (x, y) => NonHomomorphic_Binary_Func(x, y));
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool LeftHomomorphicBinaryApply(DryadLinqContext context)
|
||||
{
|
||||
string testName = "LeftHomomorphicBinaryApply";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/LeftHomomorphicBinaryApply";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => LeftHomomorphic_Binary_Func(x, y), true);
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool FullHomomorphicBinaryApply_DifferentDataSets(DryadLinqContext context)
|
||||
{
|
||||
string testName = "FullHomomorphicBinaryApply_DifferentDataSets";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/FullHomomorphicBinaryApply_DifferentDataSets";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool FullHomomorphicBinaryApply_IdenticalDataSets(DryadLinqContext context)
|
||||
{
|
||||
string testName = "FullHomomorphicBinaryApply_IdenticalDataSets";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/FullHomomorphicBinaryApply_2";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
|
||||
var jobInfo = q1.ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= Validate.outFileExists(outFile);
|
||||
result[0] = q1;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> q1 = pt1.ApplyPerPartition(pt1, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
|
||||
result[1] = q1;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
[Associative(typeof(AssociativeRecursive1))]
|
||||
public static string IntToStringCSVAggregator(string agg, int next)
|
||||
{
|
||||
return agg + "," + next.ToString();
|
||||
}
|
||||
public class AssociativeRecursive1 : IAssociative<string>
|
||||
{
|
||||
public string Seed()
|
||||
{
|
||||
return "";
|
||||
}
|
||||
public string RecursiveAccumulate(string first, string second)
|
||||
{
|
||||
return first + second;
|
||||
}
|
||||
}
|
||||
|
||||
public static bool Aggregate_WithCombiner(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Aggregate_WithCombiner";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
|
||||
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,53 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<configuration>
|
||||
<startup>
|
||||
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
|
||||
</startup>
|
||||
<runtime>
|
||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="FSharp.Core" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
|
||||
<bindingRedirect oldVersion="4.0.0.0" newVersion="4.3.0.0" />
|
||||
<bindingRedirect oldVersion="2.3.5.0" newVersion="4.3.0.0" />
|
||||
<bindingRedirect oldVersion="2.0.0.0" newVersion="4.3.0.0" />
|
||||
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.0.1.0" newVersion="4.0.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.1.1.8" newVersion="1.1.1.8" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,219 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<SchemaVersion>2.0</SchemaVersion>
|
||||
<ProjectGuid>d0153361-1b5d-4715-8ee3-aa824f89cbc3</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<RootNamespace>DryadLinqFSharpTests</RootNamespace>
|
||||
<AssemblyName>DryadLinqFSharpTests</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<Name>DryadLinqFSharpTests</Name>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<Tailcalls>false</Tailcalls>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<WarningLevel>3</WarningLevel>
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DocumentationFile>bin\Debug\DryadLinqFSharpTests.XML</DocumentationFile>
|
||||
<Prefer32Bit>true</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<Tailcalls>true</Tailcalls>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<WarningLevel>3</WarningLevel>
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DocumentationFile>bin\Release\DryadLinqFSharpTests.XML</DocumentationFile>
|
||||
<Prefer32Bit>true</Prefer32Bit>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm">
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData">
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client">
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl">
|
||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl.Drawing">
|
||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl.GraphViewerGdi">
|
||||
<HintPath>..\packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.DryadLinq">
|
||||
<HintPath>..\packages\Microsoft.Research.Dryad.0.1.4-beta001\lib\net45\Microsoft.Research.DryadLinq.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge">
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils">
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks">
|
||||
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks.Extensions">
|
||||
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop">
|
||||
<HintPath>..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Common">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.Storage">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage">
|
||||
<HintPath>..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="mscorlib" />
|
||||
<Reference Include="FSharp.Core, Version=4.3.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json">
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
<Reference Include="System.Data.Linq" />
|
||||
<Reference Include="System.Drawing" />
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.Extensions">
|
||||
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="System.Net.Http.Primitives">
|
||||
<HintPath>..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Numerics" />
|
||||
<Reference Include="System.Runtime.Serialization" />
|
||||
<Reference Include="System.ServiceModel" />
|
||||
<Reference Include="System.ServiceModel.Discovery" />
|
||||
<Reference Include="System.Spatial">
|
||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
<Private>True</Private>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="App.config" />
|
||||
<Compile Include="Program.fs" />
|
||||
<None Include="packages.config" />
|
||||
</ItemGroup>
|
||||
<PropertyGroup>
|
||||
<MinimumVisualStudioVersion Condition="'$(MinimumVisualStudioVersion)' == ''">11</MinimumVisualStudioVersion>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<Tailcalls>false</Tailcalls>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<WarningLevel>3</WarningLevel>
|
||||
<DocumentationFile>bin\Debug\DryadLinqFSharpTests.XML</DocumentationFile>
|
||||
<Prefer32Bit>true</Prefer32Bit>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<Tailcalls>true</Tailcalls>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<WarningLevel>3</WarningLevel>
|
||||
<DocumentationFile>bin\Release\DryadLinqFSharpTests.XML</DocumentationFile>
|
||||
<Prefer32Bit>true</Prefer32Bit>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets" Condition=" Exists('$(MSBuildExtensionsPath32)\..\Microsoft SDKs\F#\3.0\Framework\v4.0\Microsoft.FSharp.Targets')" />
|
||||
<Import Project="..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
|
||||
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
|
||||
<Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
|
||||
</Target>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets" Condition="Exists('..\packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
// Learn more about F# at http://fsharp.net
|
||||
// See the 'F# Tutorial' project for more help.
|
||||
|
||||
open System
|
||||
open System.Collections.Generic
|
||||
open System.Linq
|
||||
open System.IO
|
||||
open Microsoft.Research.DryadLinq
|
||||
|
||||
let context = new DryadLinqContext(1)
|
||||
let inputUri = @"partfile:///d:/DryadLinqTemp/PartFiles/foo.pt"
|
||||
let input = context.FromStore<LineRecord>(inputUri)
|
||||
|
||||
type Pair = { word : string; count : int }
|
||||
|
||||
let Histogram (source : IQueryable<LineRecord>) (k : int) =
|
||||
let words = source.SelectMany(fun (x : LineRecord) -> x.Line.Split(' ') |> Seq.ofArray)
|
||||
let groups = words.GroupBy(fun x -> x)
|
||||
let counts = groups.Select(fun (x : IGrouping<string, string>) -> { word = x.Key; count = x.Count() })
|
||||
let ordered = counts.OrderByDescending(fun x -> x.count)
|
||||
ordered.Take(k)
|
||||
|
||||
let Histogram1 (source : IQueryable<LineRecord>) (k : int) =
|
||||
query {
|
||||
for ws in source do
|
||||
for w in ws.Line.Split(' ') do
|
||||
groupBy w into g
|
||||
select { word = g.Key; count = g.Count() } into winfo
|
||||
sortByDescending winfo.count
|
||||
take k
|
||||
}
|
||||
|
||||
let Histogram2 =
|
||||
query {
|
||||
for ws in input do
|
||||
for w in ws.Line.Split(' ') do
|
||||
groupBy w into g
|
||||
select { word = g.Key; count = g.Count() } into winfo
|
||||
sortByDescending winfo.count
|
||||
take 10
|
||||
}
|
||||
|
||||
for x in Histogram1 input 10 do System.Console.WriteLine(x.word + " : " + x.count.ToString())
|
||||
|
||||
//let foo = Histogram input 10
|
||||
//foo.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/xxx.pt").SubmitAndWait()
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
|
||||
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Dryad" version="0.1.4-beta001" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -1,11 +1,11 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE}</ProjectGuid>
|
||||
<ProjectGuid>{4EA15885-003F-4B4D-A7DB-C172C2F23161}</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>DryadLinqTests</RootNamespace>
|
||||
|
|
@ -30,6 +30,25 @@
|
|||
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
|
||||
<RestorePackages>true</RestorePackages>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<OutputPath>bin\x64\Debug\</OutputPath>
|
||||
|
|
@ -53,50 +72,110 @@
|
|||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl">
|
||||
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl.Drawing">
|
||||
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Msagl.GraphViewerGdi">
|
||||
<HintPath>packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.DryadLinq, Version=0.1.3.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Research.Dryad.0.1.4-beta001\lib\net45\Microsoft.Research.DryadLinq.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge">
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils">
|
||||
<HintPath>..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks.Extensions, Version=1.0.12.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Threading.Tasks.Extensions.Desktop, Version=1.0.168.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Common, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Common.NetFramework, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight">
|
||||
<HintPath>..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core">
|
||||
<HintPath>..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.WindowsAzure.Management, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json">
|
||||
<HintPath>..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.1.8, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.Storage, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.0.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Data.Services.Client" />
|
||||
<Reference Include="System.Data.Linq" />
|
||||
<Reference Include="System.Drawing" />
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.Extensions, Version=2.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Net.Http.Primitives, Version=4.2.22.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Runtime.Serialization" />
|
||||
<Reference Include="System.ServiceModel" />
|
||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
<HintPath>packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
|
|
@ -106,8 +185,20 @@
|
|||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="ApplyAndForkTests.cs" />
|
||||
<Compile Include="BasicAPITests.cs" />
|
||||
<Compile Include="GroupByReduceTests.cs" />
|
||||
<Compile Include="MiscBugFixTests.cs" />
|
||||
<Compile Include="Program.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
<Compile Include="RangePartitionAPICoverageTests.cs" />
|
||||
<Compile Include="SerializationTests.cs" />
|
||||
<Compile Include="SerializationTestTypes.cs" />
|
||||
<Compile Include="SimpleTests.cs" />
|
||||
<Compile Include="TestLog.cs" />
|
||||
<Compile Include="TypesInQueryTests.cs" />
|
||||
<Compile Include="Utils.cs" />
|
||||
<Compile Include="WordCount.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="App.config" />
|
||||
|
|
@ -137,14 +228,21 @@
|
|||
</BootstrapperPackage>
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<Import Project="packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets" Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" />
|
||||
<Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
|
||||
<Error Condition="!Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
|
||||
<Error Condition="Exists('packages\Microsoft.Bcl.Build.1.0.14\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
|
||||
</Target>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" Text="$([System.String]::Format('$(ErrorText)', 'packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('packages\Microsoft.Research.Peloponnese.0.7.2-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets" Condition="Exists('packages\Microsoft.Research.Dryad.0.1.4-beta001\build\Microsoft.Research.Dryad.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio 2012
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadLinqTests", "DryadLinqTests.csproj", "{4EA15885-003F-4B4D-A7DB-C172C2F23161}"
|
||||
EndProject
|
||||
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "DryadLinqFSharpTests", "DryadLinqFSharpTests\DryadLinqFSharpTests.fsproj", "{D0153361-1B5D-4715-8EE3-AA824F89CBC3}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Debug|Mixed Platforms = Debug|Mixed Platforms
|
||||
Debug|x64 = Debug|x64
|
||||
Release|Any CPU = Release|Any CPU
|
||||
Release|Mixed Platforms = Release|Mixed Platforms
|
||||
Release|x64 = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Debug|x64.Build.0 = Debug|x64
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Mixed Platforms.ActiveCfg = Release|x64
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|Mixed Platforms.Build.0 = Release|x64
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|x64.ActiveCfg = Release|x64
|
||||
{4EA15885-003F-4B4D-A7DB-C172C2F23161}.Release|x64.Build.0 = Release|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Debug|x64.Build.0 = Debug|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Mixed Platforms.ActiveCfg = Release|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|Mixed Platforms.Build.0 = Release|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|x64.ActiveCfg = Release|x64
|
||||
{D0153361-1B5D-4715-8EE3-AA824F89CBC3}.Release|x64.Build.0 = Release|x64
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,967 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public static class MiscBugFixTests
|
||||
{
|
||||
public static void Run(DryadLinqContext context, string matchPattern)
|
||||
{
|
||||
TestLog.Message(" **********************");
|
||||
TestLog.Message(" MiscBugFixTests ");
|
||||
TestLog.Message(" **********************");
|
||||
|
||||
var tests = new Dictionary<string, Action>()
|
||||
{
|
||||
{"Bug11447_GroupByWithComparer", () => Bug11447_GroupByWithComparer(context) },
|
||||
{"Bug12584_HashPartitionOutputCount", () => Bug12584_HashPartitionOutputCount(context) },
|
||||
{"Bug13108_DisableSequenceEquals", () => Bug13108_DisableSequenceEquals(context) },
|
||||
{"Bug13529_and_Bug13593_IndexedOperatorCompilation", () => Bug13529_and_Bug13593_IndexedOperatorCompilation(context) },
|
||||
{"Bug13130_ReverseOperator", () => Bug13130_ReverseOperator(context) },
|
||||
{"Bug13736_IndexedTakeWhile", () => Bug13736_IndexedTakeWhile(context) },
|
||||
{"Bug13534_HashPartitionNegIndexIsError", () => Bug13534_HashPartitionNegIndexIsError(context) },
|
||||
{"Bug13474_and_Bug13483_FromDscOnBadFileSet", () => Bug13474_and_Bug13483_FromDscOnBadFileSet(context) },
|
||||
{"Bug13637_EmptyFilesInFilesets", () => Bug13637_EmptyFilesInFilesets(context) },
|
||||
{"Bug13637_LocalDebugProducingZeroRecords", () => Bug13637_LocalDebugProducingZeroRecords(context) },
|
||||
{"Bug13245_FromDsc_Submit_throws", () => Bug13245_FromDsc_Submit_throws(context) },
|
||||
{"Bug13245_QueryUsingNonHpcLinqOperator", () => Bug13245_QueryUsingNonHpcLinqOperator(context) },
|
||||
{"Bug13302_ConfigEnvironmentCleanup", () => Bug13302_ConfigEnvironmentCleanup(context) },
|
||||
{"Bug13970_MismatchedDataTypes", () => Bug13970_MismatchedDataTypes(context) },
|
||||
{"Bug14010_AlreadyDisposedContext", () => Bug14010_AlreadyDisposedContext(context) },
|
||||
{"Bug14256_LeaseOnTempDscFileset", () => Bug14256_LeaseOnTempDscFileset(context) },
|
||||
{"Bug14189_OrderPreservation", () => Bug14189_OrderPreservation(context) },
|
||||
{"Bug14190_MergeJoin_DecreasingOrder", () => Bug14190_MergeJoin_DecreasingOrder(context) },
|
||||
{"Bug14192_MultiApplySubExpressionReuse", () => Bug14192_MultiApplySubExpressionReuse(context) },
|
||||
{"Bug14870_LongIndexTakeWhile", () => Bug14870_LongIndexTakeWhile(context) },
|
||||
{"Bug15159_NotOperatorForNullableBool", () => Bug15159_NotOperatorForNullableBool(context) },
|
||||
{"Bug15371_NoDataMembersForSerialization", () => Bug15371_NoDataMembersForSerialization(context) },
|
||||
{"Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes", () => Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(context) },
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
{
|
||||
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
|
||||
{
|
||||
test.Value.Invoke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static bool Bug11447_GroupByWithComparer(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug11447_GroupByWithComparer";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
|
||||
result[0] = a;
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
|
||||
result[1] = a;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug12584_HashPartitionOutputCount(DryadLinqContext context) // ToDo
|
||||
{
|
||||
string testName = "Bug12584_HashPartitionOutputCount";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/Bug12584_HashPartitionOutputCount";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var info = pt1.HashPartition(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
info.Wait();
|
||||
|
||||
//// partition verification are only valid for cluster execution..
|
||||
|
||||
//// check that nOutputPartitions == nInputPartitions.
|
||||
//// Note: this is today's behavior, but we don't strictly guarantee this and the rules may change.
|
||||
//var inFS = context.DscService.GetFileSet(DataGenerators.SIMPLE_FILESET_NAME);
|
||||
//var inPCount = inFS.GetFiles().Count();
|
||||
//var outFS = context.DscService.GetFileSet("DevUnitTest/Bug12584_out");
|
||||
//var outPCount = outFS.GetFiles().Count();
|
||||
//passed &= TestUtils.Assert(outPCount == inPCount, "Output nPartitions should be equal to input nPartitions.");
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13108_DisableSequenceEquals(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13108_DisableSequenceEquals";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
bool result = pt1.Select(x => x).SequenceEqual(pt2.Select(x => x));
|
||||
passed &= false; // NotSupportedException should have been thrown
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
}
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13529_and_Bug13593_IndexedOperatorCompilation(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13529_and_Bug13593_IndexedOperatorCompilation";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
result[0] = pt1.Select((x, i) => x)
|
||||
.LongSelect((x, i) => x)
|
||||
.LongSelectMany((x, i) => new[] { x })
|
||||
.SelectMany((x, i) => new[] { x })
|
||||
.Where((x, i) => true)
|
||||
.LongWhere((x, i) => true)
|
||||
.TakeWhile((x, i) => true)
|
||||
.LongTakeWhile((x, i) => true)
|
||||
.SkipWhile((x, i) => false)
|
||||
.LongSkipWhile((x, i) => false)
|
||||
.ToArray();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
result[1] = pt1.Select((x, i) => x)
|
||||
.LongSelect((x, i) => x)
|
||||
.LongSelectMany((x, i) => new[] { x })
|
||||
.SelectMany((x, i) => new[] { x })
|
||||
.Where((x, i) => true)
|
||||
.LongWhere((x, i) => true)
|
||||
.TakeWhile((x, i) => true)
|
||||
.LongTakeWhile((x, i) => true)
|
||||
.SkipWhile((x, i) => false)
|
||||
.LongSkipWhile((x, i) => false)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13130_ReverseOperator(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13130_ReverseOperator";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
//data set #1
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
result[0] = pt1.Reverse().ToArray();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
result[1] = pt1.Reverse().ToArray();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
//data set #2 ToDo
|
||||
//data set #3 ToDo
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13736_IndexedTakeWhile(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13736_IndexedTakeWhile";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
// dataset #1
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var pt2 = pt1.TakeWhile((x, i) => i < 6).ToArray();
|
||||
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 6).ToArray();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var pt2 = pt1.TakeWhile((x, i) => i < 6).ToArray();
|
||||
result[1] = pt1.AsEnumerable().TakeWhile((x, i) => i < 6).ToArray();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
// dataset #2
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var pt2 = pt1.TakeWhile((x, i) => i < 125).ToArray();
|
||||
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 125).ToArray();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetGroupByReduceDataSet(context);
|
||||
var pt2 = pt1.TakeWhile((x, i) => i < 125).ToArray();
|
||||
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 125).ToArray();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13534_HashPartitionNegIndexIsError(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13534_HashPartitionNegIndexIsError";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
pt1.HashPartition(x => x, 0); // exception should be thrown
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
pt1.HashPartition(x => x, -1); // exception should be thrown
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13474_and_Bug13483_FromDscOnBadFileSet(DryadLinqContext context)
|
||||
{
|
||||
// ToDo
|
||||
return false;
|
||||
}
|
||||
public static bool Bug13637_EmptyFilesInFilesets(DryadLinqContext context)
|
||||
{
|
||||
// ToDo
|
||||
return false;
|
||||
}
|
||||
|
||||
public static bool Bug13637_LocalDebugProducingZeroRecords(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13637_LocalDebugProducingZeroRecords";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/Bug13637_LocalDebugProducingZeroRecords";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<int> pt2 = pt1.Where(x => false).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var queryInfo2 = pt2.Submit();
|
||||
queryInfo2.Wait();
|
||||
int[] data2 = pt2.ToArray();
|
||||
passed &= (data2.Length == 0);
|
||||
|
||||
// query producing no records -> anonymous output
|
||||
IQueryable<int> pt3 = pt1.Where(x => false);
|
||||
var queryInfo3 = pt3.Submit();
|
||||
queryInfo3.Wait();
|
||||
|
||||
int[] data3 = pt3.ToArray();
|
||||
passed &= (data3.Length == 0);
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13245_FromDsc_Submit_throws(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13245_FromDsc_Submit_throws";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var info = pt1.Submit();
|
||||
passed &= false; // should throw ArgEx as the input isn't well formed
|
||||
}
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static IQueryable<int> Blah(this IQueryable<int> source)
|
||||
{
|
||||
return Enumerable.Range(1, 1).AsQueryable();
|
||||
}
|
||||
public static bool Bug13245_QueryUsingNonHpcLinqOperator(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13245_QueryUsingNonHpcLinqOperator";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var info = pt1.Blah().Submit();
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static IEnumerable<int> CheckEnvVar(IEnumerable<int> en)
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable("DummyEnvVar") == null)
|
||||
throw new Exception("the expected environment variable is not defined");
|
||||
return en;
|
||||
}
|
||||
public static bool Bug13302_ConfigEnvironmentCleanup(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13302_ConfigEnvironmentCleanup";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var info = pt1.Apply((en) => CheckEnvVar(en)).Submit();
|
||||
info.Wait();
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug13970_MismatchedDataTypes(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug13970_MismatchedDataTypes";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetGroupByReduceDataSet(context);
|
||||
double? output = pt1.AverageAsQuery().Single();
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= true; // expected
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14010_AlreadyDisposedContext(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug14010_AlreadyDisposedContext";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
context.LocalDebug = false;
|
||||
try
|
||||
{
|
||||
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
|
||||
ctx.Dispose();
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
|
||||
int output = pt1.Select(x => x).First();
|
||||
passed &= false;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
|
||||
ctx.Dispose();
|
||||
int output = pt1.Select(x => x).First();
|
||||
passed &= false;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
|
||||
ctx.Dispose();
|
||||
IQueryable<int> query = pt1.Select(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, "abc"), true);
|
||||
var info = DryadLinqQueryable.Submit(query);
|
||||
passed &= false;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
passed &= true;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14256_LeaseOnTempDscFileset(DryadLinqContext context) // TODO
|
||||
{
|
||||
string testName = "Bug14256_LeaseOnTempDscFileset";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14189_OrderPreservation(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug14189_OrderPreservation";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IGrouping<int, int>[] clusterSorted, localSorted;
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
var output = pt1.OrderBy(x => x % 4)
|
||||
.Select(x => x).Select(x => x).Where(x => true) // this pipeline was not preserving order correctly.
|
||||
.GroupBy(x => x % 4)
|
||||
.ToArray();
|
||||
|
||||
passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
|
||||
// sort back on the key for deterministic output.
|
||||
clusterSorted = output.OrderBy(x => x.Key).ToArray();
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
var output = pt1.OrderBy(x => x % 4)
|
||||
.Select(x => x).Select(x => x).Where(x => true) // this pipeline was not preserving order correctly.
|
||||
.GroupBy(x => x % 4)
|
||||
.ToArray();
|
||||
|
||||
passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
|
||||
// sort back on the key for deterministic output.
|
||||
localSorted = output.OrderBy(x => x.Key).ToArray();
|
||||
}
|
||||
|
||||
// check that each group of output has the same elements as the LINQ groups.
|
||||
for (int i = 0; i < 4; i++)
|
||||
{
|
||||
var a = clusterSorted[i].OrderBy(x => x);
|
||||
var b = localSorted[i].OrderBy(x => x);
|
||||
|
||||
passed &= a.SequenceEqual(b); //the output should match linq. Error for group: + i);
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14190_MergeJoin_DecreasingOrder(DryadLinqContext context) // TODO
|
||||
{
|
||||
string testName = "Bug14190_MergeJoin_DecreasingOrder";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/Bug14190_MergeJoin_DecreasingOrder";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14192_MultiApplySubExpressionReuse(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug14192_MultiApplySubExpressionReuse";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/Bug14192_MultiApplySubExpressionReuse";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var data = pt1.Select(x => x);
|
||||
|
||||
var info = pt1.Apply(new[] { pt1, pt1 }, (sources) => new int[] { 1, 2, 3 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
info.Wait();
|
||||
|
||||
// ToDo
|
||||
//string queryPlan = TestUtils.GetRecentQueryXmlAsText();
|
||||
//int nVerticesInPlan = TestUtils.GetVertexStageCount(queryPlan);
|
||||
|
||||
//passed &= (nVerticesInPlan == 7); // "Only seven vertices should appear (before bug, there were 10 of which the last three were extraneous.");
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug14870_LongIndexTakeWhile(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug14870_LongIndexTakeWhile";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
|
||||
// ToDo - move to data generator
|
||||
int[][] data = new[] {
|
||||
new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { }, new int[] { } };
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.LongTakeWhile((x, i) => i < 8).ToArray();
|
||||
passed &= (output.Length == 8); // "Only eight items should be returned."
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug15159_NotOperatorForNullableBool(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug15159_NotOperatorForNullableBool";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<bool?>[] result = new IEnumerable<bool?>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
|
||||
result[0] = output.ToArray();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
|
||||
result[1] = output.ToArray();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public class NoDataMembersClass
|
||||
{
|
||||
|
||||
}
|
||||
public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug15371_NoDataMembersForSerialization";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
IQueryable<NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
|
||||
var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
|
||||
jobInfo.Wait();
|
||||
var result = context.FromStore<NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (DryadLinqException Ex)
|
||||
{
|
||||
passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
|
||||
Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(DryadLinqContext context)
|
||||
{
|
||||
string testName = "Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
// use an anonymous type with nullable fields, and make sure we at least have some null values in the stream
|
||||
var output = pt1.Select(x => new { FirstField = (x % 2 == 0) ? new int?(x) : default(int?), SecondField = x.ToString() })
|
||||
.GroupBy(x => x.FirstField, y => y.SecondField); // use of GB ensures we exercise the emitted GetHashCode() overload
|
||||
passed &= (output.Count() != 0); // "Query return 0 length output"
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -17,13 +17,15 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
|||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using System.Reflection;
|
||||
using Microsoft.Research.DryadLinq;
|
||||
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
|
||||
namespace DryadLinqTests
|
||||
|
|
@ -32,64 +34,38 @@ namespace DryadLinqTests
|
|||
{
|
||||
public static void Main(string[] args)
|
||||
{
|
||||
//Test1(args);
|
||||
//Test2(args);
|
||||
//Test3(args);
|
||||
//Test4(args);
|
||||
//Test5(args);
|
||||
SimpleTests.Run(args);
|
||||
|
||||
/*
|
||||
*
|
||||
Config conf = new Config("dryad-temp", "main1", @"d:\temp\TestLog\");
|
||||
DryadLinqContext context = new DryadLinqContext(Config.cluster);
|
||||
|
||||
string matchPattern = @"";
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "BasicAPITests.txt");
|
||||
BasicAPITests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "ApplyAndForkTests.txt");
|
||||
ApplyAndForkTests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "GroupByReduceTests.txt");
|
||||
GroupByReduceTests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "RangePartitionAPICoverageTests.txt");
|
||||
RangePartitionAPICoverageTests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "TypesInQueryTests.txt");
|
||||
TypesInQueryTests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "SerializationTests.txt");
|
||||
SerializationTests.Run(context, matchPattern);
|
||||
|
||||
TestLog.LogInit(Config.testLogPath + "MiscBugFixTests.txt");
|
||||
MiscBugFixTests.Run(context, matchPattern);
|
||||
*/
|
||||
|
||||
}
|
||||
|
||||
public static void Test1(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile");
|
||||
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var lines = input.Where(x => x.Line.Contains("white"));
|
||||
var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
|
||||
result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test2(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile:///d:/DryadLinqTemp/PartFiles");
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var q1 = input.Where(x => x.Line.Contains("white"));
|
||||
var q2 = input.Where(x => x.Line.Contains("the"));
|
||||
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
|
||||
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
|
||||
DryadLinqQueryable.SubmitAndWait(res1, res2);
|
||||
}
|
||||
|
||||
public static void Test3(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile:///d:/DryadLinqTemp/PartFiles");
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var words = input.SelectMany(x => x.Line.Split(' '));
|
||||
var groups = words.GroupBy(x => x);
|
||||
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
|
||||
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
|
||||
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
|
||||
result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test4(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext("svc-d2-01");
|
||||
var input = context.FromStore<LineRecord>("hdfs://svc-d2-01:8033/user/misard/foo.txt");
|
||||
var lines = input.Where(x => x.Line.Contains("white"));
|
||||
var result = lines.ToStore("hdfs://svc-d2-01:8033/user/yuanbyu/foo.txt", true);
|
||||
result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test5(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1);
|
||||
Uri dataUri = AzureUtils.ToAzureUri("msrsvc", "I4JPlk0bZ6YWypg+RJamyq0us1b+kCcuoeKlPhfiHTcVW7P4xvuzURvlRShSo1O3UDhcL2LiY4kMaarD+p1lKg==", "test", "testwrite");
|
||||
IEnumerable<LineRecord> lines = DataProvider.ReadData<LineRecord>(context, dataUri);
|
||||
foreach (var x in lines)
|
||||
{
|
||||
Console.WriteLine(x);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,8 +28,8 @@ using System.Runtime.InteropServices;
|
|||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("")]
|
||||
[assembly: AssemblyProduct("DryadLinqTests")]
|
||||
[assembly: AssemblyCopyright("Copyright © 2014")]
|
||||
[assembly: AssemblyProduct("Dryad")]
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
|
|
@ -39,7 +39,7 @@ using System.Runtime.InteropServices;
|
|||
[assembly: ComVisible(false)]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("1fce146c-8597-4cc2-92f6-ebb655cd99ed")]
|
||||
[assembly: Guid("49b725be-1187-4056-9890-7a0cb4c064a4")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
|
|
@ -51,5 +51,5 @@ using System.Runtime.InteropServices;
|
|||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("1.0.0.0")]
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||
[assembly: AssemblyVersion("0.1.5.0")]
|
||||
[assembly: AssemblyFileVersion("0.1.5.0")]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,842 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.DryadLinq.Internal;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class RangePartitionAPICoverageTests
|
||||
{
|
||||
public static void Run(DryadLinqContext context, string matchPattern)
|
||||
{
|
||||
TestLog.Message(" **********************");
|
||||
TestLog.Message(" RangePartitionAPICoverageTests ");
|
||||
TestLog.Message(" **********************");
|
||||
|
||||
var tests = new Dictionary<string, Action>()
|
||||
{
|
||||
{"RP_keySelector", () => RP_keySelector(context) },
|
||||
{"RP_keySelector_pcount", () => RP_keySelector_pcount(context) },
|
||||
{"RP_keySelector_isDescending", () => RP_keySelector_isDescending(context) },
|
||||
{"RP_keySelector_rangeKeys", () => RP_keySelector_rangeKeys(context) },
|
||||
{"RP_keySelector_isDescending_pcount", () => RP_keySelector_isDescending_pcount(context) },
|
||||
{"RP_keySelector_keyComparer_isDescending", () => RP_keySelector_keyComparer_isDescending(context) },
|
||||
{"RP_keySelector_rangeKeys_keyComparer", () => RP_keySelector_rangeKeys_keyComparer(context) },
|
||||
{"RP_keySelector_keyComparer_isDescending_pcount", () => RP_keySelector_keyComparer_isDescending_pcount(context) },
|
||||
{"RP_keySelector_rangeKeys_keyComparer_isDescending", () => RP_keySelector_rangeKeys_keyComparer_isDescending(context) },
|
||||
{"RP_singlePartition_autoSeparators", () => RP_singlePartition_autoSeparators(context) },
|
||||
{"RP_rangeSeparators_customComparer", () => RP_rangeSeparators_customComparer(context) },
|
||||
{"RP_rangeSeparators_nullCustomComparer", () => RP_rangeSeparators_nullCustomComparer(context) },
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
{
|
||||
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
|
||||
{
|
||||
test.Value.Invoke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static IEnumerable<int> CountPartitions<T>(IEnumerable<T> pt)
|
||||
{
|
||||
yield return 1;
|
||||
}
|
||||
|
||||
public static IEnumerable<T> MaxInPartition<T>(IEnumerable<T> pt)
|
||||
{
|
||||
yield return pt.Max();
|
||||
}
|
||||
|
||||
public static IEnumerable<T> MinInPartition<T>(IEnumerable<T> pt)
|
||||
{
|
||||
yield return pt.Min();
|
||||
}
|
||||
|
||||
public static IEnumerable<T2> MaxCompare<T1, T2>(IEnumerable<T1> pt, int i, IEnumerable<T2> maxes)
|
||||
{
|
||||
if (0 == i)
|
||||
{
|
||||
yield return (T2)Convert.ChangeType(1, typeof(int));
|
||||
}
|
||||
|
||||
int priorMax = (int)Convert.ChangeType(maxes.ToArray()[i - 1], typeof(T2));
|
||||
foreach (T1 item in pt)
|
||||
{
|
||||
int val = (int)Convert.ChangeType(item, typeof(T1));
|
||||
if (val <= priorMax)
|
||||
{
|
||||
yield return (T2)Convert.ChangeType(1, typeof(int));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static bool RP_keySelector(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 3, false);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_pcount(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_pcount";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_pcount";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, 2).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 2, false);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, 2);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_isDescending(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_isDescending";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_isDescending";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 3, true);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, true);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_rangeKeys(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_rangeKeys";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
IEnumerable<int>[] result2 = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_rangeKeys_1";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
//increasing ranges keys.
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 4, false);
|
||||
result[0] = pt2.ToList();
|
||||
|
||||
|
||||
string outFile2 = "unittest/output/RP_keySelector_rangeKeys_2";
|
||||
IQueryable<int> pt1_2 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
//decreasing ranges keys.
|
||||
IQueryable<int> pt2_2 = pt1_2.RangePartition(x => x, new[] { 8, 5, 2 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile2), true);
|
||||
var jobInfo2 = pt2_2.Submit();
|
||||
jobInfo2.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2_2, 4, true);
|
||||
result2[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
//increasing ranges keys.
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 });
|
||||
result[1] = pt2.ToList();
|
||||
|
||||
IQueryable<int> pt1_2 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
//decreasing ranges keys.
|
||||
IQueryable<int> pt2_2 = pt1_2.RangePartition(x => x, new[] { 8, 5, 2 });
|
||||
result2[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
Validate.Check(result2);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_isDescending_pcount(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_isDescending_pcount";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_isDescending_pcount";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, true, 2).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 2, true);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, true, 2);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_keyComparer_isDescending(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_keyComparer_isDescending";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_keyComparer_isDescending";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 3, true);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_rangeKeys_keyComparer(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_rangeKeys_keyComparer";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_rangeKeys_keyComparer";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }, Comparer<int>.Default)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 4, false);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }, Comparer<int>.Default);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_keyComparer_isDescending_pcount(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_keyComparer_isDescending_pcount";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_keyComparer_isDescending_pcount";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true, 2)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 2, true);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, Comparer<int>.Default, true, 2);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_keySelector_rangeKeys_keyComparer_isDescending(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_keySelector_rangeKeys_keyComparer_isDescending";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_keySelector_rangeKeys_keyComparer_isDescending";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer<int>.Default, true)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 4, true);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer<int>.Default, true);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_singlePartition_autoSeparators(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_singlePartition_autoSeparators";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_singlePartition_autoSeparators";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.Apply(x => x) // force a merge
|
||||
.RangePartition(x => x)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 1, false);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
IQueryable<int> pt2 = pt1.Apply(x => x) // force a merge
|
||||
.RangePartition(x => x);
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool RP_rangeSeparators_customComparer(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_rangeSeparators_customComparer";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int>[] result = new IEnumerable<int>[2];
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
string outFile = "unittest/output/RP_rangeSeparators_customComparer";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
|
||||
//these keys should be considered not-sorted
|
||||
try
|
||||
{
|
||||
var results = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, new WeirdIntComparer(), false).ToArray();
|
||||
passed &= false; // "an exception should have been thrown (non-sorted separators)."
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
//expected
|
||||
}
|
||||
|
||||
//these keys should also be considered not-sorted
|
||||
try
|
||||
{
|
||||
var results = pt1.RangePartition(x => x, new int[] { 4, 3, 2, 1 }, new WeirdIntComparer(), false).ToArray();
|
||||
passed &= false; // "an exception should have been thrown (non-sorted separators)."
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
//expected
|
||||
}
|
||||
|
||||
//these keys should work
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new int[] { 6, 6, 3, 1 }, new WeirdIntComparer(), false)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
passed &= TestRangePartitioned(pt2, 5, new WeirdIntComparer(), false);
|
||||
result[0] = pt2.ToList();
|
||||
}
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
string outFile = "unittest/output/RP_rangeSeparators_customComparer";
|
||||
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
|
||||
//these keys should be considered not-sorted
|
||||
try
|
||||
{
|
||||
var results = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, new WeirdIntComparer(), false).ToArray();
|
||||
passed &= false; // "an exception should have been thrown (non-sorted separators)."
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
//expected
|
||||
}
|
||||
|
||||
//these keys should also be considered not-sorted
|
||||
try
|
||||
{
|
||||
var results = pt1.RangePartition(x => x, new int[] { 4, 3, 2, 1 }, new WeirdIntComparer(), false).ToArray();
|
||||
passed &= false; // "an exception should have been thrown (non-sorted separators)."
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
//expected
|
||||
}
|
||||
|
||||
//these keys should work
|
||||
IQueryable<int> pt2 = pt1.RangePartition(x => x, new int[] { 6, 6, 3, 1 }, new WeirdIntComparer(), false)
|
||||
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
|
||||
var jobInfo = pt2.Submit();
|
||||
jobInfo.Wait();
|
||||
|
||||
result[1] = pt2.ToList();
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool TestRangePartitioned(IQueryable<int> pt, int expectedPCount, bool expectedIsDescending)
|
||||
{
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int> ptCount = pt.ApplyPerPartition(x => CountPartitions(x));
|
||||
int cPartitions = ptCount.Count();
|
||||
|
||||
passed &= (expectedPCount == cPartitions);
|
||||
|
||||
if (passed)
|
||||
{
|
||||
int[] ptMax = pt.ApplyPerPartition(x => MaxInPartition(x)).ToArray();
|
||||
int[] ptMin = pt.ApplyPerPartition(x => MinInPartition(x)).ToArray();
|
||||
|
||||
// compare mins and maxs of each partition
|
||||
for (int i = 1; i < cPartitions; i++)
|
||||
{
|
||||
if (false == expectedIsDescending)
|
||||
passed &= (ptMin[i] > ptMax[i - 1]);
|
||||
else
|
||||
passed &= (ptMax[i] < ptMin[i - 1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
return passed;
|
||||
}
|
||||
|
||||
|
||||
[Serializable]
|
||||
public class WeirdIntComparer : IComparer<int>
|
||||
{
|
||||
public int Compare(int x, int y)
|
||||
{
|
||||
int xx = x % 2;
|
||||
int yy = y % 2;
|
||||
return xx.CompareTo(yy);
|
||||
}
|
||||
}
|
||||
|
||||
public static bool RP_rangeSeparators_nullCustomComparer(DryadLinqContext context)
|
||||
{
|
||||
string testName = "RP_rangeSeparators_nullCustomComparer";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
|
||||
var results1 = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null, false).ToArray();
|
||||
var results2 = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null).ToArray();
|
||||
//passing is not throwing.
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool TestRangePartitioned(IQueryable<int> pt, int expectedPCount, IComparer<int> comparer, bool expectedIsDescending)
|
||||
{
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<int> ptCount = pt.ApplyPerPartition(x => CountPartitions(x));
|
||||
int cPartitions = ptCount.Count();
|
||||
|
||||
passed &= (expectedPCount == cPartitions);
|
||||
|
||||
if (passed)
|
||||
{
|
||||
int[] ptMax = pt.ApplyPerPartition(x => MaxInPartition(x)).ToArray();
|
||||
int[] ptMin = pt.ApplyPerPartition(x => MinInPartition(x)).ToArray();
|
||||
|
||||
// compare mins and maxs of each partition
|
||||
for (int i = 1; i < cPartitions; i++)
|
||||
{
|
||||
passed &= (comparer.Compare(ptMin[i], ptMax[i - 1]) == 0); // ToDo
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
return passed;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,595 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Decorated and undecorated types
|
||||
//
|
||||
|
||||
// A UDT with no attribute shouldn't autoserialize without an explicit [Serializable] attribute
|
||||
public class UDT_Undecorated
|
||||
{
|
||||
public UDT_Undecorated(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
}
|
||||
|
||||
// UDT which is marked as serializable. This should be autoserialized
|
||||
[Serializable]
|
||||
public class UDT_MarkedSerializable
|
||||
{
|
||||
public UDT_MarkedSerializable(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
}
|
||||
|
||||
// UDT that has the old style static serialization methods. This should be rejected.
|
||||
public class UDT_StaticSerializer
|
||||
{
|
||||
public UDT_StaticSerializer(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
|
||||
public static UDT_StaticSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
var val = new UDT_StaticSerializer(0);
|
||||
val.m_field1 = reader.ReadInt32();
|
||||
val.m_field2 = reader.ReadInt32();
|
||||
|
||||
return val;
|
||||
}
|
||||
|
||||
public static void Write(DryadLinqBinaryWriter writer, UDT_StaticSerializer val)
|
||||
{
|
||||
writer.Write(val.m_field1);
|
||||
writer.Write(val.m_field2);
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// UDTs with custom serializers
|
||||
//
|
||||
|
||||
// UDT that has an attribute declaring itself as its custom serializer
|
||||
[CustomDryadLinqSerializer(typeof(UDT_SelfCustomSerializer))]
|
||||
public struct UDT_SelfCustomSerializer : IDryadLinqSerializer<UDT_SelfCustomSerializer>
|
||||
{
|
||||
public UDT_SelfCustomSerializer(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
|
||||
public UDT_SelfCustomSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
var val = new UDT_SelfCustomSerializer(0);
|
||||
val.m_field1 = reader.ReadInt32();
|
||||
val.m_field2 = reader.ReadInt32();
|
||||
|
||||
return val;
|
||||
}
|
||||
|
||||
public void Write(DryadLinqBinaryWriter writer, UDT_SelfCustomSerializer val)
|
||||
{
|
||||
writer.Write(val.m_field1);
|
||||
writer.Write(val.m_field2);
|
||||
}
|
||||
}
|
||||
|
||||
// UDT that declares another type as its custom serializer
|
||||
[CustomDryadLinqSerializer(typeof(CustomUDTSerializer))]
|
||||
public struct UDT_ExternalCustomSerializer
|
||||
{
|
||||
public UDT_ExternalCustomSerializer(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
}
|
||||
|
||||
// this is the custom serializer for UDT_ExternalCustomSerializer
|
||||
public class CustomUDTSerializer : IDryadLinqSerializer<UDT_ExternalCustomSerializer>
|
||||
{
|
||||
public UDT_ExternalCustomSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
var val = new UDT_ExternalCustomSerializer(0);
|
||||
val.m_field1 = reader.ReadInt32();
|
||||
val.m_field2 = reader.ReadInt32();
|
||||
|
||||
return val;
|
||||
}
|
||||
|
||||
public void Write(DryadLinqBinaryWriter writer, UDT_ExternalCustomSerializer val)
|
||||
{
|
||||
writer.Write(val.m_field1);
|
||||
writer.Write(val.m_field2);
|
||||
}
|
||||
}
|
||||
|
||||
// UDT with a CustomHpcSerializer attribute that points to an invalid type for the serializer
|
||||
[CustomDryadLinqSerializer(typeof(int))]
|
||||
public struct UDT_BadCustomSerializerType1
|
||||
{
|
||||
public UDT_BadCustomSerializerType1(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
}
|
||||
|
||||
// UDT with a CustomHpcSerializer attribute that points to a serializer type that targets a different UDT
|
||||
[CustomDryadLinqSerializer(typeof(CustomUDTSerializer))]
|
||||
public struct UDT_BadCustomSerializerType2
|
||||
{
|
||||
public UDT_BadCustomSerializerType2(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
}
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Inheritance
|
||||
//
|
||||
|
||||
// A UDT that has sub types. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_BaseType
|
||||
{
|
||||
public UDT_BaseType(int val)
|
||||
{
|
||||
m_baseTypeField = val + 42;
|
||||
}
|
||||
|
||||
public int m_baseTypeField;
|
||||
}
|
||||
|
||||
// A UDT that has derives from a type other than object. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_DerivedType : UDT_BaseType
|
||||
{
|
||||
public UDT_DerivedType(int val)
|
||||
: base(val)
|
||||
{
|
||||
m_derivedTypeField = val + 84;
|
||||
}
|
||||
|
||||
public int m_derivedTypeField;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Field types
|
||||
//
|
||||
|
||||
// A UDT with no data. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_EmptyType
|
||||
{
|
||||
public UDT_EmptyType(int val)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
// A UDT with a public field of type System.Object. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_ObjectField
|
||||
{
|
||||
public UDT_ObjectField(int val)
|
||||
{
|
||||
m_intField = val + 84;
|
||||
m_objectField = null;
|
||||
}
|
||||
|
||||
public int m_intField;
|
||||
public object m_objectField;
|
||||
}
|
||||
|
||||
// A UDT with a public field of type System.Object[]. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_ObjectArrayField
|
||||
{
|
||||
public UDT_ObjectArrayField(int val)
|
||||
{
|
||||
m_intField = val + 84;
|
||||
m_objectArrayField = new object[1];
|
||||
m_objectArrayField[0] = null;
|
||||
}
|
||||
|
||||
public int m_intField;
|
||||
public object[] m_objectArrayField;
|
||||
}
|
||||
|
||||
// A UDT with a public field of type List<System.Object>. This should be rejected
|
||||
[Serializable]
|
||||
public class UDT_ObjectListField
|
||||
{
|
||||
public UDT_ObjectListField(int val)
|
||||
{
|
||||
m_intField = val + 84;
|
||||
m_objectListField = new List<object>();
|
||||
m_objectListField.Add(null);
|
||||
}
|
||||
|
||||
public int m_intField;
|
||||
public List<object> m_objectListField;
|
||||
}
|
||||
|
||||
// A UDT with an object field, and a custom serializer. This should not be rejected
|
||||
[CustomDryadLinqSerializer(typeof(UDT_ObjectFieldAndCustomSerializer))]
|
||||
public class UDT_ObjectFieldAndCustomSerializer : IDryadLinqSerializer<UDT_ObjectFieldAndCustomSerializer>
|
||||
{
|
||||
private object m_objectRecord = (object)"";
|
||||
|
||||
// Required by CustomHpcSerializer
|
||||
public UDT_ObjectFieldAndCustomSerializer() { }
|
||||
|
||||
public UDT_ObjectFieldAndCustomSerializer(int val)
|
||||
{
|
||||
m_objectRecord = (object)String.Format("{0}", val);
|
||||
}
|
||||
|
||||
#region IHpcSerializer implementation
|
||||
|
||||
public UDT_ObjectFieldAndCustomSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
string tmp = reader.ReadString();
|
||||
int val = Int32.Parse(tmp);
|
||||
|
||||
return new UDT_ObjectFieldAndCustomSerializer(val);
|
||||
}
|
||||
|
||||
public void Write(DryadLinqBinaryWriter writer, UDT_ObjectFieldAndCustomSerializer record)
|
||||
{
|
||||
writer.Write((string)record.m_objectRecord);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// An empty UDT with a custom serializer. This should be rejected, because even though the user has control over serialization
|
||||
// we will encounter runtime problems if the custom serializer reads/writer 0 bytes. This behavior is simply to discourage empty CS code.
|
||||
[CustomDryadLinqSerializer(typeof(UDT_EmptyTypeWithCustomSerializer))]
|
||||
public class UDT_EmptyTypeWithCustomSerializer : IDryadLinqSerializer<UDT_EmptyTypeWithCustomSerializer>
|
||||
{
|
||||
// Required by CustomHpcSerializer
|
||||
public UDT_EmptyTypeWithCustomSerializer() { }
|
||||
|
||||
public UDT_EmptyTypeWithCustomSerializer(int val)
|
||||
{
|
||||
}
|
||||
|
||||
#region IHpcSerializer implementation
|
||||
|
||||
public UDT_EmptyTypeWithCustomSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
return new UDT_EmptyTypeWithCustomSerializer(0);
|
||||
}
|
||||
|
||||
public void Write(DryadLinqBinaryWriter writer, UDT_EmptyTypeWithCustomSerializer record)
|
||||
{
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Visibility
|
||||
//
|
||||
|
||||
// UDT with a field of non-public type. We cannot handle these
|
||||
[Serializable]
|
||||
public class UDT_FieldOfNonPublicType
|
||||
{
|
||||
private enum SecretCodeLevel
|
||||
{
|
||||
Secret,
|
||||
SuperSecret,
|
||||
}
|
||||
|
||||
public UDT_FieldOfNonPublicType(int val)
|
||||
{
|
||||
m_field = (SecretCodeLevel)val;
|
||||
}
|
||||
|
||||
private SecretCodeLevel m_field;
|
||||
}
|
||||
|
||||
// UDT with a private field of a public type. We do handle these using emitted IL code.
|
||||
[Serializable]
|
||||
public class UDT_PrivateFieldOfPublicType
|
||||
{
|
||||
public UDT_PrivateFieldOfPublicType(int val)
|
||||
{
|
||||
m_field = val + 1;
|
||||
}
|
||||
|
||||
private int m_field;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Nesting
|
||||
//
|
||||
|
||||
[Serializable]
|
||||
public class UDT_Nested_InnerAndOuterSerializable
|
||||
{
|
||||
public UDT_Nested_InnerAndOuterSerializable(int val)
|
||||
{
|
||||
m_field = val + 1;
|
||||
m_field2 = new NestedSerializable(val);
|
||||
}
|
||||
|
||||
private int m_field;
|
||||
private NestedSerializable m_field2;
|
||||
|
||||
[Serializable]
|
||||
public class NestedSerializable
|
||||
{
|
||||
public NestedSerializable(int val)
|
||||
{
|
||||
m_field = val / 2.0;
|
||||
}
|
||||
|
||||
private double m_field;
|
||||
}
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class UDT_Nested_InnerEnum_InnerAndOuterSerializable
|
||||
{
|
||||
public UDT_Nested_InnerEnum_InnerAndOuterSerializable(int val)
|
||||
{
|
||||
m_field = val + 1;
|
||||
m_field2 = (NestedEnum)(val % 3);
|
||||
}
|
||||
|
||||
private int m_field;
|
||||
private NestedEnum m_field2;
|
||||
|
||||
//[Serializable]
|
||||
public enum NestedEnum
|
||||
{
|
||||
Foo = 0,
|
||||
Bar = 1,
|
||||
Baz = 2,
|
||||
}
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class UDT_Nested_OuterSerializableInnerNotSerializable
|
||||
{
|
||||
public UDT_Nested_OuterSerializableInnerNotSerializable(int val)
|
||||
{
|
||||
m_field = val + 1;
|
||||
m_field2 = new NestedNotSerializable(val);
|
||||
}
|
||||
|
||||
private int m_field;
|
||||
private NestedNotSerializable m_field2;
|
||||
|
||||
public class NestedNotSerializable
|
||||
{
|
||||
public NestedNotSerializable(int val)
|
||||
{
|
||||
m_field = val / 2.0;
|
||||
}
|
||||
|
||||
private double m_field;
|
||||
}
|
||||
}
|
||||
|
||||
public class UDT_Nested_OuterNotSerializableInnerSerializable
|
||||
{
|
||||
public UDT_Nested_OuterNotSerializableInnerSerializable(int val)
|
||||
{
|
||||
m_field = val + 1;
|
||||
m_field2 = new NestedSerializable(val);
|
||||
}
|
||||
|
||||
private int m_field;
|
||||
private NestedSerializable m_field2;
|
||||
|
||||
[Serializable]
|
||||
public class NestedSerializable
|
||||
{
|
||||
public NestedSerializable(int val)
|
||||
{
|
||||
m_field = val / 2.0;
|
||||
}
|
||||
|
||||
private double m_field;
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Self reference
|
||||
//
|
||||
|
||||
// First level circular type
|
||||
[Serializable]
|
||||
public class UDT_FirstLevelCircular
|
||||
{
|
||||
public UDT_FirstLevelCircular(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
m_circularRef = null;
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
|
||||
public UDT_FirstLevelCircular m_circularRef;
|
||||
}
|
||||
|
||||
// First level circular type with an array reference to self
|
||||
[Serializable]
|
||||
public class UDT_FirstLevelCircularArrayRef
|
||||
{
|
||||
public UDT_FirstLevelCircularArrayRef(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
m_circularRefArray = new UDT_FirstLevelCircular[5];
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
|
||||
public UDT_FirstLevelCircular[] m_circularRefArray;
|
||||
}
|
||||
|
||||
// Second level circular type
|
||||
[Serializable]
|
||||
public class UDT_SecondLevelCircular
|
||||
{
|
||||
public UDT_SecondLevelCircular(int val)
|
||||
{
|
||||
m_field1 = val + 1;
|
||||
m_field2 = val + 2;
|
||||
m_child = new UDT_CircularRefChild(this);
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public int m_field2;
|
||||
|
||||
public UDT_CircularRefChild m_child;
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class UDT_CircularRefChild
|
||||
{
|
||||
public UDT_CircularRefChild(UDT_SecondLevelCircular parent)
|
||||
{
|
||||
m_parent = parent;
|
||||
}
|
||||
|
||||
public UDT_SecondLevelCircular m_parent;
|
||||
}
|
||||
|
||||
// Circular type with custom serializer. Should not be rejected
|
||||
[CustomDryadLinqSerializer(typeof(UDT_CircularTypeWithCustomSerializer))]
|
||||
public class UDT_CircularTypeWithCustomSerializer : IDryadLinqSerializer<UDT_CircularTypeWithCustomSerializer>
|
||||
{
|
||||
public UDT_CircularTypeWithCustomSerializer() { }
|
||||
public UDT_CircularTypeWithCustomSerializer(int val)
|
||||
{
|
||||
m_field1 = val;
|
||||
|
||||
// create each new object with log2(val) self references hanging off of m_next
|
||||
if (val == 0)
|
||||
m_next = null;
|
||||
else
|
||||
m_next = new UDT_CircularTypeWithCustomSerializer(val / 2);
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public UDT_CircularTypeWithCustomSerializer m_next;
|
||||
|
||||
//
|
||||
// sample recursive custom serializer
|
||||
//
|
||||
public UDT_CircularTypeWithCustomSerializer Read(DryadLinqBinaryReader reader)
|
||||
{
|
||||
UDT_CircularTypeWithCustomSerializer obj = new UDT_CircularTypeWithCustomSerializer();
|
||||
|
||||
bool bHasValidNext = reader.ReadBool();
|
||||
obj.m_field1 = reader.ReadInt32();
|
||||
|
||||
if (bHasValidNext)
|
||||
{
|
||||
obj.m_next = this.Read(reader); // recursively read the next
|
||||
}
|
||||
else
|
||||
{
|
||||
obj.m_next = null; // terminate recursion
|
||||
}
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
public void Write(DryadLinqBinaryWriter writer, UDT_CircularTypeWithCustomSerializer x)
|
||||
{
|
||||
if (x.m_next != null)
|
||||
{
|
||||
writer.Write(true); // bHasValidNext for the reader side
|
||||
writer.Write(x.m_field1);
|
||||
this.Write(writer, x.m_next); // write out recursively
|
||||
}
|
||||
else
|
||||
{
|
||||
writer.Write(false); // bHasValidNext = false for the reader side, makes sure recursive reads stop
|
||||
writer.Write(x.m_field1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This type itself isn't circular, but contains a field of a circular type that is custom serialized. Autoserialization should work for this guy
|
||||
[Serializable]
|
||||
public class UDT_TypeContainingCustomSerializedCircularType
|
||||
{
|
||||
public UDT_TypeContainingCustomSerializedCircularType(int val)
|
||||
{
|
||||
m_field1 = val;
|
||||
m_circularType = new UDT_CircularTypeWithCustomSerializer(val);
|
||||
}
|
||||
|
||||
public int m_field1;
|
||||
public UDT_CircularTypeWithCustomSerializer m_circularType;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,380 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class SerializationTests
|
||||
{
|
||||
public static void Run(DryadLinqContext context, string matchPattern)
|
||||
{
|
||||
TestLog.Message(" **********************");
|
||||
TestLog.Message(" SerializationTests ");
|
||||
TestLog.Message(" **********************");
|
||||
|
||||
context.LocalDebug = false;
|
||||
|
||||
IQueryable<int> source = DataGenerator.GetSimpleFileSets(context);
|
||||
|
||||
var tests = new Dictionary<string, Action>()
|
||||
{
|
||||
{"UDT_Undecorated", () =>
|
||||
SerializationTests.TestUDT<UDT_Undecorated>(context, source, "Basic UDT with no attribute. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
|
||||
|
||||
{"UDT_MarkedSerializable", () =>
|
||||
SerializationTests.TestUDT<UDT_MarkedSerializable>(context, source, "Basic UDT which is marked as serializable.", null, 0)},
|
||||
|
||||
{"UDT_StaticSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_StaticSerializer>(context, source, "Basic UDT with old stype static serializers. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
|
||||
|
||||
{"UDT_SelfCustomSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_SelfCustomSerializer>(context, source, "UDT implementing custom serializer for itself.", null, 0)},
|
||||
|
||||
{"UDT_ExternalCustomSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_ExternalCustomSerializer>(context, source, "UDT with a custom serializer", null, 0)},
|
||||
|
||||
{"UDT_BadCustomSerializerType1", () =>
|
||||
SerializationTests.TestUDT<UDT_BadCustomSerializerType1>(context, source, "UDT decorated with an invalid custom serializer type.", typeof(DryadLinqException), 0)},
|
||||
|
||||
{"UDT_BadCustomSerializerType2", () =>
|
||||
SerializationTests.TestUDT<UDT_BadCustomSerializerType2>(context, source, "UDT decorated with another UDT's custom serializer.", typeof(DryadLinqException), 0)},
|
||||
|
||||
{"UDT_BaseType", () =>
|
||||
SerializationTests.TestUDT<UDT_BaseType>(context, source, "UDT with subtypes. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleSubtypes"))},
|
||||
|
||||
{"UDT_DerivedType", () =>
|
||||
SerializationTests.TestUDT<UDT_DerivedType>(context, source, "UDT with a base type. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleDerivedtypes"))},
|
||||
|
||||
{"UDT_FieldOfNonPublicType", () =>
|
||||
SerializationTests.TestUDT<UDT_FieldOfNonPublicType>(context, source, "UDT with a field of a non-public type. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("UDTHasFieldOfNonPublicType"))},
|
||||
|
||||
{"UDT_Nested_OuterSerializableInnerNotSerializable", () =>
|
||||
SerializationTests.TestUDT<UDT_Nested_OuterSerializableInnerNotSerializable>(context, source, "Nested UDT, outer type Serializable, inner type not Serializable. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable"))},
|
||||
|
||||
{"UDT_Nested_OuterNotSerializableInnerSerializable", () =>
|
||||
SerializationTests.TestUDT<UDT_Nested_OuterNotSerializableInnerSerializable>(context, source, "Nested UDT, outer type not Serializable, inner type Serializable. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable"))},
|
||||
|
||||
{"UDT_Nested_InnerAndOuterSerializable", () =>
|
||||
SerializationTests.TestUDT<UDT_Nested_InnerAndOuterSerializable>(context, source, "Nested UDT, both outer and innter types marked Serializable.", null, 0)},
|
||||
|
||||
{"UDT_Nested_InnerEnum_InnerAndOuterSerializable", () =>
|
||||
SerializationTests.TestUDT<UDT_Nested_InnerEnum_InnerAndOuterSerializable>(context, source, "Nested UDT, inner enum, both outer and innter types marked Serializable.", null, 0)},
|
||||
|
||||
{"int?", () =>
|
||||
SerializationTests.TestUDT<int?>(context, source, "Nullable type as the UDT. Should be autoserialized.", null, 0)},
|
||||
|
||||
{"UDT_FirstLevelCircular", () =>
|
||||
SerializationTests.TestUDT<UDT_FirstLevelCircular>(context, source, "Circular UDT. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
|
||||
|
||||
{"UDT_FirstLevelCircularArrayRef", () =>
|
||||
SerializationTests.TestUDT<UDT_FirstLevelCircularArrayRef>(context, source, "Circular UDT with array reference to self. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
|
||||
|
||||
{"UDT_SecondLevelCircular", () =>
|
||||
SerializationTests.TestUDT<UDT_SecondLevelCircular>(context, source, "Circular UDT with indirect reference to self. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes"))},
|
||||
|
||||
{"UDT_CircularTypeWithCustomSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_CircularTypeWithCustomSerializer>(context, source, "Circular UDT with a custom serializer. Should not be rejected.", null, 0)},
|
||||
|
||||
{"UDT_TypeContainingCustomSerializedCircularType", () =>
|
||||
SerializationTests.TestUDT<UDT_TypeContainingCustomSerializedCircularType>(context, source, "An UDT containing a field of a circular UDT with a custom serializer. Should be autoserialized.", null, 0)},
|
||||
|
||||
{"UDT_ObjectField", () =>
|
||||
SerializationTests.TestUDT<UDT_ObjectField>(context, source, "UDT with a field of type System.Object. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"UDT_ObjectArrayField", () =>
|
||||
SerializationTests.TestUDT<UDT_ObjectArrayField>(context, source, "UDT with a field of type System.Object[]. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"UDT_ObjectListField", () =>
|
||||
SerializationTests.TestUDT<UDT_ObjectListField>(context, source, "UDT with a field of type List<object>. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"object", () =>
|
||||
SerializationTests.TestUDT<object>(context, source, "UDT = System.Object. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"object[]", () =>
|
||||
SerializationTests.TestUDT<object[]>(context, source, "UDT = System.Object[]. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"List<object>", () =>
|
||||
SerializationTests.TestUDT<List<object>>(context, source, "UDT = List<object>. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields"))},
|
||||
|
||||
{"UDT_EmptyType", () =>
|
||||
SerializationTests.TestUDT<UDT_EmptyType>(context, source, "Empty UDT. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers"))},
|
||||
|
||||
{"UDT_ObjectFieldAndCustomSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_ObjectFieldAndCustomSerializer>(context, source, "UDT with an Object field, and a custom serializer. Should not be rejected.", null, 0)},
|
||||
|
||||
{"UDT_EmptyTypeWithCustomSerializer", () =>
|
||||
SerializationTests.TestUDT<UDT_EmptyTypeWithCustomSerializer>(context, source, "Empty UDT with a custom serializer. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers"))},
|
||||
|
||||
{"UDT_PrivateFieldOfPublicType", () =>
|
||||
SerializationTests.TestUDT<UDT_PrivateFieldOfPublicType>(context, source, "UDT with a private field of a public type. Should be autoserialized.", null, 0)},
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
{
|
||||
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
|
||||
{
|
||||
test.Value.Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// the generic entry method used by most tests in this file
|
||||
public static bool TestUDT<TRecord>(DryadLinqContext context, IQueryable<int> source, string testMessage, Type expectedExceptionType, int expectedErrorCode)
|
||||
{
|
||||
string testName = testMessage;
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
// first make sure SelectFunc knows how to handle the type.
|
||||
try
|
||||
{
|
||||
object o = SelectFunc<TRecord>(0);
|
||||
}
|
||||
catch
|
||||
{
|
||||
TestLog.Message(String.Format("SelectFunc doesn't support type {0}", typeof(TRecord).Name));
|
||||
passed &= false;
|
||||
goto Done;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var results = source.Select(x => SelectFunc<TRecord>(x));
|
||||
|
||||
int count = 0;
|
||||
foreach (var r in results) count++;
|
||||
|
||||
if (count == 0)
|
||||
{
|
||||
TestLog.Message("...FAILED! No elements returned from query");
|
||||
passed &= false;
|
||||
goto Done;
|
||||
}
|
||||
}
|
||||
catch (Exception exp)
|
||||
{
|
||||
if (expectedExceptionType == null)
|
||||
{
|
||||
TestLog.Message(String.Format("...FAILED! Caught {0} while none was expected: ", exp.GetType().Name));
|
||||
passed &= false;
|
||||
goto Done;
|
||||
}
|
||||
else if (expectedExceptionType != exp.GetType())
|
||||
{
|
||||
TestLog.Message(String.Format("...FAILED! Caught {0} while while {1} was expected: {0}", exp.GetType().Name, expectedExceptionType.Name));
|
||||
passed &= false;
|
||||
goto Done;
|
||||
}
|
||||
else if (exp is DryadLinqException && expectedErrorCode != 0 && ((DryadLinqException)exp).ErrorCode != expectedErrorCode)
|
||||
{
|
||||
TestLog.Message("...FAILED! Caught DryadLinqException but fault code is wrong.");
|
||||
passed &= false;
|
||||
goto Done;
|
||||
}
|
||||
|
||||
TestLog.Message("....PASSED (caught correct exception)");
|
||||
goto Done;
|
||||
}
|
||||
|
||||
if (expectedExceptionType == null)
|
||||
{
|
||||
TestLog.Message(".....PASSED");
|
||||
}
|
||||
else
|
||||
{
|
||||
TestLog.Message("...FAILED! No exception was thrown from query.");
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
Done:
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static TRecord SelectFunc<TRecord>(int x)
|
||||
{
|
||||
if (typeof(TRecord) == typeof(object))
|
||||
{
|
||||
return (TRecord)(new object());
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(object[]))
|
||||
{
|
||||
return (TRecord)((object)new object[0]);
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(List<object>))
|
||||
{
|
||||
return (TRecord)((object)new List<object>());
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_Undecorated))
|
||||
{
|
||||
return (TRecord)((object)new UDT_Undecorated(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_SelfCustomSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_SelfCustomSerializer(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_MarkedSerializable))
|
||||
{
|
||||
return (TRecord)((object)new UDT_MarkedSerializable(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_StaticSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_StaticSerializer(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_ExternalCustomSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_ExternalCustomSerializer(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_BadCustomSerializerType1))
|
||||
{
|
||||
return (TRecord)((object)new UDT_BadCustomSerializerType1(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_BadCustomSerializerType2))
|
||||
{
|
||||
return (TRecord)((object)new UDT_BadCustomSerializerType2(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_BaseType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_BaseType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_DerivedType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_DerivedType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_DerivedType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_DerivedType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_FieldOfNonPublicType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_FieldOfNonPublicType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_PrivateFieldOfPublicType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_PrivateFieldOfPublicType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_Nested_OuterSerializableInnerNotSerializable))
|
||||
{
|
||||
return (TRecord)((object)new UDT_Nested_OuterSerializableInnerNotSerializable(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_Nested_OuterNotSerializableInnerSerializable))
|
||||
{
|
||||
return (TRecord)((object)new UDT_Nested_OuterNotSerializableInnerSerializable(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_Nested_InnerAndOuterSerializable))
|
||||
{
|
||||
return (TRecord)((object)new UDT_Nested_InnerAndOuterSerializable(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_Nested_InnerEnum_InnerAndOuterSerializable))
|
||||
{
|
||||
return (TRecord)((object)new UDT_Nested_InnerEnum_InnerAndOuterSerializable(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(int?))
|
||||
{
|
||||
return (TRecord)((object)((int?)x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_FirstLevelCircular))
|
||||
{
|
||||
return (TRecord)((object)new UDT_FirstLevelCircular(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_FirstLevelCircularArrayRef))
|
||||
{
|
||||
return (TRecord)((object)new UDT_FirstLevelCircularArrayRef(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_SecondLevelCircular))
|
||||
{
|
||||
return (TRecord)((object)new UDT_SecondLevelCircular(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_CircularTypeWithCustomSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_CircularTypeWithCustomSerializer(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_TypeContainingCustomSerializedCircularType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_TypeContainingCustomSerializedCircularType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_ObjectField))
|
||||
{
|
||||
return (TRecord)((object)new UDT_ObjectField(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_ObjectArrayField))
|
||||
{
|
||||
return (TRecord)((object)new UDT_ObjectArrayField(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_ObjectListField))
|
||||
{
|
||||
return (TRecord)((object)new UDT_ObjectListField(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_ObjectFieldAndCustomSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_ObjectFieldAndCustomSerializer(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_EmptyType))
|
||||
{
|
||||
return (TRecord)((object)new UDT_EmptyType(x));
|
||||
}
|
||||
|
||||
if (typeof(TRecord) == typeof(UDT_EmptyTypeWithCustomSerializer))
|
||||
{
|
||||
return (TRecord)((object)new UDT_EmptyTypeWithCustomSerializer(x));
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Unrecognized TRecord");
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class C1
|
||||
{
|
||||
public int x;
|
||||
public float y;
|
||||
|
||||
public C1(int a, float b)
|
||||
{
|
||||
this.x = a;
|
||||
this.y = b;
|
||||
}
|
||||
}
|
||||
|
||||
public class SimpleTests
|
||||
{
|
||||
public static void Run(string[] args)
|
||||
{
|
||||
Test1(args);
|
||||
//Test2(args);
|
||||
//Test3(args);
|
||||
//Test4(args);
|
||||
//Test5(args);
|
||||
}
|
||||
|
||||
public static void Test1(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile");
|
||||
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var lines = input.Where(x => x.Line.Contains("white"));
|
||||
foreach (var x in lines) Console.WriteLine(x);
|
||||
//var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
|
||||
//result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test2(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile");
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var q1 = input.Where(x => x.Line.Contains("white"));
|
||||
var q2 = input.Where(x => x.Line.Contains("the"));
|
||||
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
|
||||
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
|
||||
DryadLinqQueryable.SubmitAndWait(res1, res2);
|
||||
}
|
||||
|
||||
public static void Test3(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1, "partfile");
|
||||
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
|
||||
var words = input.SelectMany(x => x.Line.Split(' '));
|
||||
var groups = words.GroupBy(x => x);
|
||||
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
|
||||
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
|
||||
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
|
||||
result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test4(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext("svc-d2-01");
|
||||
var input = context.FromStore<LineRecord>("hdfs://svc-d2-01:8033/user/misard/foo.txt");
|
||||
var lines = input.Where(x => x.Line.Contains("white"));
|
||||
var result = lines.ToStore("hdfs://svc-d2-01:8033/user/yuanbyu/foo.txt", true);
|
||||
result.SubmitAndWait();
|
||||
}
|
||||
|
||||
public static void Test5(string[] args)
|
||||
{
|
||||
DryadLinqContext context = new DryadLinqContext(1);
|
||||
Uri dataUri = AzureUtils.ToAzureUri("msrsvc", "test", "testwrite");
|
||||
IEnumerable<LineRecord> lines = DataProvider.ReadData<LineRecord>(context, dataUri);
|
||||
foreach (var x in lines)
|
||||
{
|
||||
Console.WriteLine(x);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class TestResult
|
||||
{
|
||||
public DryadLinqContext Context { get; private set; }
|
||||
public string TestName { get; private set; }
|
||||
public bool Passed { get; private set; }
|
||||
//public string Error { get; private set; }
|
||||
|
||||
public TestResult(string testName, DryadLinqContext context, bool passed)
|
||||
{
|
||||
TestName = testName;
|
||||
Passed = passed;
|
||||
Context = context;
|
||||
}
|
||||
}
|
||||
|
||||
public class TestLog
|
||||
{
|
||||
static internal int nTestsRun;
|
||||
static internal int nTestsPassed;
|
||||
static internal string fileName = "";
|
||||
|
||||
internal static void LogInit(string path)
|
||||
{
|
||||
fileName = path;
|
||||
|
||||
// create empty file
|
||||
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName))
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: QA test harnesses may rely on the formatting
|
||||
internal static void TestStart(string testName)
|
||||
{
|
||||
Message(" --- Starting: [" + testName + "] --- ");
|
||||
}
|
||||
|
||||
// NOTE: QA tests may rely on the formatting
|
||||
internal static void LogResult(TestResult result)
|
||||
{
|
||||
nTestsRun++;
|
||||
if (result.Passed)
|
||||
{
|
||||
nTestsPassed++;
|
||||
}
|
||||
|
||||
Message(" * " + (result.Passed ? "Pass" : "FAIL") );
|
||||
Message(" --- Completed: [" + result.TestName + "] --- ");
|
||||
NewLine();
|
||||
}
|
||||
internal static void LogResult(bool result)
|
||||
{
|
||||
nTestsRun++;
|
||||
if (result)
|
||||
{
|
||||
nTestsPassed++;
|
||||
}
|
||||
NewLine();
|
||||
}
|
||||
|
||||
internal static void NewLine()
|
||||
{
|
||||
if (fileName.Length > 0)
|
||||
{
|
||||
// append data
|
||||
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName, true))
|
||||
{
|
||||
file.WriteLine();
|
||||
}
|
||||
}
|
||||
Console.WriteLine();
|
||||
}
|
||||
internal static void Message(string msg)
|
||||
{
|
||||
if (fileName.Length > 0)
|
||||
{
|
||||
// append data
|
||||
using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName, true))
|
||||
{
|
||||
file.WriteLine(msg);
|
||||
}
|
||||
}
|
||||
Console.WriteLine(msg);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,515 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class TypesInQueryTests
|
||||
{
|
||||
public static void Run(DryadLinqContext context, string matchPattern)
|
||||
{
|
||||
TestLog.Message(" **********************");
|
||||
TestLog.Message(" TypesInQueryTests ");
|
||||
TestLog.Message(" **********************");
|
||||
|
||||
var tests = new Dictionary<string, Action>()
|
||||
{
|
||||
{"NonSealedTypeRecords", () => NonSealedTypeRecords(context) },
|
||||
{"DerivedTypeRecords", () => DerivedTypeRecords(context) },
|
||||
{"ObjectRecords", () => ObjectRecords(context) },
|
||||
{"BadRecordsNotSerialized", () => BadRecordsNotSerialized(context) },
|
||||
{"GroupByWithAnonymousTypes_Bug15675", () => GroupByWithAnonymousTypes_Bug15675(context) },
|
||||
{"GroupByWithAnonymousTypes_Pipeline", () => GroupByWithAnonymousTypes_Pipeline(context) },
|
||||
{"GroupByWithAnonymousTypes_MultipleAnonymousTypes", () => GroupByWithAnonymousTypes_MultipleAnonymousTypes(context) },
|
||||
{"GroupByWithAnonymousTypes_GenericWithAnonTypeParam", () => GroupByWithAnonymousTypes_GenericWithAnonTypeParam(context) },
|
||||
{"GroupByWithAnonymousTypes_ArrayOfAnon", () => GroupByWithAnonymousTypes_ArrayOfAnon(context) },
|
||||
{"GroupByWithAnonymousTypes_NestedAnonTypes", () => GroupByWithAnonymousTypes_NestedAnonTypes(context) },
|
||||
};
|
||||
|
||||
foreach (var test in tests)
|
||||
{
|
||||
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
|
||||
{
|
||||
test.Value.Invoke();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static bool NonSealedTypeRecords(DryadLinqContext context)
|
||||
{
|
||||
string testName = "NonSealedTypeRecords";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<NonSealedClass>[] result = new IEnumerable<NonSealedClass>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => new NonSealedClass()).ToArray();
|
||||
result[0] = output;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => new NonSealedClass()).ToArray();
|
||||
result[1] = output;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool DerivedTypeRecords(DryadLinqContext context)
|
||||
{
|
||||
string testName = "DerivedTypeRecords";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<ParentClass>[] result1 = new IEnumerable<ParentClass>[2];
|
||||
IEnumerable<ChildClass>[] result2 = new IEnumerable<ChildClass>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output1 = pt1.Select(x => new ParentClass()).ToArray();
|
||||
var output2 = pt1.Select(x => new ChildClass()).ToArray();
|
||||
result1[0] = output1;
|
||||
result2[0] = output2;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output1 = pt1.Select(x => new ParentClass()).ToArray();
|
||||
var output2 = pt1.Select(x => new ChildClass()).ToArray();
|
||||
result1[1] = output1;
|
||||
result2[1] = output2;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result1);
|
||||
Validate.Check(result2);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool ObjectRecords(DryadLinqContext context)
|
||||
{
|
||||
string testName = "ObjectRecords";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<object>[] result = new IEnumerable<object>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => new object()).ToArray();
|
||||
result[0] = output;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => new object()).ToArray();
|
||||
result[1] = output;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool BadRecordsNotSerialized(DryadLinqContext context)
|
||||
{
|
||||
string testName = "BadRecordsNotSerialized";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
IEnumerable<string>[] result = new IEnumerable<string>[2];
|
||||
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => (x % 2 == 0 ? new ChildClass() : new ParentClass())).Select(x => x is ChildClass ? "child" : "parent").ToArray();
|
||||
result[0] = output;
|
||||
}
|
||||
|
||||
// local
|
||||
{
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var output = pt1.Select(x => (x % 2 == 0 ? new ChildClass() : new ParentClass())).Select(x => x is ChildClass ? "child" : "parent").ToArray();
|
||||
result[1] = output;
|
||||
}
|
||||
|
||||
// compare result
|
||||
try
|
||||
{
|
||||
Validate.Check(result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestLog.Message("Error: " + ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
//Bug 15675 -- GroupBy creates a DryadOrderByNode which is confused by the anonymous type.
|
||||
// specifically, the call to DryadSort incorrectly referenced both the anonymous type and the code-gen proxy for the anonymous type.
|
||||
// -- the fix was to clean up how & when we process the types involved in the query so that only the proxy would be referenced.
|
||||
public static bool GroupByWithAnonymousTypes_Bug15675(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_Bug15675";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => new { Num = i % 10 })
|
||||
.GroupBy(x => x.Num, x => x.Num)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => new { Num = i % 10 })
|
||||
.GroupBy(x => x.Num, x => x.Num)
|
||||
.ToArray();
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool GroupByWithAnonymousTypes_Pipeline(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_Pipeline";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => new { Num = i % 10 })
|
||||
.Where(x => true)
|
||||
.GroupBy(x => x.Num, x => x.Num)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => new { Num = i % 10 })
|
||||
.Where(x => true)
|
||||
.GroupBy(x => x.Num, x => x.Num)
|
||||
.ToArray();
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool GroupByWithAnonymousTypes_MultipleAnonymousTypes(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_MultipleAnonymousTypes";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => new { Num = i % 10 })
|
||||
.Where(x => true)
|
||||
.Select(i => new { Num2 = i.Num })
|
||||
.GroupBy(x => x.Num2, x => x.Num2)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => new { Num = i % 10 })
|
||||
.Where(x => true)
|
||||
.Select(i => new { Num2 = i.Num })
|
||||
.GroupBy(x => x.Num2, x => x.Num2)
|
||||
.ToArray();
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool GroupByWithAnonymousTypes_GenericWithAnonTypeParam(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_GenericWithAnonTypeParam";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
{
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => MakeNewMyGenericType(new { Num = i % 10 }))
|
||||
.GroupBy(x => x.Field.Num, x => x.Field.Num)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => MakeNewMyGenericType(new { Num = i % 10 }))
|
||||
.GroupBy(x => x.Field.Num, x => x.Field.Num)
|
||||
.ToArray();
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool GroupByWithAnonymousTypes_ArrayOfAnon(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_ArrayOfAnon";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => new[] { new { Num = i % 10 } })
|
||||
.GroupBy(x => x[0].Num, x => x[0].Num)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => new[] { new { Num = i % 10 } })
|
||||
.GroupBy(x => x[0].Num, x => x[0].Num)
|
||||
.ToArray();
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
public static bool GroupByWithAnonymousTypes_NestedAnonTypes(DryadLinqContext context)
|
||||
{
|
||||
string testName = "GroupByWithAnonymousTypes_NestedAnonTypes";
|
||||
TestLog.TestStart(testName);
|
||||
|
||||
bool passed = true;
|
||||
try
|
||||
{
|
||||
// cluster
|
||||
context.LocalDebug = false;
|
||||
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result1 = pt1.Select(i => new { Num = new { NumInner = new int?(i % 10) } }) // nullable-fields present particular challenge.
|
||||
.GroupBy(x => x.Num.NumInner, x => x.Num.NumInner)
|
||||
.ToArray();
|
||||
|
||||
// local
|
||||
context.LocalDebug = true;
|
||||
IQueryable<int> pt2 = DataGenerator.GetSimpleFileSets(context);
|
||||
var result2 = pt2.Select(i => new { Num = new { NumInner = new int?(i % 10) } }) // nullable-fields present particular challenge.
|
||||
.GroupBy(x => x.Num.NumInner, x => x.Num.NumInner)
|
||||
.ToArray();
|
||||
|
||||
|
||||
passed &= (result1.Count() == result2.Count());
|
||||
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
|
||||
}
|
||||
catch (Exception Ex)
|
||||
{
|
||||
TestLog.Message("Error: " + Ex.Message);
|
||||
passed &= false;
|
||||
}
|
||||
|
||||
TestLog.LogResult(new TestResult(testName, context, passed));
|
||||
return passed;
|
||||
}
|
||||
|
||||
// This is a helper method to allow 'naming the anonymous type'
|
||||
// -- there is no name to give 'anonymous' inline to the original query,
|
||||
// but a generic-method only has to call it by pseudonym T, which is a useful workaround.
|
||||
public static MyGenericType<T> MakeNewMyGenericType<T>(T item)
|
||||
{
|
||||
return new MyGenericType<T>(item);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class MyGenericType<T>
|
||||
{
|
||||
public T Field;
|
||||
|
||||
public MyGenericType(T data)
|
||||
{
|
||||
Field = data;
|
||||
}
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class NonSealedClass
|
||||
{
|
||||
public int X;
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class ParentClass
|
||||
{
|
||||
public int X;
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class ChildClass : ParentClass
|
||||
{
|
||||
public int Y;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,410 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.DryadLinq.Internal;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using Microsoft.WindowsAzure.Storage;
|
||||
using Microsoft.WindowsAzure.Storage.Blob;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class Config
|
||||
{
|
||||
public static string accountName = @"";
|
||||
public static string storageKey = @"";
|
||||
public static string containerName = @"";
|
||||
public static string cluster = @"";
|
||||
|
||||
public static string testLogPath = @"";
|
||||
|
||||
public Config(string clusterName, string container, string logPath)
|
||||
{
|
||||
cluster = clusterName;
|
||||
|
||||
AzureSubscriptions azs = new AzureSubscriptions();
|
||||
Task<AzureCluster> clusterTask = azs.GetClusterAsync(clusterName);
|
||||
clusterTask.Wait();
|
||||
|
||||
accountName = clusterTask.Result.StorageAccount;
|
||||
storageKey = clusterTask.Result.StorageKey;
|
||||
containerName = container;
|
||||
testLogPath = logPath;
|
||||
}
|
||||
}
|
||||
|
||||
public class DataGenerator
|
||||
{
|
||||
public DataGenerator()
|
||||
{
|
||||
}
|
||||
|
||||
public static IQueryable<int> CreateSimpleFileSetsEx()
|
||||
{
|
||||
int[] input = { 0, 1, 2 };
|
||||
IEnumerable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
|
||||
IEnumerable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
|
||||
IEnumerable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 4, 4));
|
||||
|
||||
return rangePartition.AsQueryable();
|
||||
}
|
||||
|
||||
public static IEnumerable<IEnumerable<int>> CreateSimpleFileSets()
|
||||
{
|
||||
IEnumerable<IEnumerable<int>> data = new int[][]
|
||||
{
|
||||
new[] { 1, 2, 3, 4 },
|
||||
new[] { 5, 6, 7, 8 },
|
||||
new[] { 9, 10, 11, 12 },
|
||||
};
|
||||
return data;
|
||||
}
|
||||
|
||||
public static IEnumerable<IEnumerable<int>> CreateGroupByReduceDataSet()
|
||||
{
|
||||
// we need quite a few elements to ensure the combiner will be activated in Stage#1 groupBy.
|
||||
// 33 elements per partition should suffice, but 100 per partition is safer.
|
||||
IEnumerable<IEnumerable<int>> data = new int[][]
|
||||
{
|
||||
Enumerable.Range(1,100).ToArray(),
|
||||
Enumerable.Range(101,100).ToArray(),
|
||||
};
|
||||
return data;
|
||||
}
|
||||
|
||||
public static IEnumerable<IEnumerable<int>> CreateRangePartitionDataSet()
|
||||
{
|
||||
// we need a lot of data to ensure sampler will get some data.
|
||||
// A few thousand should suffice.
|
||||
IEnumerable<IEnumerable<int>> data = new int[][]
|
||||
{
|
||||
Enumerable.Range(1,1000).ToArray(),
|
||||
Enumerable.Range(20000,2000).ToArray(),
|
||||
Enumerable.Range(40000,5000).ToArray(),
|
||||
};
|
||||
return data;
|
||||
}
|
||||
|
||||
public static IQueryable<int> GetSimpleFileSets(DryadLinqContext context)
|
||||
{
|
||||
//IEnumerable<IEnumerable<int>> data = new int[][]
|
||||
// {
|
||||
// new[] { 0, 1, 2, 3 },
|
||||
// new[] { 4, 5, 6, 7 },
|
||||
// new[] { 8, 9, 10, 11},
|
||||
// };
|
||||
|
||||
//IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
|
||||
// "unittest/inputdata/SimpleFile.txt"));
|
||||
IQueryable<int> input = context.FromEnumerable(new int[1]);
|
||||
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
|
||||
IQueryable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
|
||||
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 4, 4));
|
||||
//IQueryable<int> store = rangePartition.ToStore(@"unittest/inputdata/SimpleFile.txt");
|
||||
return rangePartition;
|
||||
}
|
||||
|
||||
public static IQueryable<int> GetGroupByReduceDataSet(DryadLinqContext context)
|
||||
{
|
||||
//IEnumerable<IEnumerable<int>> data = new int[][] {
|
||||
// Enumerable.Range(1,100).ToArray(),
|
||||
// Enumerable.Range(101,100).ToArray(),
|
||||
// };
|
||||
|
||||
IQueryable<int> input = context.FromEnumerable(new int[1]);
|
||||
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 2)); // {0, 1}
|
||||
IQueryable<int> partitions = range.HashPartition(x => x, 2); // create 2 partitions
|
||||
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 100 + 1, 100));
|
||||
return rangePartition;
|
||||
}
|
||||
|
||||
public static IQueryable<int> GetRangePartitionDataSet(DryadLinqContext context)
|
||||
{
|
||||
// we need a lot of data to ensure sampler will get some data.
|
||||
// A few thousand should suffice.
|
||||
//IEnumerable<IEnumerable<int>> data = new int[][] {
|
||||
// Enumerable.Range(1,1000).ToArray(),
|
||||
// Enumerable.Range(20000,2000).ToArray(),
|
||||
// Enumerable.Range(40000,5000).ToArray(),
|
||||
// };
|
||||
|
||||
IQueryable<int> input = context.FromEnumerable(new int[1]);
|
||||
IQueryable<int> range = input.Apply(x => Enumerable.Range(0, 3)); // {0, 1, 2}
|
||||
IQueryable<int> partitions = range.HashPartition(x => x, 3); // create 3 partitions
|
||||
IQueryable<int> rangePartition = partitions.SelectMany(x => Enumerable.Range(x * 20000 + 1, 1000));
|
||||
return rangePartition;
|
||||
}
|
||||
}
|
||||
|
||||
[Serializable]
|
||||
public class ReverseComparer<T> : IComparer<T>
|
||||
{
|
||||
IComparer<T> _originalComparer;
|
||||
public ReverseComparer(IComparer<T> originalComparer)
|
||||
{
|
||||
_originalComparer = originalComparer ?? Comparer<T>.Default;
|
||||
}
|
||||
|
||||
public int Compare(T x, T y)
|
||||
{
|
||||
return (_originalComparer.Compare(y, x)); //note reversed order of operands
|
||||
}
|
||||
|
||||
public override bool Equals(object obj)
|
||||
{
|
||||
ReverseComparer<T> objTyped = obj as ReverseComparer<T>;
|
||||
|
||||
return objTyped != null && _originalComparer.Equals(objTyped._originalComparer);
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
// Modify the hash code so that it differs from the hash code for the underlying comparer.
|
||||
// It would also probably be good enough to just return _originalComparer.GetHashCode().
|
||||
return unchecked((_originalComparer.GetHashCode() + 123457) * 10007);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public class Utils
|
||||
{
|
||||
public static bool DeleteFile(string accountName, string accountKey, string containerName, string fileName, bool delSubDirs)
|
||||
{
|
||||
try
|
||||
{
|
||||
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + accountName + ";AccountKey=" + accountKey);
|
||||
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
|
||||
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
|
||||
container.CreateIfNotExists();
|
||||
BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
|
||||
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
|
||||
container.SetPermissions(containerPermissions);
|
||||
|
||||
if (false == delSubDirs)
|
||||
{
|
||||
CloudBlockBlob remoteFile = container.GetBlockBlobReference(fileName);
|
||||
remoteFile.DeleteIfExists();
|
||||
}
|
||||
|
||||
if (true == delSubDirs)
|
||||
{
|
||||
foreach (IListBlobItem item in container.ListBlobs(fileName, true))
|
||||
{
|
||||
CloudBlockBlob blob = (CloudBlockBlob)item;
|
||||
blob.DeleteIfExists();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
public static bool FileExists(string accountName, string accountKey, string containerName, string fileName)
|
||||
{
|
||||
try
|
||||
{
|
||||
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + accountName + ";AccountKey=" + accountKey);
|
||||
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
|
||||
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
|
||||
container.CreateIfNotExists();
|
||||
BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
|
||||
containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob;
|
||||
container.SetPermissions(containerPermissions);
|
||||
|
||||
IEnumerable<IListBlobItem> files = container.ListBlobs(fileName, true);
|
||||
return (files.Count() > 0);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
internal static DryadLinqContext MakeBasicConfig(string cluster) //???
|
||||
{
|
||||
var context = new DryadLinqContext(cluster);
|
||||
try
|
||||
{
|
||||
context.JobFriendlyName = "DryadLinq_DevUnitTests";
|
||||
context.CompileForVertexDebugging = true;
|
||||
context.JobEnvironmentVariables.Add("DummyEnvVar", "hello"); //note: this is consumed by a unit-test.
|
||||
|
||||
if (File.Exists("Microsoft.Hpc.Linq.pdb")) // TODO: fix references
|
||||
{
|
||||
context.ResourcesToAdd.Add("Microsoft.Hpc.Linq.pdb");
|
||||
}
|
||||
|
||||
if (File.Exists("Microsoft.Hpc.Dsc.Client.pdb")) // TODO: fix references
|
||||
{
|
||||
context.ResourcesToAdd.Add("Microsoft.Hpc.Dsc.Client.pdb");
|
||||
}
|
||||
|
||||
// To prevent job from running forever, and blocking other test
|
||||
context.JobRuntimeLimit = (int)TimeSpan.FromMinutes(30).TotalSeconds;
|
||||
|
||||
|
||||
//config.AllowConcurrentUserDelegatesInSingleProcess = false;
|
||||
|
||||
// If we are on Azure, we have to set the nodeGroup to "NodeRole" so that the default of "ComputeNodes" is not used
|
||||
// This fixes "FromEnumerableTests" on Azure which queries the active node-group.
|
||||
// Note also, the headnode for an azure deployment defaults to "HPCCluster" (at least from James' script)
|
||||
int onAzureInt = 0;
|
||||
string onAzureString = Environment.GetEnvironmentVariable("CCP_SCHEDULERONAZURE");
|
||||
if (onAzureString != null)
|
||||
{
|
||||
int.TryParse(onAzureString, out onAzureInt);
|
||||
}
|
||||
|
||||
if (onAzureInt == 1)
|
||||
{
|
||||
context.NodeGroup = "NodeRole";
|
||||
}
|
||||
|
||||
}
|
||||
catch (DryadLinqException)
|
||||
{
|
||||
}
|
||||
return context;
|
||||
}
|
||||
|
||||
internal static DryadLinqRecordReader<TRecord> MakeDryadRecordReader<TRecord>(DryadLinqContext context, string readPath)
|
||||
{
|
||||
DryadLinqFactory<TRecord> factory = (DryadLinqFactory<TRecord>)DryadLinqCodeGen.GetFactory(context, typeof(TRecord));
|
||||
NativeBlockStream nativeStream = ReflectionHelper.CreateDryadLinqFileStream(readPath, FileMode.Open, FileAccess.Read);
|
||||
// ??? NativeBlockStream nativeStream = ReflectionHelper.CreateDryadLinqFileStream(readPath, FileMode.Open, FileAccess.Read, DscCompressionScheme.None);
|
||||
DryadLinqRecordReader<TRecord> reader = factory.MakeReader(nativeStream);
|
||||
return reader;
|
||||
}
|
||||
}
|
||||
|
||||
public class Validate
|
||||
{
|
||||
public static void
|
||||
Check<T>(
|
||||
IEnumerable<T>[] ss,
|
||||
IComparer<T> comparer = null,
|
||||
bool sort = true,
|
||||
bool verbose = false,
|
||||
IComparer<T> sortcomparer = null
|
||||
)
|
||||
{
|
||||
|
||||
if (ss.Length == 0) return;
|
||||
|
||||
if (comparer == null)
|
||||
{
|
||||
comparer = Comparer<T>.Default;
|
||||
if (comparer == null)
|
||||
{
|
||||
throw new ArgumentNullException("Can't not be null.");
|
||||
}
|
||||
}
|
||||
if (sortcomparer == null)
|
||||
sortcomparer = comparer;
|
||||
|
||||
T[][] aa = new T[ss.Length][];
|
||||
for (int i = 0; i < aa.Length; i++)
|
||||
{
|
||||
aa[i] = ss[i].ToArray();
|
||||
if (sort) Array.Sort(aa[i], sortcomparer);
|
||||
}
|
||||
int len = aa[0].Length;
|
||||
for (int i = 1; i < aa.Length; i++)
|
||||
{
|
||||
if (aa[i].Length != len)
|
||||
{
|
||||
throw new Exception("Wrong number of elements.");
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < len; i++)
|
||||
{
|
||||
T elem = aa[0][i];
|
||||
for (int j = 1; j < aa.Length; j++)
|
||||
{
|
||||
if (verbose)
|
||||
{
|
||||
//TestOutput.WriteLine("Comparing {0} to {1}", elem.ToString(), aa[j][i].ToString());
|
||||
}
|
||||
if (comparer.Compare(elem, aa[j][i]) != 0)
|
||||
{
|
||||
throw new Exception("Elements failed to match: " + elem + " != " + aa[j][i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal static bool outFileExists(string outFile)
|
||||
{
|
||||
try
|
||||
{
|
||||
return Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static class ReflectionHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// Create DryadLinqFileStream object via reflection
|
||||
/// </summary>
|
||||
///<param name="parameters"></param>
|
||||
/// <returns></returns>
|
||||
public static NativeBlockStream CreateDryadLinqFileStream(params object[] parameters)
|
||||
{
|
||||
return Assembly.LoadWithPartialName("Microsoft.Hpc.Linq").GetType("Microsoft.Hpc.Linq.Internal.HpcLinqFileStream") //???
|
||||
.GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, parameters.Select(p => p.GetType()).ToArray(), null)
|
||||
.Invoke(parameters) as NativeBlockStream;
|
||||
}
|
||||
|
||||
private static Type s_errorCodeType = null;
|
||||
public static int GetDryadLinqErrorCode(string name)
|
||||
{
|
||||
if (s_errorCodeType == null)
|
||||
{
|
||||
Assembly asm = Assembly.Load("Microsoft.Research.DryadLinq");
|
||||
Type[] types = asm.GetTypes();
|
||||
foreach (var t in types)
|
||||
{
|
||||
if (t.Name == "DryadLinqErrorCode")
|
||||
{
|
||||
s_errorCodeType = t;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
var finfo = s_errorCodeType.GetField(name);
|
||||
return (int)finfo.GetValue(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
// comment the following line to run on Azure
|
||||
#define local
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Research.DryadLinq;
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
|
||||
namespace DryadLinqTests
|
||||
{
|
||||
public class WordCount
|
||||
{
|
||||
public static void WordCountExample()
|
||||
{
|
||||
#if local
|
||||
// This overload runs the computation on your local computer using a single worker
|
||||
var config = new DryadLinqContext(1);
|
||||
|
||||
var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
|
||||
// You can create inputs from any IEnumerable source using this method
|
||||
var input = config.FromEnumerable(lines);
|
||||
#else
|
||||
string clusterName = "Replace with your HDInsight 3.0 cluster name";
|
||||
// to use the davinci.txt example input below, select your cluster's default
|
||||
// storage account and container, which automatically includes the sample text
|
||||
string accountName = "Replace with a storage account name";
|
||||
string containerName = "Replace with a storage container name";
|
||||
|
||||
// This overload creates an Azure-based computation
|
||||
var config = new DryadLinqContext(clusterName);
|
||||
config.JobFriendlyName = "DryadLINQ Sample Wordcount";
|
||||
|
||||
// plain text files should be read as type LineRecord
|
||||
var input = config.FromStore<LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
|
||||
"example/data/gutenberg/davinci.txt"));
|
||||
#endif
|
||||
|
||||
var words = input.SelectMany(x => x.Line.Split(' '));
|
||||
var groups = words.GroupBy(x => x);
|
||||
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
|
||||
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
|
||||
|
||||
#if local
|
||||
// any collection computed by the query can be materialized back at the client,
|
||||
// not just the 'output' collection. For large collections this is expensive!
|
||||
foreach (LineRecord line in toOutput)
|
||||
{
|
||||
Console.WriteLine(line.Line);
|
||||
}
|
||||
#else
|
||||
// the 'true' parameter to ToStore means the output will be over-written if you run
|
||||
// the job more than once
|
||||
var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
|
||||
"wc-out.txt"), true).SubmitAndWait();
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,13 +1,23 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl.Async" version="1.0.168" targetFramework="net45" />
|
||||
<package id="Microsoft.Bcl.Build" version="1.0.14" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.1.8" targetFramework="net45" />
|
||||
<package id="Microsoft.Net.Http" version="2.2.22" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Dryad" version="0.1.4-beta001" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.DryadLINQ.MSAGL" version="3.0.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.2-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Common" version="1.1.1" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Common.Dependencies" version="1.0.1" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management" version="1.2.0" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.1.8" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.Storage" version="1.1.1" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="4.0.1" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -24,13 +25,13 @@
|
|||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -54,6 +55,13 @@
|
|||
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<NuGetPackageImportStamp>b0b5ea03</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<TargetName>$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<TargetName>$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Midl>
|
||||
|
|
@ -61,7 +69,7 @@
|
|||
</Midl>
|
||||
<ClCompile>
|
||||
<Optimization>Disabled</Optimization>
|
||||
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\..\peloponnese\HadoopBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<PreprocessorDefinitions>WIN32;_DEBUG;_LIB;WIN32_LEAN_AND_MEAN;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<MinimalRebuild>false</MinimalRebuild>
|
||||
<BasicRuntimeChecks>Default</BasicRuntimeChecks>
|
||||
|
|
@ -72,10 +80,6 @@
|
|||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
|
||||
</ClCompile>
|
||||
<Lib>
|
||||
<AdditionalDependencies>Microsoft.Research.Peloponnese.HadoopBridge.lib</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>..\..\..\..\..\peloponnese\bin\$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
|
||||
</Lib>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<Midl>
|
||||
|
|
@ -88,13 +92,9 @@
|
|||
</PrecompiledHeader>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\..\peloponnese\HadoopBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
|
||||
</ClCompile>
|
||||
<Lib>
|
||||
<AdditionalDependencies>Microsoft.Research.Peloponnese.HadoopBridge.lib</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>..\..\..\..\..\peloponnese\bin\$(Configuration)</AdditionalLibraryDirectories>
|
||||
</Lib>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="include\channelbuffer.h" />
|
||||
|
|
@ -149,13 +149,16 @@
|
|||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
|
||||
</ImportGroup>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
|
||||
</packages>
|
||||
|
|
@ -929,6 +929,37 @@ RChannelBufferHdfsWriter::RChannelBufferHdfsWriter(const char* uri)
|
|||
Hdfs::InstanceAccessor ia(bridge);
|
||||
ia.Discard();
|
||||
}
|
||||
|
||||
char envUserName[128];
|
||||
DWORD ret = ::GetEnvironmentVariableA("USER", envUserName, sizeof(envUserName));
|
||||
if (ret > 0 && ret < sizeof(envUserName)-1)
|
||||
{
|
||||
m_user.Set(envUserName);
|
||||
DrLogI("Hdfs writer set user to %s from environment", m_user.GetString());
|
||||
}
|
||||
else
|
||||
{
|
||||
if (ret == 0 && GetLastError() != ERROR_ENVVAR_NOT_FOUND)
|
||||
{
|
||||
DrLogW("Can't get USER environment variable: %s", DRERRORSTRING(DrGetLastError()));
|
||||
}
|
||||
if (ret >= sizeof(envUserName)-1)
|
||||
{
|
||||
DrLogW("USER environment is too long: %d", ret);
|
||||
}
|
||||
|
||||
DWORD bufSize = sizeof(envUserName);
|
||||
BOOL bRet = ::GetUserNameA(envUserName, &bufSize);
|
||||
if (bRet == 0)
|
||||
{
|
||||
DrLogW("Can't get username: %s", DRERRORSTRING(DrGetLastError()));
|
||||
}
|
||||
else
|
||||
{
|
||||
m_user.Set(envUserName);
|
||||
DrLogI("Hdfs writer set user to %s from username", m_user.GetString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1021,7 +1052,7 @@ bool RChannelBufferHdfsWriter::Open(Hdfs::Instance** pInstance,
|
|||
Hdfs::InstanceAccessor ia(instance);
|
||||
|
||||
Hdfs::Writer* writer;
|
||||
bool openedWriter = ia.OpenWriter(filePath.GetString(), false, &writer);
|
||||
bool openedWriter = ia.OpenCreate(filePath.GetString(), 1024 * 1024, -1, &writer);
|
||||
if (!openedWriter)
|
||||
{
|
||||
char* errorMsg = ia.GetExceptionMessage();
|
||||
|
|
@ -1036,6 +1067,21 @@ bool RChannelBufferHdfsWriter::Open(Hdfs::Instance** pInstance,
|
|||
ia.Discard();
|
||||
return false;
|
||||
}
|
||||
bool setPermissions = ia.SetOwnerAndPermission(filePath.GetString(), m_user.GetString(), NULL, 0644);
|
||||
if (!setPermissions)
|
||||
{
|
||||
char* errorMsg = ia.GetExceptionMessage();
|
||||
DrStr64 description;
|
||||
description.SetF("Can't set HDFS file permissions '%s': %s",
|
||||
m_uri.GetString(), errorMsg);
|
||||
HadoopNative::DisposeString(errorMsg);
|
||||
|
||||
m_completionItem.Attach(MakeErrorItem(DryadError_ChannelOpenError,
|
||||
description.GetString()));
|
||||
|
||||
ia.Discard();
|
||||
return false;
|
||||
}
|
||||
|
||||
DrLogI("Opened Hdfs writer for %s", m_uri.GetString());
|
||||
|
||||
|
|
|
|||
|
|
@ -149,6 +149,7 @@ private:
|
|||
bool Open(Hdfs::Instance** pInstance, Hdfs::Writer** pWriter);
|
||||
bool AddToQueue(WriteEntry* entry);
|
||||
|
||||
DrStr64 m_user;
|
||||
DrStr64 m_uri;
|
||||
DryadBList<WriteEntry> m_queue;
|
||||
UInt32 m_queueLength;
|
||||
|
|
|
|||
|
|
@ -2205,7 +2205,7 @@ void RChannelBufferWriterNativeFile::EagerCloseFile()
|
|||
}
|
||||
else
|
||||
{
|
||||
DrLogI( "Set final file pointer");
|
||||
DrLogI( "Set final file pointer. Length %I64u", finalLength.QuadPart);
|
||||
}
|
||||
|
||||
ok = SetEndOfFile(m_bufferedFileHandle);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -21,13 +21,13 @@
|
|||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>false</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>false</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -19,14 +19,14 @@
|
|||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>MultiByte</CharacterSet>
|
||||
<CLRSupport>false</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -22,7 +24,7 @@
|
|||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
</PropertyGroup>
|
||||
|
|
@ -30,7 +32,7 @@
|
|||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -54,6 +56,13 @@
|
|||
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<NuGetPackageImportStamp>7f5f48b5</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<TargetName>$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<TargetName>$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Midl>
|
||||
|
|
@ -72,10 +81,6 @@
|
|||
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
|
||||
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
|
||||
</ClCompile>
|
||||
<Lib>
|
||||
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
|
||||
</Lib>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<Midl>
|
||||
|
|
@ -91,10 +96,6 @@
|
|||
<AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;..\channel\include;src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
<AdditionalUsingDirectories>C:\Program Files %28x86%29\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5;%(AdditionalUsingDirectories)</AdditionalUsingDirectories>
|
||||
</ClCompile>
|
||||
<Lib>
|
||||
<AdditionalDependencies>%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalLibraryDirectories>%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
|
||||
</Lib>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClInclude Include="include\dryadvertex.h" />
|
||||
|
|
@ -123,13 +124,19 @@
|
|||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
</ImportGroup>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="Native" />
|
||||
</packages>
|
||||
|
|
@ -277,7 +277,6 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
|
|||
DryadVertexProgramRef* pProgram)
|
||||
{
|
||||
DryadVertexProgramBase* program;
|
||||
|
||||
//
|
||||
// If factory is not supplied, try to get it from factory registry.
|
||||
// If still unable, fail with error.
|
||||
|
|
@ -295,12 +294,14 @@ DrError VertexFactoryRegistry::MakeVertex(UInt32 vertexId,
|
|||
return DryadError_VertexInitialization;
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Get vertex factory. If one cannot be found, report initialization error
|
||||
//
|
||||
factory = LookUpFactory(argumentList[0]);
|
||||
if (factory == NULL)
|
||||
{
|
||||
DrLogW("Factory Registry called with unknown factory UID %s.", argumentList[0].GetString());
|
||||
DrStr128 errorString;
|
||||
errorString.SetF("Factory Registry called with unknown factory UID %s",
|
||||
argumentList[0].GetString());
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ using System.Runtime.InteropServices;
|
|||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using Microsoft.Research.Peloponnese.Azure;
|
||||
|
||||
namespace Microsoft.Research.Dryad.Channel
|
||||
{
|
||||
|
|
@ -74,7 +74,7 @@ namespace Microsoft.Research.Dryad.Channel
|
|||
{
|
||||
Log.LogInformation("Opening read for " + source.AbsoluteUri);
|
||||
string account, key, container, blobName;
|
||||
AzureUtils.FromAzureUri(source, out account, out key, out container, out blobName);
|
||||
Utils.FromAzureUri(source, out account, out key, out container, out blobName);
|
||||
client = new AzureDfsClient(account, key, container, false, new PeloponneseLogger(Log.Logger));
|
||||
client.SetParallelThreadCount(4);
|
||||
|
||||
|
|
@ -94,7 +94,7 @@ namespace Microsoft.Research.Dryad.Channel
|
|||
private async Task OpenBlob()
|
||||
{
|
||||
string account, key, container, blobName;
|
||||
AzureUtils.FromAzureUri(source, out account, out key, out container, out blobName);
|
||||
Utils.FromAzureUri(source, out account, out key, out container, out blobName);
|
||||
|
||||
NameValueCollection query = System.Web.HttpUtility.ParseQueryString(source.Query);
|
||||
|
||||
|
|
@ -105,9 +105,11 @@ namespace Microsoft.Research.Dryad.Channel
|
|||
++blobIndex;
|
||||
}
|
||||
|
||||
Log.LogInformation("Opening read for blob " + blobName);
|
||||
Uri blobUri = Utils.ToAzureUri(account, container, blobName, null, key);
|
||||
|
||||
readStream = (await client.GetDfsFileStreamAsync(blobName, ExecutionTimeout, new PeloponneseLogger(Log.Logger))).Stream;
|
||||
Log.LogInformation("Opening read for blob " + blobUri.AbsoluteUri);
|
||||
|
||||
readStream = (await client.GetDfsStreamReaderAsync(blobUri, ExecutionTimeout, new PeloponneseLogger(Log.Logger)));
|
||||
|
||||
long offset = -1;
|
||||
if (query["offset"] != null)
|
||||
|
|
@ -215,7 +217,7 @@ namespace Microsoft.Research.Dryad.Channel
|
|||
int toRead = (int)Math.Min((long)managedBuffer.Length, bytesToRead);
|
||||
|
||||
Log.LogInformation("About to read buffer length " + toRead);
|
||||
readData.nRead = await AzureUtils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
|
||||
readData.nRead = await Utils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
|
||||
{
|
||||
return await readStream.ReadAsync(managedBuffer, 0, toRead);
|
||||
});
|
||||
|
|
@ -301,7 +303,7 @@ namespace Microsoft.Research.Dryad.Channel
|
|||
{
|
||||
int toRead = (int)Math.Min(buffer.LongLength, endOffset - currentOffset);
|
||||
|
||||
int nRead = await AzureUtils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
|
||||
int nRead = await Utils.WrapInRetry(new PeloponneseLogger(Log.Logger), async () =>
|
||||
{
|
||||
return await readStream.ReadAsync(buffer, 0, toRead);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ using System.Linq;
|
|||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Research.Peloponnese.Storage;
|
||||
using Microsoft.Research.Peloponnese.Azure;
|
||||
|
||||
namespace Microsoft.Research.Dryad.Channel
|
||||
{
|
||||
|
|
|
|||
|
|
@ -24,32 +24,8 @@ using System.Runtime.InteropServices;
|
|||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("managedchannel")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("")]
|
||||
[assembly: AssemblyProduct("managedchannel")]
|
||||
[assembly: AssemblyCopyright("Copyright © 2014")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.ManagedChannel")]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("f806a093-c5ae-4752-82d3-26bcc2341b29")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("1.0.0.0")]
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||
|
|
|
|||
|
|
@ -4,24 +4,44 @@
|
|||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.3.0.0" newVersion="4.3.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.HDInsight.Threading" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.0" newVersion="1.2.1.0" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
|
|
@ -9,11 +11,12 @@
|
|||
<OutputType>Library</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>Microsoft.Research.Dryad.Channel</RootNamespace>
|
||||
<AssemblyName>DryadManagedChannel</AssemblyName>
|
||||
<AssemblyName>Microsoft.Research.Dryad.ManagedChannel</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir>
|
||||
<RestorePackages>true</RestorePackages>
|
||||
<NuGetPackageImportStamp>72e7670e</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
|
|
@ -35,59 +38,38 @@
|
|||
<AllowUnsafeBlocks>false</AllowUnsafeBlocks>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Hadoop.Client, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.HadoopBridge, Version=0.0.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.Research.Peloponnese.Utils, Version=0.7.0.0, Culture=neutral, processorArchitecture=AMD64">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=4.3.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.1.0.7, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Microsoft.WindowsAzure.Storage, Version=3.1.0.1, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\Newtonsoft.Json.6.0.5\lib\net45\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Data.Services.Client" />
|
||||
<Reference Include="System.Spatial, Version=5.6.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<Reference Include="System.Net" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.WebRequest" />
|
||||
<Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\..\..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll</HintPath>
|
||||
<HintPath>..\..\..\..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
|
|
@ -97,6 +79,9 @@
|
|||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\..\..\..\SharedAssemblyInfo.cs">
|
||||
<Link>Properties\SharedAssemblyInfo.cs</Link>
|
||||
</Compile>
|
||||
<Compile Include="AzureReader.cs" />
|
||||
<Compile Include="AzureWriter.cs" />
|
||||
<Compile Include="FileWriter.cs" />
|
||||
|
|
@ -116,10 +101,16 @@
|
|||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Data.Edm" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.1" targetFramework="net45" />
|
||||
<package id="Microsoft.Hadoop.Client" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.ConfigurationManager" version="2.0.3" targetFramework="net45" />
|
||||
<package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.1.0.7" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.2" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.1" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="3.1.0.1" targetFramework="net45" />
|
||||
<package id="Newtonsoft.Json" version="6.0.5" targetFramework="net45" />
|
||||
<package id="System.Spatial" version="5.6.2" targetFramework="net45" />
|
||||
<package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -17,32 +17,27 @@ See the Apache Version 2.0 License for specific language governing permissions a
|
|||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace Microsoft.Research.DryadLinq
|
||||
namespace VertexHostMain
|
||||
{
|
||||
internal class YarnScheduler : IScheduler
|
||||
internal class ConfigDependency : Microsoft.Research.Peloponnese.Shared.AssemblyDependencyAttribute
|
||||
{
|
||||
public string m_headNode;
|
||||
|
||||
public void Connect(string headNode)
|
||||
public
|
||||
ConfigDependency()
|
||||
: base("Microsoft.Research.Dryad.VertexHost.exe.config", false)
|
||||
{
|
||||
this.m_headNode = headNode;
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
class Program
|
||||
{
|
||||
this.m_headNode = null;
|
||||
}
|
||||
|
||||
public IServerVersion GetServerVersion()
|
||||
static int Main(string[] args)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
return VertexHost.Main(args);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
[assembly: VertexHostMain.ConfigDependency()]
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.VertexHost")]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("5534b858-8472-4dc3-8f24-5b59f8805156")]
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{DE2FFF43-3B47-4987-A9ED-0C0BE74C0352}</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>VertexHostMain</RootNamespace>
|
||||
<AssemblyName>Microsoft.Research.Dryad.VertexHost</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<TargetFrameworkProfile />
|
||||
<NuGetPackageImportStamp>fb5afcd2</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<OutputPath>..\..\..\..\bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<DebugType>full</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
|
||||
<OutputPath>..\..\..\..\bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<Optimize>true</Optimize>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<PlatformTarget>x64</PlatformTarget>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Web" />
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
<Reference Include="Microsoft.CSharp" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="..\..\..\..\SharedAssemblyInfo.cs">
|
||||
<Link>Properties\SharedAssemblyInfo.cs</Link>
|
||||
</Compile>
|
||||
<Compile Include="Program.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\vertexHost\VertexHost.vcxproj">
|
||||
<Project>{0cf3d1d5-9bbe-4175-979b-ec6138ef4f37}</Project>
|
||||
<Name>VertexHostBody</Name>
|
||||
</ProjectReference>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="app.config" />
|
||||
<None Include="packages.config" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
</Target>
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
<?xml version="1.0"?>
|
||||
<configuration>
|
||||
<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/></startup>
|
||||
<runtime>
|
||||
<gcServer enabled="true"></gcServer>
|
||||
<gcConcurrent enabled="false"></gcConcurrent>
|
||||
<generatePublisherEvidence enabled="false"/>
|
||||
<gcAllowVeryLargeObjects enabled="true" />
|
||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="net45" />
|
||||
</packages>
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -21,13 +21,13 @@
|
|||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
LIBRARY "DryadLINQNativeChannels.dll"
|
||||
LIBRARY "Microsoft.Research.Dryad.DryadLinq.NativeWrapper.dll"
|
||||
|
||||
EXPORTS
|
||||
GetTotalLength
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -19,14 +19,14 @@
|
|||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -52,8 +52,8 @@
|
|||
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">DryadLINQNativeChannels</TargetName>
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">DryadLINQNativeChannels</TargetName>
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad.DryadLinq.NativeWrapper</TargetName>
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad.DryadLinq.NativeWrapper</TargetName>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Midl>
|
||||
|
|
@ -111,6 +111,7 @@
|
|||
<ClInclude Include="stdafx.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="version.cpp" />
|
||||
<ClCompile Include="wrappernativeinfostubs.cpp" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@
|
|||
<ClCompile Include="wrappernativeinfostubs.cpp">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="version.cpp">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="DryadLINQNativeChannels.def">
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
#ifdef _MANAGED
|
||||
#pragma managed
|
||||
|
||||
using namespace System::Reflection;
|
||||
using namespace System::Runtime::InteropServices;
|
||||
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.DryadLinq.NativeWrapper")];
|
||||
[assembly: AssemblyDescription("")];
|
||||
[assembly: AssemblyConfiguration("")];
|
||||
[assembly: AssemblyCompany("Microsoft Corporation")];
|
||||
[assembly: AssemblyProduct("Microsoft.Research.Dryad.DryadLinq.NativeWrapper")];
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
|
||||
[assembly: AssemblyTrademark("")];
|
||||
[assembly: AssemblyCulture("")];
|
||||
|
||||
[assembly: ComVisible(false)];
|
||||
|
||||
[assembly: AssemblyVersion("0.2.1.0")];
|
||||
[assembly: AssemblyFileVersion("0.2.1.0")];
|
||||
#endif
|
||||
|
|
@ -28,7 +28,6 @@ limitations under the License.
|
|||
#include <mscoree.h>
|
||||
#include <wrappernativeinfo.h>
|
||||
|
||||
|
||||
#pragma managed
|
||||
|
||||
#pragma warning(disable:4947) // so that we can use Assembly::LoadWithPartialName()
|
||||
|
|
@ -198,18 +197,25 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
|
|||
logName = System::IO::Path::Combine(logDirectory, logName);
|
||||
}
|
||||
|
||||
int threadsPerWorker = 1;
|
||||
System::String^ threadsPerWorkerStr = System::Environment::GetEnvironmentVariable("DRYAD_THREADS_PER_WORKER");
|
||||
if (threadsPerWorkerStr != nullptr)
|
||||
{
|
||||
threadsPerWorker = Int32::Parse(threadsPerWorkerStr);
|
||||
}
|
||||
DrLogI("ManagedWrapperVertex: threadsPerWorker %u", threadsPerWorker);
|
||||
|
||||
DrLogI("ManagedWrapperVertex: %p %u %u", nativeInfo, numberOfInputChannels, numberOfOutputChannels);
|
||||
DrLogI("ManagedWrapperVertex: Calling %s.%s", GetArgument(2), GetArgument(3));
|
||||
DrLogging::FlushLog();
|
||||
|
||||
|
||||
DrStr128 errorMsg;
|
||||
DrError error;
|
||||
|
||||
{
|
||||
//
|
||||
// Instead of invoking the vertex entry point directly from here, we delegate it to the bridge method in the Microsoft.Hpc.Linq assembly, specifically:
|
||||
// static int Microsoft.Hpc.Linq.Internal.VertexEnv.VertexBridge(string vertexBridgeArgs)
|
||||
// Instead of invoking the vertex entry point directly from here, we delegate it to the bridge method in the Microsoft.Research.DryadLinq assembly, specifically:
|
||||
// static int Microsoft.Research.DryadLinq.Internal.VertexEnv.VertexBridge(string vertexBridgeArgs)
|
||||
//
|
||||
// This indirect method of invoking the vertex entry point is used so that any type load / assembly load problems coming from user code
|
||||
// can be caught and reported with full details using the same mechanism that other vertex failures go through (exception dumped into vertexexception.txt etc.)
|
||||
|
|
@ -217,8 +223,10 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
|
|||
// The format of vertexBridgeArgs is simply a comma separated string packing vertex assembly, class, method name, and the *actual* vertex method args (==the native channel string)
|
||||
// L"<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
|
||||
//
|
||||
System::String ^bridgeAssemblyPartialName = gcnew System::String(L"Microsoft.Research.DryadLinq");
|
||||
System::String ^bridgeClassName = gcnew System::String(L"Microsoft.Research.DryadLinq.Internal.VertexEnv");
|
||||
System::String^ classFullName = gcnew System::String(GetArgument(2));
|
||||
System::String^ assemblyName = classFullName->Substring(0, classFullName->LastIndexOf('.'));
|
||||
System::String ^bridgeAssemblyName = gcnew System::String(assemblyName);
|
||||
System::String ^bridgeClassName = gcnew System::String(assemblyName + ".Internal.VertexEnv");
|
||||
System::String ^bridgeMethodName = gcnew System::String(L"VertexBridge");
|
||||
|
||||
//
|
||||
|
|
@ -235,21 +243,19 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
|
|||
vertexMethodArgs->Append(gcnew System::String(arg.GetString()));
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Get assembly path, class name, and method name, and construct the vertex bridge args with the following format:
|
||||
// "<vertexAssembly>,<vertexClassName>,<vertexMethodName>,<vertexMethodArgs>"
|
||||
//
|
||||
System::Text::StringBuilder ^vertexBridgeArg = gcnew System::Text::StringBuilder();
|
||||
vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"c:\\HpcTemp\\user\\jobID\\Microsoft.Hpc.Linq0.dll";
|
||||
vertexBridgeArg->Append(gcnew System::String(GetArgument(1))); // path to vertex DLL as passed to the vertex host, e.g. L"\\HpcTemp\\user\\jobID\\Microsoft.Research.DryadLinq0.dll";
|
||||
vertexBridgeArg->Append(",");
|
||||
vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Hpc.Linq.HpcLinq__Vertex";
|
||||
vertexBridgeArg->Append(gcnew System::String(GetArgument(2))); // full name of class that contains vertex entry method, e.g. L"Microsoft.Research.DryadLinq.DryadLinq__Vertex";
|
||||
vertexBridgeArg->Append(",");
|
||||
vertexBridgeArg->Append(gcnew System::String(GetArgument(3))); // vertex entry method name L"Select__1";
|
||||
vertexBridgeArg->Append(",");
|
||||
vertexBridgeArg->Append(vertexMethodArgs->ToString());
|
||||
|
||||
|
||||
DrLogI("ManagedWrapperVertex: Calling into Vertex Bridge to invoke Vertex Entry: %s", GetArgument(3));
|
||||
DrLogging::FlushLog();
|
||||
|
||||
|
|
@ -260,10 +266,20 @@ void ManagedWrapperVertex::Main(WorkQueue* workQueue,
|
|||
//
|
||||
try
|
||||
{
|
||||
System::Console::WriteLine("Assembly name " + bridgeAssemblyPartialName);
|
||||
System::Reflection::Assembly ^vertexBridgeAsm = System::Reflection::Assembly::LoadWithPartialName(bridgeAssemblyPartialName);
|
||||
System::Type ^vertexBridgeType = vertexBridgeAsm->GetType(gcnew System::String(bridgeClassName));
|
||||
System::Reflection::MethodInfo ^vertexBridgeMethod = vertexBridgeType->GetMethod(gcnew System::String(bridgeMethodName),
|
||||
System::Reflection::Assembly ^vertexBridgeAsm;
|
||||
try
|
||||
{
|
||||
vertexBridgeAsm = System::Reflection::Assembly::Load(bridgeAssemblyName);
|
||||
}
|
||||
catch (System::Exception ^ie)
|
||||
{
|
||||
DrLogI("ManagedWrapperVertex: Failed to load assembly %s: %s", bridgeAssemblyName, ie->ToString());
|
||||
System::String^ asmLoc = System::IO::Path::Combine("..", bridgeAssemblyName + ".dll");
|
||||
vertexBridgeAsm = System::Reflection::Assembly::LoadFrom(asmLoc);
|
||||
}
|
||||
System::Type ^vertexBridgeType = vertexBridgeAsm->GetType(bridgeClassName);
|
||||
System::Reflection::MethodInfo ^vertexBridgeMethod
|
||||
= vertexBridgeType->GetMethod(bridgeMethodName,
|
||||
static_cast<System::Reflection::BindingFlags>(System::Reflection::BindingFlags::NonPublic |
|
||||
System::Reflection::BindingFlags::Static));
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -21,13 +21,13 @@
|
|||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>StaticLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -18,19 +18,20 @@
|
|||
<Keyword>Win32Proj</Keyword>
|
||||
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\..\</SolutionDir>
|
||||
<RestorePackages>true</RestorePackages>
|
||||
<ProjectName>VertexHostBody</ProjectName>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>Application</ConfigurationType>
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>true</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -56,6 +57,13 @@
|
|||
<CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
|
||||
<CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<NuGetPackageImportStamp>26f18b05</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<TargetName>Microsoft.Research.Dryad.$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<TargetName>Microsoft.Research.Dryad.$(ProjectName)</TargetName>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Midl>
|
||||
|
|
@ -112,6 +120,7 @@
|
|||
</PostBuildEvent>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemGroup>
|
||||
<ClCompile Include="version.cpp" />
|
||||
<ClCompile Include="vertexHost.cpp" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
|
|
@ -124,13 +133,13 @@
|
|||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
</ImportGroup>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<ItemGroup>
|
||||
<ClCompile Include="vertexHost.cpp" />
|
||||
<ClCompile Include="version.cpp" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="app.config" />
|
||||
|
|
|
|||
|
|
@ -1,26 +1,42 @@
|
|||
<?xml version="1.0" encoding="utf-8" ?>
|
||||
<configuration>
|
||||
<runtime>
|
||||
<gcServer enabled="true"></gcServer>
|
||||
<gcConcurrent enabled="false"></gcConcurrent>
|
||||
<generatePublisherEvidence enabled="false"/>
|
||||
<gcAllowVeryLargeObjects enabled="true" />
|
||||
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.1.0" newVersion="5.6.1.0" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-5.6.2.0" newVersion="5.6.2.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-3.1.0.1" newVersion="3.1.0.1" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-4.2.1.0" newVersion="4.2.1.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.Hadoop.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight.Framework" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
<dependentAssembly>
|
||||
<assemblyIdentity name="Microsoft.WindowsAzure.Management.HDInsight" publicKeyToken="31bf3856ad364e35" culture="neutral" />
|
||||
<bindingRedirect oldVersion="0.0.0.0-1.2.1.1" newVersion="1.2.1.1" />
|
||||
</dependentAssembly>
|
||||
</assemblyBinding>
|
||||
</runtime>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
|
||||
</packages>
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
Copyright (c) Microsoft Corporation
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License
|
||||
at http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
|
||||
THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
|
||||
EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF
|
||||
TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
|
||||
|
||||
|
||||
See the Apache Version 2.0 License for specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
*/
|
||||
|
||||
#ifdef _MANAGED
|
||||
#pragma managed
|
||||
|
||||
using namespace System::Reflection;
|
||||
using namespace System::Runtime::InteropServices;
|
||||
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad.VertexHost")];
|
||||
[assembly: AssemblyDescription("")];
|
||||
[assembly: AssemblyConfiguration("")];
|
||||
[assembly: AssemblyCompany("Microsoft Corporation")];
|
||||
[assembly: AssemblyProduct("Microsoft.Research.Dryad.VertexHost")];
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
|
||||
[assembly: AssemblyTrademark("")];
|
||||
[assembly: AssemblyCulture("")];
|
||||
|
||||
[assembly: ComVisible(false)];
|
||||
|
||||
[assembly: AssemblyVersion("0.2.1.0")];
|
||||
[assembly: AssemblyFileVersion("0.2.1.0")];
|
||||
#endif
|
||||
|
|
@ -174,27 +174,27 @@ void SetLoggingLevel()
|
|||
GetLoggingFileName(logFileName);
|
||||
DrLogging::Initialize(logFileName);
|
||||
|
||||
WCHAR traceLevel [MAX_PATH];
|
||||
HRESULT hr = DrGetEnvironmentVariable(L"DRYAD_TRACE_LEVEL", traceLevel);
|
||||
WCHAR loggingLevel [MAX_PATH];
|
||||
HRESULT hr = DrGetEnvironmentVariable(L"DRYAD_LOGGING_LEVEL", loggingLevel);
|
||||
if(hr == DrError_OK)
|
||||
{
|
||||
if(wcscmp(traceLevel, L"OFF") == 0)
|
||||
if(wcscmp(loggingLevel, L"OFF") == 0)
|
||||
{
|
||||
DrLogging::SetLoggingLevel(LogLevel_Off);
|
||||
}
|
||||
else if(wcscmp(traceLevel, L"CRITICAL") == 0)
|
||||
else if(wcscmp(loggingLevel, L"CRITICAL") == 0)
|
||||
{
|
||||
DrLogging::SetLoggingLevel(LogLevel_Assert);
|
||||
}
|
||||
else if(wcscmp(traceLevel, L"ERROR") == 0)
|
||||
else if(wcscmp(loggingLevel, L"ERROR") == 0)
|
||||
{
|
||||
DrLogging::SetLoggingLevel(LogLevel_Error);
|
||||
}
|
||||
else if(wcscmp(traceLevel, L"WARN") == 0)
|
||||
else if(wcscmp(loggingLevel, L"WARN") == 0)
|
||||
{
|
||||
DrLogging::SetLoggingLevel(LogLevel_Warning);
|
||||
}
|
||||
else if(wcscmp(traceLevel, L"INFO") == 0)
|
||||
else if(wcscmp(loggingLevel, L"INFO") == 0)
|
||||
{
|
||||
DrLogging::SetLoggingLevel(LogLevel_Info);
|
||||
}
|
||||
|
|
@ -249,16 +249,24 @@ static void ExceptionHandler(System::Object^ sender, System::UnhandledExceptionE
|
|||
//
|
||||
// Start up vertex host
|
||||
//
|
||||
[System::Security::SecurityCriticalAttribute]
|
||||
[System::Runtime::ExceptionServices::HandleProcessCorruptedStateExceptionsAttribute]
|
||||
#if defined(_AMD64_)
|
||||
int wmain(int argc, wchar_t* wargv[])
|
||||
#else
|
||||
int __cdecl wmain(int argc, wchar_t* wargv[])
|
||||
#endif
|
||||
public ref class VertexHost
|
||||
{
|
||||
public:
|
||||
[System::Security::SecurityCriticalAttribute]
|
||||
[System::Runtime::ExceptionServices::HandleProcessCorruptedStateExceptionsAttribute]
|
||||
static int Main(array<System::String^>^ managedArgs)
|
||||
{
|
||||
try
|
||||
{
|
||||
int argc = managedArgs->Length;
|
||||
wchar_t** wargv = new wchar_t*[argc+1];
|
||||
for (int i=0; i<argc; ++i)
|
||||
{
|
||||
pin_ptr<const wchar_t> wch = PtrToStringChars(managedArgs[i]);
|
||||
wargv[i] = _wcsdup(wch);
|
||||
}
|
||||
wargv[argc] = NULL;
|
||||
|
||||
//
|
||||
// Enable logging based on environment variable
|
||||
//
|
||||
|
|
@ -276,7 +284,7 @@ int __cdecl wmain(int argc, wchar_t* wargv[])
|
|||
//
|
||||
// trace for startup
|
||||
//
|
||||
DrLogI("Vertex Host starting");
|
||||
DrLogE("Vertex Host starting");
|
||||
|
||||
//
|
||||
// Get environment variable to know whether to break into debugger
|
||||
|
|
@ -295,6 +303,12 @@ int __cdecl wmain(int argc, wchar_t* wargv[])
|
|||
char** argv;
|
||||
DrGetUtf8CommandArgs(argc, wargv, &argv);
|
||||
|
||||
for (int i=0; i<managedArgs->Length; ++i)
|
||||
{
|
||||
free(wargv[i]);
|
||||
}
|
||||
delete [] wargv;
|
||||
|
||||
//
|
||||
// Initialize the dryad communication layer with the command line arguments
|
||||
//
|
||||
|
|
@ -346,7 +360,8 @@ int __cdecl wmain(int argc, wchar_t* wargv[])
|
|||
DrLogA("Unhandled exception: %s", DrString(e->ToString()).GetChars());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// Simple data class which contains the byte array and its length.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" />
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" />
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
|
|
@ -25,13 +28,13 @@
|
|||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CLRSupport>Pure</CLRSupport>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<CLRSupport>Pure</CLRSupport>
|
||||
<PlatformToolset>v110</PlatformToolset>
|
||||
<PlatformToolset>v120</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
|
||||
<ImportGroup Label="ExtensionSettings">
|
||||
|
|
@ -59,6 +62,7 @@
|
|||
<CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Microsoft.Research.Dryad</TargetName>
|
||||
<TargetName Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Microsoft.Research.Dryad</TargetName>
|
||||
<NuGetPackageImportStamp>46822e65</NuGetPackageImportStamp>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Midl>
|
||||
|
|
@ -223,13 +227,22 @@
|
|||
</ItemGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
|
||||
<ImportGroup Label="ExtensionTargets">
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" />
|
||||
<Import Project="..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets" Condition="Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" />
|
||||
</ImportGroup>
|
||||
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
|
||||
<PropertyGroup>
|
||||
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
|
||||
</PropertyGroup>
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.0.7.1-beta\build\Microsoft.Research.Peloponnese.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Shared.0.8.1-beta\build\Microsoft.Research.Peloponnese.Shared.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Hadoop.0.8.1-beta\build\Microsoft.Research.Peloponnese.Hadoop.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.Utils.0.8.1-beta\build\Microsoft.Research.Peloponnese.Utils.targets'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.props'))" />
|
||||
<Error Condition="!Exists('..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Research.Peloponnese.AzureStorage.0.8.1-beta\build\Microsoft.Research.Peloponnese.AzureStorage.targets'))" />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
@ -5,10 +5,6 @@
|
|||
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
|
||||
<Extensions>h;hpp;hxx;hm;inl;inc;xsd</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="Resource Files">
|
||||
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
|
||||
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx</Extensions>
|
||||
</Filter>
|
||||
<Filter Include="Source Files">
|
||||
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
|
||||
<Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ limitations under the License.
|
|||
|
||||
#pragma once
|
||||
|
||||
using namespace Microsoft::Research::Peloponnese::Storage;
|
||||
using namespace Microsoft::Research::Peloponnese::Azure;
|
||||
|
||||
DRCLASS(DrAzureInputStream) : public DrInputStream
|
||||
{
|
||||
|
|
|
|||
|
|
@ -129,37 +129,168 @@ DrHdfsInputStream::DrHdfsInputStream()
|
|||
m_hdfsInstance = DrNull;
|
||||
}
|
||||
|
||||
HRESULT DrHdfsInputStream::Open(DrUniversePtr universe, DrNativeString streamUri)
|
||||
HRESULT DrHdfsInputStream::Open(DrUniversePtr universe, DrNativeString streamUri, DrNativeString recordType)
|
||||
{
|
||||
DrString uri = DrString(streamUri);
|
||||
DrString record = DrString(recordType);
|
||||
|
||||
DrLogI("Opening instance for %s", uri.GetChars());
|
||||
DrLogI("Opening instance for %s record type %s", uri.GetChars(), record.GetChars());
|
||||
|
||||
return OpenInternal(universe, uri);
|
||||
return OpenInternal(universe, uri, record);
|
||||
}
|
||||
|
||||
|
||||
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri)
|
||||
#ifdef _MANAGED
|
||||
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType)
|
||||
{
|
||||
m_streamUri = streamUri;
|
||||
HRESULT err = S_OK;
|
||||
|
||||
#ifdef _MANAGED
|
||||
|
||||
try
|
||||
{
|
||||
#endif
|
||||
DrLogI("Opening instance for %s: %s", streamUri.GetChars(), recordType.GetChars());
|
||||
m_hdfsInstance = GetHdfsServiceInstance(streamUri);
|
||||
|
||||
HdfsFileInfo^ stream = m_hdfsInstance->GetFileInfo(streamUri.GetString(), true);
|
||||
m_fileNameArray = stream->fileNameArray;
|
||||
UInt32 totalPartitionCount;
|
||||
|
||||
if (recordType.Compare("Microsoft.Research.DryadLinq.LineRecord") == 0)
|
||||
{
|
||||
DrLogI("Getting block-level file info for %s", streamUri.GetChars());
|
||||
totalPartitionCount = static_cast<UInt32>(stream->blockArray->Length);
|
||||
}
|
||||
else
|
||||
{
|
||||
DrLogI("Getting file info for %s", streamUri.GetChars());
|
||||
totalPartitionCount = m_fileNameArray->Length;
|
||||
}
|
||||
|
||||
DrLogI("Partition count %d", totalPartitionCount);
|
||||
|
||||
/* Allocate these arrays even if they're size 0, to avoid
|
||||
NullReferenceException later */
|
||||
m_affinity = DrNew DrAffinityArray(totalPartitionCount);
|
||||
m_partOffsets = DrNew DrUINT64Array(totalPartitionCount);
|
||||
m_partFileIds = DrNew DrUINT32Array(totalPartitionCount);
|
||||
|
||||
if (recordType.Compare("Microsoft.Research.DryadLinq.LineRecord") == 0)
|
||||
{
|
||||
for (UINT32 i = 0; i < totalPartitionCount; ++i)
|
||||
{
|
||||
HdfsBlockInfo^ partition = stream->blockArray[i];
|
||||
m_affinity[i] = DrNew DrAffinity();
|
||||
m_affinity[i]->SetWeight(partition->Size);
|
||||
m_partOffsets[i] = partition->Offset;
|
||||
m_partFileIds[i] = partition->fileIndex;
|
||||
|
||||
for (int j = 0; j < partition->Hosts->Length; ++j)
|
||||
{
|
||||
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
|
||||
if (location != DrNull)
|
||||
{
|
||||
m_affinity[i]->AddLocality(location);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
int fileBlockIndex = 0;
|
||||
for (UINT32 i = 0; i < totalPartitionCount; ++i)
|
||||
{
|
||||
m_partOffsets[i] = 0;
|
||||
m_partFileIds[i] = i;
|
||||
|
||||
HdfsBlockInfo^ partition = stream->blockArray[fileBlockIndex];
|
||||
DrAssert(partition->fileIndex == i);
|
||||
|
||||
long long fileSize = partition->Size;
|
||||
|
||||
HashSet<DrResourceRef>^ locations = DrNew HashSet<DrResourceRef>();
|
||||
for (int j = 0; j < partition->Hosts->Length; ++j)
|
||||
{
|
||||
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
|
||||
if (location != DrNull)
|
||||
{
|
||||
locations->Add(location);
|
||||
}
|
||||
}
|
||||
|
||||
++fileBlockIndex;
|
||||
|
||||
while (fileBlockIndex < stream->blockArray->Length && stream->blockArray[fileBlockIndex]->fileIndex == i)
|
||||
{
|
||||
partition = stream->blockArray[fileBlockIndex];
|
||||
fileSize += partition->Size;
|
||||
|
||||
if (locations->Count > 0)
|
||||
{
|
||||
HashSet<DrResourceRef>^ newLocations = DrNew HashSet<DrResourceRef>();
|
||||
for (int j = 0; j < partition->Hosts->Length; ++j)
|
||||
{
|
||||
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
|
||||
if (location != DrNull)
|
||||
{
|
||||
newLocations->Add(location);
|
||||
}
|
||||
}
|
||||
|
||||
locations->IntersectWith(newLocations);
|
||||
}
|
||||
}
|
||||
|
||||
m_affinity[i] = DrNew DrAffinity();
|
||||
m_affinity[i]->SetWeight(fileSize);
|
||||
|
||||
System::Text::StringBuilder^ locationText;
|
||||
if (locations->Count > 0)
|
||||
{
|
||||
locationText = gcnew System::Text::StringBuilder("File " + m_fileNameArray[i] + " merged locations:");
|
||||
}
|
||||
else
|
||||
{
|
||||
locationText = gcnew System::Text::StringBuilder("File " + m_fileNameArray[i] + " no shared locations");
|
||||
}
|
||||
|
||||
HashSet<DrResourceRef>::Enumerator^ enumerator = locations->GetEnumerator();
|
||||
while (enumerator->MoveNext())
|
||||
{
|
||||
m_affinity[i]->AddLocality(enumerator->Current);
|
||||
locationText->Append(" ");
|
||||
locationText->Append(enumerator->Current->GetName().GetString());
|
||||
}
|
||||
|
||||
DrString locationLog(locationText->ToString());
|
||||
DrLogI("%s", locationLog.GetChars());
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (System::Exception ^e)
|
||||
{
|
||||
m_error = e->ToString();
|
||||
DrString msg(m_error);
|
||||
DrLogE("Got HDFS exception %s", msg.GetChars());
|
||||
err = System::Runtime::InteropServices::Marshal::GetHRForException(e);
|
||||
}
|
||||
finally
|
||||
{
|
||||
// TODO: How do we clean this up?
|
||||
//hdfsInstance->Discard();
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
#else
|
||||
HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType)
|
||||
{
|
||||
m_streamUri = streamUri;
|
||||
HRESULT err = S_OK;
|
||||
|
||||
|
||||
DrLogI("Opening instance for %s", streamUri.GetChars());
|
||||
m_hdfsInstance = GetHdfsServiceInstance(streamUri);
|
||||
|
||||
#ifdef _MANAGED
|
||||
DrLogI("Getting file info for %s", streamUri.GetChars());
|
||||
HdfsFileInfo^ stream = m_hdfsInstance->GetFileInfo(streamUri.GetString(), true);
|
||||
m_fileNameArray = stream->fileNameArray;
|
||||
UInt32 totalPartitionCount = static_cast<UInt32>(stream->blockArray->Length);
|
||||
|
||||
#else
|
||||
bool ret = HdfsBridgeNative::Initialize();
|
||||
if (!ret)
|
||||
{
|
||||
|
|
@ -195,7 +326,6 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
|
|||
totalPartitionCount = fs.GetNumberOfBlocks();
|
||||
|
||||
m_fileNameArray = (const char **)fs.GetFileNameArray();
|
||||
#endif
|
||||
|
||||
DrLogI("Partition count %d", totalPartitionCount);
|
||||
|
||||
|
|
@ -207,21 +337,13 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
|
|||
|
||||
for (UINT32 i=0; i<totalPartitionCount; ++i)
|
||||
{
|
||||
#ifdef _MANAGED
|
||||
HdfsBlockInfo^ partition = stream->blockArray[i];
|
||||
#else
|
||||
HdfsBridgeNative::HdfsBlockLocInfo* partition = fs.GetBlockInfo(i);
|
||||
#endif
|
||||
m_affinity[i] = DrNew DrAffinity();
|
||||
m_affinity[i]->SetWeight(partition->Size);
|
||||
m_partOffsets[i] = partition->Offset;
|
||||
m_partFileIds[i] = partition->fileIndex;
|
||||
|
||||
#ifdef _MANAGED
|
||||
for (int j = 0; j < partition->Hosts->Length; ++j)
|
||||
#else
|
||||
for (int j = 0; j < partition->numberOfHosts; ++j)
|
||||
#endif
|
||||
{
|
||||
DrResourceRef location = universe->LookUpResource(partition->Hosts[j]);
|
||||
if (location != DrNull)
|
||||
|
|
@ -229,28 +351,12 @@ HRESULT DrHdfsInputStream::OpenInternal(DrUniversePtr universe, DrString streamU
|
|||
m_affinity[i]->AddLocality(location);
|
||||
}
|
||||
}
|
||||
#ifndef _MANAGED
|
||||
delete partition;
|
||||
#endif
|
||||
}
|
||||
#ifdef _MANAGED
|
||||
}
|
||||
catch (System::Exception ^e)
|
||||
{
|
||||
m_error = e->ToString();
|
||||
DrString msg(m_error);
|
||||
DrLogE("Got HDFS exception %s", msg.GetChars());
|
||||
err = System::Runtime::InteropServices::Marshal::GetHRForException(e);
|
||||
}
|
||||
finally
|
||||
{
|
||||
// TODO: How do we clean this up?
|
||||
//hdfsInstance->Discard();
|
||||
}
|
||||
#endif
|
||||
|
||||
return err;
|
||||
}
|
||||
#endif
|
||||
|
||||
DrNativeString DrHdfsInputStream::GetError()
|
||||
{
|
||||
|
|
@ -468,6 +574,25 @@ HRESULT DrHdfsOutputStream::FinalizeSuccessfulParts(DrOutputPartitionArrayRef pa
|
|||
errorText.SetF("%s", DrString(m_error).GetChars());
|
||||
return E_FAIL;
|
||||
}
|
||||
|
||||
String^ userName = Environment::GetEnvironmentVariable("USER");
|
||||
if (userName == nullptr)
|
||||
{
|
||||
userName = Environment::UserName;
|
||||
}
|
||||
try
|
||||
{
|
||||
m_hdfsInstance->SetOwnerAndPermission(m_baseUri, userName, nullptr, Convert::ToInt16("0755", 8));
|
||||
}
|
||||
catch (Exception^ e)
|
||||
{
|
||||
DrString drDst(m_baseUri);
|
||||
DrString err(e->ToString());
|
||||
DrLogE("Can't set %s permissions finalizing HDFS output: %s", drDst.GetChars(), err.GetChars());
|
||||
m_error = "Can't set " + m_baseUri + " permissions finalizing HDFS output: " + e->ToString();
|
||||
errorText.SetF("%s", DrString(m_error).GetChars());
|
||||
return E_FAIL;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ DRCLASS(DrHdfsInputStream) : public DrInputStream
|
|||
{
|
||||
public:
|
||||
DrHdfsInputStream();
|
||||
HRESULT Open(DrUniversePtr universe, DrNativeString streamUri);
|
||||
HRESULT OpenInternal(DrUniversePtr universe, DrString streamUri);
|
||||
HRESULT Open(DrUniversePtr universe, DrNativeString streamUri, DrNativeString recordType);
|
||||
HRESULT OpenInternal(DrUniversePtr universe, DrString streamUri, DrString recordType);
|
||||
DrNativeString GetError();
|
||||
|
||||
virtual DrString GetStreamName() DROVERRIDE;
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
|
|||
DrStringR remoteName,
|
||||
DrPartitionInputStream::OverridePtr over,
|
||||
bool mustOverride,
|
||||
bool pathIsRooted,
|
||||
DrString line,
|
||||
DrUniversePtr universe)
|
||||
{
|
||||
|
|
@ -112,9 +113,17 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
|
|||
sep = lineCopy.IndexOfChar(',');
|
||||
if (sep == DrStr_InvalidIndex)
|
||||
{
|
||||
DrLogW("Malformed line %s: no list of machines", line.GetChars());
|
||||
int n = sscanf_s(lineCopy.GetChars(), "%I64u", &parsedSize);
|
||||
if (n != 1)
|
||||
{
|
||||
DrLogW("Malformed line %s: can't parse size", line.GetChars());
|
||||
return false;
|
||||
}
|
||||
|
||||
affinity->SetWeight(parsedSize);
|
||||
|
||||
lineCopy = DrString("");
|
||||
}
|
||||
else
|
||||
{
|
||||
DrString partitionSizeString;
|
||||
|
|
@ -134,11 +143,17 @@ static bool ParseReplicatedFromPartitionLine(int partitionNumber,
|
|||
}
|
||||
|
||||
if (lineCopy.GetCharsLength() == 0)
|
||||
{
|
||||
if (!pathIsRooted || mustOverride)
|
||||
{
|
||||
DrLogW("Malformed line %s: no partition machines", line.GetChars());
|
||||
return false;
|
||||
}
|
||||
|
||||
remoteName.Set(" %Invalid% ");
|
||||
return true;
|
||||
}
|
||||
|
||||
int numberOfReplicas = 0;
|
||||
while (lineCopy.GetCharsLength() > 0)
|
||||
{
|
||||
|
|
@ -231,6 +246,12 @@ HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString st
|
|||
mustOverride = true;
|
||||
}
|
||||
|
||||
bool pathIsRooted = false;
|
||||
if (m_pathNameOnComputer.IndexOfChar(':') != DrStr_InvalidIndex)
|
||||
{
|
||||
pathIsRooted = true;
|
||||
}
|
||||
|
||||
int numberOfParts;
|
||||
int n = sscanf_s(partitionSizeLine.GetChars(), "%d", &numberOfParts);
|
||||
if (n != 1)
|
||||
|
|
@ -280,6 +301,7 @@ HRESULT DrPartitionInputStream::OpenInternal(DrUniversePtr universe, DrString st
|
|||
remoteName,
|
||||
m_override[i],
|
||||
mustOverride,
|
||||
pathIsRooted,
|
||||
partitionLine,
|
||||
universe) == false)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -19,5 +19,22 @@ limitations under the License.
|
|||
*/
|
||||
|
||||
#ifdef _MANAGED
|
||||
[assembly:System::Runtime::InteropServices::ComVisible(false)];
|
||||
#pragma managed
|
||||
|
||||
using namespace System::Reflection;
|
||||
using namespace System::Runtime::InteropServices;
|
||||
|
||||
[assembly: AssemblyTitle("Microsoft.Research.Dryad")];
|
||||
[assembly: AssemblyDescription("")];
|
||||
[assembly: AssemblyConfiguration("")];
|
||||
[assembly: AssemblyCompany("Microsoft Corporation")];
|
||||
[assembly: AssemblyProduct("Microsoft.Research.Dryad")];
|
||||
[assembly: AssemblyCopyright("Copyright © Microsoft Corporation. All rights reserved.")];
|
||||
[assembly: AssemblyTrademark("")];
|
||||
[assembly: AssemblyCulture("")];
|
||||
|
||||
[assembly: ComVisible(false)];
|
||||
|
||||
[assembly: AssemblyVersion("1.0.0.0")];
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")];
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -570,7 +570,7 @@ DrString DrClusterInternal::TranslateFileToURI(DrString leafName, DrString direc
|
|||
}
|
||||
|
||||
void DrClusterInternal::ScheduleProcess(DrAffinityListRef affinities,
|
||||
DrString name, DrString commandLine,
|
||||
DrString name, DrString commandLineArgs,
|
||||
DrProcessTemplatePtr processTemplate,
|
||||
DrPSRListenerPtr listener)
|
||||
{
|
||||
|
|
@ -604,7 +604,7 @@ void DrClusterInternal::ScheduleProcess(DrAffinityListRef affinities,
|
|||
DrLogI("Starting schedule process for %s.%s",
|
||||
processTemplate->GetProcessClass().GetChars(), name.GetChars());
|
||||
|
||||
IProcess^ rawProcess = m_cluster->NewProcess(process, commandLine.GetString());
|
||||
IProcess^ rawProcess = m_cluster->NewProcess(process, processTemplate->GetCommandLineBase().GetString(), commandLineArgs.GetString());
|
||||
process->SetProcess(rawProcess);
|
||||
|
||||
DrLogI("Assigned GUID %s to process for %s.%s",
|
||||
|
|
|
|||
|
|
@ -45,9 +45,10 @@ public:
|
|||
DrResourcePtr srcResource, DrResourcePtr dstResource, int compressionMode) = 0;
|
||||
|
||||
virtual void ScheduleProcess(DrAffinityListRef affinities,
|
||||
DrString name, DrString commandLine,
|
||||
DrString name, DrString commandLineArgs,
|
||||
DrProcessTemplatePtr processTemplate,
|
||||
DrPSRListenerPtr listener) = 0;
|
||||
|
||||
virtual void CancelScheduleProcess(DrProcessHandlePtr process) = 0;
|
||||
|
||||
virtual void WaitForStateChange(DrProcessHandlePtr process, DrPSRListenerPtr listener) = 0;
|
||||
|
|
|
|||
|
|
@ -165,9 +165,10 @@ public:
|
|||
DrResourcePtr srcResource, DrResourcePtr dstResource, int compressionMode) DROVERRIDE;
|
||||
|
||||
virtual void ScheduleProcess(DrAffinityListRef affinities,
|
||||
DrString name, DrString commandLine,
|
||||
DrString name, DrString commandLineArgs,
|
||||
DrProcessTemplatePtr processTemplate,
|
||||
DrPSRListenerPtr listener) DROVERRIDE;
|
||||
|
||||
virtual void CancelScheduleProcess(DrProcessHandlePtr process) DROVERRIDE;
|
||||
|
||||
virtual void WaitForStateChange(DrProcessHandlePtr process, DrPSRListenerPtr listener) DROVERRIDE;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.Research.Peloponnese" version="0.7.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.AzureStorage" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Hadoop" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Shared" version="0.8.1-beta" targetFramework="Native" />
|
||||
<package id="Microsoft.Research.Peloponnese.Utils" version="0.8.1-beta" targetFramework="Native" />
|
||||
</packages>
|
||||
|
|
@ -24,7 +24,8 @@ limitations under the License.
|
|||
#include <msclr\lock.h>
|
||||
|
||||
using namespace System::IO;
|
||||
using namespace Microsoft::Research::Peloponnese::Storage;
|
||||
using namespace Microsoft::Research::Peloponnese::Azure;
|
||||
using namespace Microsoft::Research::Peloponnese::Hdfs;
|
||||
using namespace msclr;
|
||||
|
||||
//
|
||||
|
|
@ -126,7 +127,7 @@ public:
|
|||
DrCalypsoReporter::DrCalypsoReporter(DrNativeString uriString)
|
||||
{
|
||||
System::Uri^ uri = DrNew System::Uri(uriString);
|
||||
if (uri->Scheme == AzureUtils::BlobScheme)
|
||||
if (uri->Scheme == Utils::BlobScheme)
|
||||
{
|
||||
m_logStream = DrNew AzureLogAppendStream(uri, 0x20, false, false, gcnew PeloponneseLogger());
|
||||
m_flushInterval = 1000;
|
||||
|
|
|
|||
|
|
@ -470,8 +470,7 @@ void DrCohort::StartProcess(DrGraphPtr graph, int version)
|
|||
processName.SetF("%s v.%d", m_description.GetChars(), version);
|
||||
|
||||
DrString commandLine;
|
||||
commandLine.SetF("%s --noredirect --startfrompn %d",
|
||||
m_processTemplate->GetCommandLineBase().GetChars(), m_list->Size());
|
||||
commandLine.SetF("--noredirect --startfrompn %d", m_list->Size());
|
||||
|
||||
for (i=0; i<m_list->Size(); ++i)
|
||||
{
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue