From 5c260ffe0b865bdb35e6a99ff37f36a7d07ccf2d Mon Sep 17 00:00:00 2001 From: Dennis Fetterly Date: Tue, 10 Jun 2014 16:05:37 -0700 Subject: [PATCH] version 0.1.5 release --- .gitignore | 2 + ClusterInterface/ClusterInterface.csproj | 72 +- ClusterInterface/HttpScheduler.cs | 4 +- ClusterInterface/Interfaces.cs | 3 +- ClusterInterface/Process.cs | 23 +- ClusterInterface/Properties/AssemblyInfo.cs | 6 +- ClusterInterface/app.config | 2 +- ClusterInterface/packages.config | 24 +- Dryad.sln | 6 - .../DryadLinqGraphManager.csproj | 58 +- .../Properties/AssemblyInfo.cs | 6 +- DryadLinqGraphManager/app.config | 10 +- DryadLinqGraphManager/packages.config | 24 +- DryadLinqTests/App.config | 24 +- DryadLinqTests/ApplyAndForkTests.cs | 370 +++- DryadLinqTests/BasicAPITests.cs | 1532 ++++++++++------- .../DryadLinqFSharpTests/App.config | 53 + .../DryadLinqFSharpTests.fsproj | 219 +++ .../DryadLinqFSharpTests/Program.fs | 46 + .../DryadLinqFSharpTests/packages.config | 23 + DryadLinqTests/DryadLinqTests.csproj | 178 +- DryadLinqTests/DryadLinqTests.sln | 46 + DryadLinqTests/GroupByReduceTests.cs | 942 ++++++---- DryadLinqTests/MiscBugFixTests.cs | 948 ++++++++++ DryadLinqTests/Program.cs | 57 +- DryadLinqTests/Properties/AssemblyInfo.cs | 27 +- .../RangePartitionAPICoverageTests.cs | 817 ++++++++- DryadLinqTests/SerializationTestTypes.cs | 576 +++++++ DryadLinqTests/SerializationTests.cs | 361 ++++ DryadLinqTests/SimpleTests.cs | 89 + DryadLinqTests/TestLog.cs | 99 ++ DryadLinqTests/TypesInQueryTests.cs | 496 ++++++ DryadLinqTests/Utils.cs | 254 ++- DryadLinqTests/Validate.cs | 267 --- DryadLinqTests/WordCount.cs | 59 + DryadLinqTests/packages.config | 24 +- .../VertexHost/system/channel/channel.vcxproj | 8 +- .../VertexHost/system/channel/packages.config | 2 +- .../system/dprocess/src/vertexfactory.cpp | 5 +- .../managedchannel/Properties/AssemblyInfo.cs | 6 +- .../system/managedchannel/app.config | 2 +- .../managedchannel/managedchannel.csproj | 58 +- .../system/managedchannel/packages.config | 24 +- .../ManagedWrapperVertex.cpp | 17 +- .../VertexHost/vertex/vertexHost/app.config | 8 +- GraphManager/kernel/DrCluster.cpp | 6 +- GraphManager/kernel/DrCluster.h | 3 +- GraphManager/kernel/DrClusterInternal.h | 3 +- GraphManager/vertex/DrCohort.cpp | 3 +- LinqToDryad/DataProvider.cs | 12 +- LinqToDryad/DryadLinqContext.cs | 65 +- LinqToDryad/DryadLinqFaultCodes.cs | 1 + LinqToDryad/DryadLinqJobExecutor.cs | 4 +- LinqToDryad/DryadLinqQuery.cs | 93 +- LinqToDryad/DryadLinqQueryGen.cs | 1 - LinqToDryad/DryadLinqQueryNode.cs | 6 +- LinqToDryad/DryadLinqQueryable.cs | 5 +- LinqToDryad/LinqToDryad.csproj | 58 +- LinqToDryad/LocalJobSubmission.cs | 19 +- LinqToDryad/Properties/AssemblyInfo.cs | 10 +- LinqToDryad/TypeSystem.cs | 73 +- LinqToDryad/YarnJobSubmission.cs | 32 +- LinqToDryad/app.config | 2 +- LinqToDryad/packages.config | 24 +- LocalScheduler/LocalScheduler.csproj | 58 +- LocalScheduler/app.config | 10 +- LocalScheduler/packages.config | 24 +- Microsoft.Research.Dryad.nuspec | 10 +- ProcessService/HttpServer.cs | 8 +- ProcessService/ProcessService.cs | 59 +- ProcessService/ProcessService.csproj | 58 +- ProcessService/Properties/AssemblyInfo.cs | 6 +- ProcessService/app.config | 2 +- ProcessService/packages.config | 24 +- .../Building the Job Browser.aml | 6 +- .../Resources/Running a job on HDInsight.aml | 8 +- XmlDoc/Content/Welcome.aml | 4 +- ...icrosoft.Research.DryadLinq.Samples.nuspec | 4 +- 78 files changed, 6603 insertions(+), 1975 deletions(-) create mode 100644 DryadLinqTests/DryadLinqFSharpTests/App.config create mode 100644 DryadLinqTests/DryadLinqFSharpTests/DryadLinqFSharpTests.fsproj create mode 100644 DryadLinqTests/DryadLinqFSharpTests/Program.fs create mode 100644 DryadLinqTests/DryadLinqFSharpTests/packages.config create mode 100644 DryadLinqTests/DryadLinqTests.sln create mode 100644 DryadLinqTests/MiscBugFixTests.cs create mode 100644 DryadLinqTests/SerializationTestTypes.cs create mode 100644 DryadLinqTests/SerializationTests.cs create mode 100644 DryadLinqTests/SimpleTests.cs create mode 100644 DryadLinqTests/TestLog.cs create mode 100644 DryadLinqTests/TypesInQueryTests.cs delete mode 100644 DryadLinqTests/Validate.cs create mode 100644 DryadLinqTests/WordCount.cs diff --git a/.gitignore b/.gitignore index 88cb93c..1e9f422 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ x64/ build/ [Bb]in/ [Oo]bj/ +/Calypso/packages +/DryadLinqTests/packages diff --git a/ClusterInterface/ClusterInterface.csproj b/ClusterInterface/ClusterInterface.csproj index e2f16ff..e82b177 100644 --- a/ClusterInterface/ClusterInterface.csproj +++ b/ClusterInterface/ClusterInterface.csproj @@ -1,6 +1,6 @@  - + Debug @@ -45,75 +45,81 @@ False ..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll - + False - ..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll + ..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll False - ..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll + ..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll False - ..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll + ..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll False - ..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll + ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll - - ..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll + + False + ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll - - ..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll + + False + ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll False - ..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll + ..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll False - ..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll + ..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll ..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll - - ..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll - - + False - ..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll + ..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll - + False - ..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll + ..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll - + False - ..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll + ..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll + + + False + ..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll False - ..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll + ..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll - + False - ..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll + ..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll - - ..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll + + False + ..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll - - ..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll + + False + ..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll - - ..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll + + False + ..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll @@ -147,10 +153,10 @@ This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - + + - + + \ No newline at end of file diff --git a/DryadLinqTests/DryadLinqFSharpTests/Program.fs b/DryadLinqTests/DryadLinqFSharpTests/Program.fs new file mode 100644 index 0000000..0c6d19b --- /dev/null +++ b/DryadLinqTests/DryadLinqFSharpTests/Program.fs @@ -0,0 +1,46 @@ +// Learn more about F# at http://fsharp.net +// See the 'F# Tutorial' project for more help. + +open System +open System.Collections.Generic +open System.Linq +open System.IO +open Microsoft.Research.DryadLinq + +let context = new DryadLinqContext(1) +let inputUri = @"partfile:///d:/DryadLinqTemp/PartFiles/foo.pt" +let input = context.FromStore(inputUri) + +type Pair = { word : string; count : int } + +let Histogram (source : IQueryable) (k : int) = + let words = source.SelectMany(fun (x : LineRecord) -> x.Line.Split(' ') |> Seq.ofArray) + let groups = words.GroupBy(fun x -> x) + let counts = groups.Select(fun (x : IGrouping) -> { word = x.Key; count = x.Count() }) + let ordered = counts.OrderByDescending(fun x -> x.count) + ordered.Take(k) + +let Histogram1 (source : IQueryable) (k : int) = + query { + for ws in source do + for w in ws.Line.Split(' ') do + groupBy w into g + select { word = g.Key; count = g.Count() } into winfo + sortByDescending winfo.count + take k + } + +let Histogram2 = + query { + for ws in input do + for w in ws.Line.Split(' ') do + groupBy w into g + select { word = g.Key; count = g.Count() } into winfo + sortByDescending winfo.count + take 10 + } + +for x in Histogram1 input 10 do System.Console.WriteLine(x.word + " : " + x.count.ToString()) + +//let foo = Histogram input 10 +//foo.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/xxx.pt").SubmitAndWait() diff --git a/DryadLinqTests/DryadLinqFSharpTests/packages.config b/DryadLinqTests/DryadLinqFSharpTests/packages.config new file mode 100644 index 0000000..ea1f6d8 --- /dev/null +++ b/DryadLinqTests/DryadLinqFSharpTests/packages.config @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/DryadLinqTests/DryadLinqTests.csproj b/DryadLinqTests/DryadLinqTests.csproj index c88877b..0503d31 100644 --- a/DryadLinqTests/DryadLinqTests.csproj +++ b/DryadLinqTests/DryadLinqTests.csproj @@ -1,34 +1,36 @@  - + Debug AnyCPU - {3F74DAD4-5758-4BCF-BEEB-F1898C5BDCCE} + {4EA15885-003F-4B4D-A7DB-C172C2F23161} Exe Properties DryadLinqTests DryadLinqTests v4.5 512 - publish\ - true - Disk - false - Foreground - 7 - Days - false - false - true - 0 - 1.0.0.%2a - false - false - true - ..\ - true + + + x64 + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 true @@ -53,91 +55,110 @@ False - ..\packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll + packages\Microsoft.Data.Edm.5.6.1\lib\net40\Microsoft.Data.Edm.dll False - ..\packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll + packages\Microsoft.Data.OData.5.6.1\lib\net40\Microsoft.Data.OData.dll False - ..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll + packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll - + False - ..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll + packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll - - False - ..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll + + packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.dll - + + packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.Drawing.dll + + + packages\Microsoft.Research.DryadLINQ.MSAGL.3.0.0.1\lib\Microsoft.Msagl.GraphViewerGdi.dll + + False - ..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll + packages\Microsoft.Research.Dryad.0.1.4-beta001\lib\net45\Microsoft.Research.DryadLinq.dll + + + packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll + + + packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll False - ..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll + packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll - - ..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll + + False + packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll - - ..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll + + False + packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll False - ..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll + packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll False - ..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll + packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll - ..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll + packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll - - ..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll - - + False - ..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll + packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll - + False - ..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll + packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll - + False - ..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll + packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll + + + False + packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll False - ..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll + packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll - + False - ..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll + packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll False - ..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll + packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll - + + - - ..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll + + False + packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll - - ..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll + + False + packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll + + False - ..\packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll + packages\System.Spatial.5.6.1\lib\net40\System.Spatial.dll @@ -150,53 +171,38 @@ + + + + + + - + - - - {d33c34cc-6db2-417c-88b7-299830711774} - LinqToDryad - - - - - False - Microsoft .NET Framework 4.5 %28x86 and x64%29 - true - - - False - .NET Framework 3.5 SP1 Client Profile - false - - - False - .NET Framework 3.5 SP1 - false - - - + - - + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - + + + - + + @@ -29,7 +30,7 @@ Dryad provides reliable, distributed computing on thousands of servers for large - + @@ -60,7 +61,6 @@ Dryad provides reliable, distributed computing on thousands of servers for large - - + diff --git a/ProcessService/HttpServer.cs b/ProcessService/HttpServer.cs index 08c066b..01f75ab 100644 --- a/ProcessService/HttpServer.cs +++ b/ProcessService/HttpServer.cs @@ -236,12 +236,14 @@ namespace Microsoft.Research.Dryad.ProcessService try { string commandLine; + string arguments; using (var sr = new System.IO.StreamReader(context.Request.InputStream)) { - commandLine = sr.ReadToEnd(); + commandLine = sr.ReadLine(); + arguments = sr.ReadLine(); } - logger.Log("Received create for process " + processId + " cmdline: " + commandLine); + logger.Log("Received create for process " + processId + " cmdline: " + commandLine + " arguments: " + arguments); if (parent.Create(processId)) { @@ -254,7 +256,7 @@ namespace Microsoft.Research.Dryad.ProcessService await server.ReportError(context, HttpStatusCode.Conflict, "Process " + processId + " already exists"); } - parent.Launch(processId, commandLine); + parent.Launch(processId, commandLine, arguments); return; } diff --git a/ProcessService/ProcessService.cs b/ProcessService/ProcessService.cs index 696ffc7..2ca49d5 100644 --- a/ProcessService/ProcessService.cs +++ b/ProcessService/ProcessService.cs @@ -565,7 +565,42 @@ namespace Microsoft.Research.Dryad.ProcessService return true; } - public void Launch(int processId, string commandLine) + private void SplitCmdLine(string cmdLine, out string cmd, out string args) + { + cmd = ""; + args = ""; + int lastSpacePos = 0; + bool rootedPath = Path.IsPathRooted(cmdLine); + string candPath = cmdLine; + + while (true) + { + if (!rootedPath) + { + candPath = Path.Combine(Environment.CurrentDirectory, candPath); + } + if (File.Exists(candPath)) + { + cmd = candPath; + args = (lastSpacePos > 0) ? cmdLine.Substring(lastSpacePos) : ""; + return; + } + + int spacePos = cmdLine.IndexOf(' ', lastSpacePos); + if (spacePos != -1) + { + candPath = cmdLine.Substring(0, spacePos); + lastSpacePos = spacePos + 1; + } + else + { + break; + } + } + throw new ApplicationException("Couldn't split command line into command line and arguments."); + } + + public void Launch(int processId, string commandLine, string arguments) { ProcessRecord process; lock (processTable) @@ -588,29 +623,23 @@ namespace Microsoft.Research.Dryad.ProcessService startInfo.RedirectStandardError = true; startInfo.WorkingDirectory = Path.Combine(serviceWorkingDirectory, processId.ToString()); logger.Log(String.Format("Working directory: '{0}'", startInfo.WorkingDirectory)); - - string[] args = commandLine.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); - string arg = ""; - for (int i = 1; i < args.Length; i++) - { - arg += args[i] + " "; - } - + // Use either FQ path or path relative to job path - if (Path.IsPathRooted(args[0])) + if (Path.IsPathRooted(commandLine)) { - startInfo.FileName = args[0]; + startInfo.FileName = commandLine; } else { - startInfo.FileName = Path.Combine(serviceWorkingDirectory, args[0]); + startInfo.FileName = Path.Combine(serviceWorkingDirectory, commandLine); } + + startInfo.Arguments = arguments; logger.Log(String.Format("FileName: '{0}'", startInfo.FileName)); - // Add environment variable to vertex host process - startInfo.Arguments = arg; - logger.Log(String.Format("args: '{0}'", arg)); + logger.Log(String.Format("args: '{0}'", arguments)); + // Add environment variable to vertex host process Uri genericUri = new Uri(processServer.BaseURI); Uri localUri = new Uri(genericUri.Scheme + "://localhost:" + genericUri.Port + genericUri.PathAndQuery); string processUpdateURI = localUri.ToString() + processId.ToString(); diff --git a/ProcessService/ProcessService.csproj b/ProcessService/ProcessService.csproj index 4b3731e..8b6c0a7 100644 --- a/ProcessService/ProcessService.csproj +++ b/ProcessService/ProcessService.csproj @@ -1,6 +1,6 @@  - + Debug @@ -47,81 +47,81 @@ False ..\packages\Microsoft.Data.Services.Client.5.6.1\lib\net40\Microsoft.Data.Services.Client.dll - + False - ..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.Hadoop.Client.dll + ..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.Hadoop.Client.dll False - ..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll + ..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.HadoopBridge.dll False - ..\packages\Microsoft.Research.Peloponnese.0.7.2-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll + ..\packages\Microsoft.Research.Peloponnese.0.7.5-beta\lib\net45\Microsoft.Research.Peloponnese.Utils.dll False - ..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.dll + ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll False - ..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.dll + ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll - + False - ..\packages\Microsoft.Bcl.Async.1.0.166\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll + ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll False - ..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.dll + ..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.dll False - ..\packages\Microsoft.WindowsAzure.Common.1.0.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll + ..\packages\Microsoft.WindowsAzure.Common.1.1.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll ..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll False - ..\packages\Microsoft.WindowsAzure.Management.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.dll + ..\packages\Microsoft.WindowsAzure.Management.1.2.0\lib\net40\Microsoft.WindowsAzure.Management.dll - + False - ..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll + ..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll - + False - ..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll + ..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll - + False - ..\packages\Microsoft.Hadoop.Client.1.1.0.7\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll + ..\packages\Microsoft.Hadoop.Client.1.1.1.8\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll False - ..\packages\Microsoft.WindowsAzure.Management.Storage.1.0.0\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll + ..\packages\Microsoft.WindowsAzure.Management.Storage.1.1.1\lib\net40\Microsoft.WindowsAzure.Management.Storage.dll - + False - ..\packages\WindowsAzure.Storage.3.1.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll + ..\packages\WindowsAzure.Storage.4.0.1\lib\net40\Microsoft.WindowsAzure.Storage.dll False - ..\packages\Newtonsoft.Json.6.0.2\lib\net45\Newtonsoft.Json.dll + ..\packages\Newtonsoft.Json.6.0.3\lib\net45\Newtonsoft.Json.dll - + False - ..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Extensions.dll + ..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll - + False - ..\packages\Microsoft.Net.Http.2.2.19\lib\net45\System.Net.Http.Primitives.dll + ..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll @@ -158,10 +158,10 @@ This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - + + - +