Initial commit

CommonCode/AzureUtils.cs

@@ -0,0 +1,104 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//------------------------------------------------------------------------------
+// <summary>
+//      Utils used by L2H for dealing with Azure
+// </summary>
+//------------------------------------------------------------------------------
+
+namespace Microsoft.Research.Dryad
+{
+    using System;
+
+    public class AzureUtils
+    {
+        /// <summary>
+        /// Flag to denote whether process is on Azure or not
+        /// </summary>
+        private static bool? isOnAzure = null;
+
+        /// <summary>
+        /// Name of HPC node
+        /// </summary>
+        private static string hostName = null;
+
+        /// <summary>
+        /// Determine whether current process on Azure or not
+        /// </summary>
+        public static bool IsOnAzure
+        {
+            get
+            {
+                if (isOnAzure == null)
+                {
+                    isOnAzure = Environment.GetEnvironmentVariable("CCP_ONAZURE") != null;
+                }
+
+                return (bool)isOnAzure;
+            }
+        }
+
+        /// <summary>
+        /// Returns name of node or HPC alias if in Azure
+        /// </summary>
+        /// <returns>name to use for current node</returns>
+        public static string CurrentHostName
+        {
+            get
+            {
+                if(string.IsNullOrEmpty(hostName))
+                {
+                    if (Microsoft.Research.Dryad.AzureUtils.IsOnAzure)
+                    {
+                        hostName = Environment.GetEnvironmentVariable(@"HPC_NODE_NAME");
+                        if (string.IsNullOrEmpty(hostName))
+                        {
+                            throw new Exception("Unable to get HPC_NODE_NAME environment variable");
+                        }
+                    }
+                    else
+                    {
+                        hostName = Environment.MachineName;
+                    }
+                }
+
+                return hostName;
+            }
+        }
+
+        /// <summary>
+        /// This needs to be set in the Azure bootstrapper code.
+        /// </summary>
+
+        internal static bool IsDatabaseShared
+        {
+            get
+            {
+                if (AzureUtils.IsOnAzure)
+                {
+                    return (Environment.GetEnvironmentVariable(@"HPC_SHARED_DATABASE", EnvironmentVariableTarget.Machine) != null);
+                }
+
+                return false;
+            }
+        }
+    }
+}

CommonCode/Constants.cs

@@ -0,0 +1,169 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//------------------------------------------------------------------------------
+// <summary>
+//      Constants used by managed code in Dryad
+// </summary>
+//------------------------------------------------------------------------------
+
+namespace Microsoft.Research.Dryad
+{
+    using System;
+
+    internal class Constants
+    {
+        //
+        // Constants for all WCF nettcp bindings
+        //
+        public const int MaxReceivedMessageSize = 16 * 1024 * 1024;
+        public const int MaxBufferPoolSize = 16 * 1024 * 1024;
+        public const int MaxConnections = 1024;
+        public const int ListenBacklog = 256;
+        public static readonly TimeSpan SendTimeout = new TimeSpan(0, 2, 0);
+        public static readonly TimeSpan ReceiveTimeout = new TimeSpan(0, 10, 0);
+        public static readonly TimeSpan VertexSendTimeout = new TimeSpan(0, 1, 0);
+
+        // For Seal and Delete Node, use 6 minutes for the WCF timeout because of the 5 minute SQL timeout for the DB call
+        // Otherwise, default to 2 minutes
+        // TODO: Post-SP3, re-examine longest running operations, as they slow down service failure and failover time
+        public static readonly TimeSpan DscOperationTimeout = new TimeSpan(0, 2, 0);
+        public static readonly TimeSpan DscExtendedOperationTimeout = new TimeSpan(0, 6, 0);
+        
+        public static readonly String DryadConnectionString = String.Empty;
+      
+        public const string CommonRegistryPath = @"SOFTWARE\Microsoft\HPC";
+        public const string HpcSchedulerNameString = "ClusterName";
+        public const string HpcInstallPath = "BinDir";
+        public const string DscServerName = "DscServiceNodeName";
+        public const string DscConnectionFormat = @"net.tcp://{0}:{1}/HpcDsc/Service/DscService";
+        public const string DscServiceDefaultScheme = "hpcdsc";
+        public const UInt32 DscServiceDefaultPort = 6498;
+
+
+        public const Int32 HdfsServiceDefaultHttpPort = 50070;
+
+
+        public const string ServiceLocationString = @"ServiceLocation";
+
+        public const String jobManager = "XCJOBMANAGER";
+
+        public const string vertexAddrFormat = "net.tcp://{0}:8050/{1}/";  // net.tcp://<machine>:8050/<JobTaskId>/
+        public const string vertexCallbackAddrFormat = "net.tcp://{0}:8051/{1}/"; // net.tcp://<machine>:8051/<DryadProcId>/
+        public const string vertexCallbackServiceName = "DryadVertexCallback";
+        public const string vertexServiceName = "DryadVertexService";
+        public const string vertexFileServiceName = "DryadVertexFileService";
+        public const int vertexFileChunkSize = 1024 * 16;
+
+        public const string vertexCountEnvVar = "HPC_VERTEXCOUNT";
+        public const string vertexEnvVarFormat = "HPC_VERTEX{0}";
+        public const string vertexSvcInstanceEnvVar = "HPC_VERTEXSVCINST";
+        public const string vertexSvcLocalAddrEnvVar = "CCP_DRYADVERTEXLOCALADDRESS";
+
+        public const string schedulerTypeEnvVar = "CCP_SCHEDULERTYPE";
+        public const string schedulerTypeLocal = "LOCAL";
+        public const string schedulerTypeCluster = "CLUSTER";
+        public const string schedulerTypeAzure = "AZURE";
+        public const string debugAzure = "DEBUG_AZURE";
+        public const string schedulerTypeYarn = "YARN";
+
+
+        // Recognized values are: OFF, CRITICAL, ERROR, WARN, INFO, VERBOSE
+        public const string traceLevelEnvVar = "CCP_DRYADTRACELEVEL";
+        public const string traceOff = "OFF";
+        public const string traceCritical = "CRITICAL";
+        public const string traceError = "ERROR";
+        public const string traceWarning = "WARN";
+        public const string traceInfo = "INFO";
+        public const string traceVerbose = "VERBOSE";
+
+        public const int traceOffNum = 0;
+        public const int traceCriticalNum = 1;
+        public const int traceErrorNum = 3;
+        public const int traceWarningNum = 7;
+        public const int traceInfoNum = 15;
+        public const int traceVerboseNum = 31;
+
+        public const string VertexSecurityEnvVar = "HPC_VERTEX_SECURITY";
+
+        // SchedulerHelper environment variables
+        public const string clusterNameEnvVar = "CCP_CLUSTER_NAME";
+        public const string jobIdEnvVar = "CCP_JOBID";
+        public const string taskIdEnvVar = "CCP_TASKID";
+        public const string nodesEnvVar = "CCP_NODES";
+        public const string jobNameEnvVar = "CCP_JOBNAME";
+        public const string requiredNodesEnvVar = "CCP_REQUIREDNODES";
+        public const string localProcessComputeNodesEnvVar = "CCP_LOCALPROCESSCOMPUTENODES";
+
+        // DrError.h values used in managed code
+        // need to keep this section in sync with drerror.h changes...
+        public const uint DrError_VertexReceivedTermination = 0x830A0003;
+        public const uint DrError_VertexCompleted = 0x830A0016;
+        public const uint DrError_VertexError = 0x830A0017;
+        public const uint DrError_VertexInitialization = 0x830A0019;
+        public const uint DrError_ProcessingInterrupted = 0x830A001A;
+        public const uint DrError_VertexHostLostCommunication = 0x830A0FFF;
+
+        // DSC Share Names
+        public const string DscTempShare = "HpcTemp";
+        public const string DscDataShare = "HpcData";
+        public const string RuntimeShareConfig = "HPC_RUNTIMESHARE";
+
+        // Cluster name
+        public const string ClusterNameConfig = "CCP_CLUSTER_NAME";
+
+        // NodeAdmin constants
+        // Retain time set to one day
+        // todo: this should be configurable
+        public static readonly TimeSpan RetainTime = new TimeSpan(1, 0, 0, 0);
+        public static readonly TimeSpan FileTimeStampMarginForGC = new TimeSpan(0, 0, 5, 0);
+        public const string runningJobEnvVar = "CCP_RUNNING_JOBS";
+        public const string replicaPathFormat = @"\\{0}\HpcData\{1}.data";
+        public const string nodeAdminMutexName = "A19A8AC1-4129-46e2-BB81-ED7EE3265B05";
+        public const string nodeAdminUsage = "Syntax:\n\t" + 
+                                                "HpcDscNodeAdmin [/r] [/g] [/wd] [/e] [/v] [/u]\n\n" +
+                                             "Parameters:\n\t" + 
+                                                "/? \t- Display this help message.\n\t" +
+                                                "/g \t- Delete files not managed by DSC from the HpcData share.\n\t" +
+                                                "/wd\t- Delete old job working directories from the HpcTemp share.\n\t" +
+                                                "/r \t- Replicate DSC files onto this node.\n\t" +
+                                                "/e \t- Print full error traces.\n\t" +
+                                                "/u \t- Resets HpcReplication account password.\n\t" +
+                                               "/v \t- Print verbose activity traces.\n";
+
+        // HpcReplication user account
+        internal const string HpcReplicationUserName = "HpcReplication";
+
+        // Client retry period is 1 second for first retry, increasing up to 12 seconds for a total of 30 seconds
+        // These timeouts are intended to ride through transient network failures
+        internal const int StartRetryPeriod = 1000;
+        internal const int MaxRetryPeriod = 12000;
+        internal const int TotalRetryPeriod = 30000;
+        internal const int ClientRetryCount = 4;
+
+        // Runtime retry period is 10 seconds for first retry, increasing up to 60 seconds for a total of 6 minutes
+        // Runtime timeouts intended to ride through a failover and more severe network disruptions with the goal 
+        // of keeping running jobs alive
+        internal const int RuntimeStartRetryPeriod = 10000;
+        internal const int RuntimeMaxRetryPeriod = 60000;
+        internal const int RuntimeTotalRetryPeriod = 360000;
+        internal const int RuntimeClientRetryCount = 7;
+    }
+}

CommonCode/DiscLocalMonitor.cs

@@ -0,0 +1,163 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+namespace Microsoft.Research.Dryad
+{
+    using System;
+    using System.Collections;
+    using System.Collections.Generic;
+    using System.Text;
+    using System.IO;
+    using System.Threading;
+    using System.Diagnostics;
+    using System.ServiceModel;
+    using System.ServiceModel.Channels;
+    using System.ServiceModel.Description;
+    using System.Net.Security;
+    using System.Runtime.Serialization;
+
+    [ServiceContract(SessionMode = SessionMode.Allowed)]
+    public interface IDiscLocalJobMonitor
+    {
+        [OperationContract]
+        void UpdateJobProgress(string JobId, string JobMessage, double JobProgress);
+
+        [OperationContract]
+        void UpdateJobState(string JobId, string JobState);
+    }
+    
+    [System.CodeDom.Compiler.GeneratedCodeAttribute("System.ServiceModel", "3.0.0.0")]
+    public interface IDiscLocalJobMonitorServiceChannel : IDiscLocalJobMonitor, System.ServiceModel.IClientChannel
+    {
+    }
+    
+    [System.Diagnostics.DebuggerStepThroughAttribute()]
+    [System.CodeDom.Compiler.GeneratedCodeAttribute("System.ServiceModel", "3.0.0.0")]
+    public partial class DiscLocalJobMonitorClient : System.ServiceModel.ClientBase<IDiscLocalJobMonitor>, IDiscLocalJobMonitor
+    {
+        
+        public DiscLocalJobMonitorClient()
+        {
+        }
+        
+        public DiscLocalJobMonitorClient(System.ServiceModel.Channels.Binding binding, System.ServiceModel.EndpointAddress remoteAddress) :
+            base(binding, remoteAddress)
+        {
+        }
+            
+        public void UpdateJobProgress(string JobId, string JobMessage, double JobProgress)
+        {
+            base.Channel.UpdateJobProgress(JobId, JobMessage, JobProgress);
+        }
+
+        public void UpdateJobState(string JobId, string JobState)
+        {
+            base.Channel.UpdateJobState(JobId, JobState);
+        }
+    }
+
+    public class DiscLocalMonitorHelper            
+    {
+        public string DiscLocalMonitorMachine = @"localhost";
+        public DiscLocalMonitorHelper()
+        {
+        }
+        ~DiscLocalMonitorHelper()
+        {
+            m_client = null;
+        }
+
+        public NetTcpBinding LocalMonitorBinding
+        {
+            get
+            {
+                NetTcpBinding binding = new NetTcpBinding(SecurityMode.Transport, false);
+                binding.Security.Transport.ClientCredentialType = TcpClientCredentialType.Windows;
+                binding.Security.Transport.ProtectionLevel = ProtectionLevel.None;
+                return binding;
+            }
+        }
+        
+        public string LocalMonitorEpr
+        {
+            get
+            {
+                return String.Format("net.tcp://{0}:8042/Service/DiscLocalJobMonitor", DiscLocalMonitorMachine);
+            }
+        }
+
+        private DiscLocalJobMonitorClient m_client = null;
+        private bool faultedClient = false;
+        public DiscLocalJobMonitorClient Client
+        {
+            get
+            {
+                if (faultedClient)
+                {
+                    return null;
+                }            
+                if (m_client != null)
+                {
+                    return m_client;
+                }
+                m_client = new DiscLocalJobMonitorClient(LocalMonitorBinding, new EndpointAddress(LocalMonitorEpr));
+                return m_client;               
+            }
+        }
+        
+        public void UpdateProgress(string JobId, string JobMessage, double JobProgress)
+        {
+            try
+            {
+                if (this.Client != null)
+                {
+                    this.Client.UpdateJobProgress(JobId, JobMessage, JobProgress);
+                }
+            }
+            catch (Exception e)
+            {
+                faultedClient = true;
+                m_client = null;
+                Console.WriteLine("ERROR: DiscLocalMonitorHelper '{0}'", e);
+            }                    
+
+        }
+
+        public void UpdateJobState(string JobId, string JobState)
+        {
+            try
+            {
+                if (this.Client != null)
+                {
+                    this.Client.UpdateJobState(JobId, JobState);
+                }
+            }
+            catch (Exception e)
+            {
+                faultedClient = true;
+                m_client = null;
+                Console.WriteLine("ERROR: DiscLocalMonitorHelper '{0}'", e);
+            }                    
+
+        }
+        
+    }
+    
+}

CommonCode/DryadTracing.cs

@@ -0,0 +1,693 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Diagnostics;
+using System.Diagnostics.Eventing;
+using System.IO;
+using Microsoft.Win32;
+using System.Runtime.InteropServices;
+using System.Security.Principal;
+using System.Security.Permissions;
+using System.Threading;
+
+namespace Microsoft.Research.Dryad
+{
+    internal struct DryadEventDescriptor
+    {
+        private int m_level;
+        private string m_message;
+
+        public int Level
+        {
+            get { return m_level; }
+        }
+
+        public string Message
+        {
+            get { return m_message; }
+        }
+
+        public DryadEventDescriptor(int level, string message)
+        {
+            m_level = level;
+            m_message = message;
+        }
+    }
+
+    internal sealed class DISCTextProvider : IDisposable
+    {
+        private bool m_disposed = false;
+        private uint m_rolloverCount = 0;
+        private static readonly long m_maxFileBytes = 10 * 1024 * 1024;
+        private object m_lock = new object();
+        private static readonly string m_logFileFormat = "{0}{1:D6}{2}";
+        private string m_baseFilePath = String.Empty;
+        private StreamWriter m_provider = null;
+        private Timer m_flushTimer = null;
+        private static readonly int m_flushTimeout = 100;
+
+        public DISCTextProvider(string path)
+        {
+            m_baseFilePath = path;
+            OpenLogFile();
+            m_flushTimer = new Timer(new TimerCallback(this.FlushLogTimer), null, 1000, 1000);
+        }
+
+        private void FlushLogTimer(Object state)
+        {
+            try
+            {
+                Flush();
+            }
+            catch
+            {
+            }
+        }
+
+        // Must be called while holding m_lock
+        private bool ArchiveLogFile()
+        {
+            string archivePath = String.Format(m_logFileFormat, Path.Combine(Path.GetDirectoryName(m_baseFilePath),Path.GetFileNameWithoutExtension(m_baseFilePath)), m_rolloverCount, Path.GetExtension(m_baseFilePath));
+
+            for (int i = 0; i < 3; i++)
+            {
+                try
+                {
+                    File.Move(m_baseFilePath, archivePath);
+                    return true;
+                }
+                catch
+                {
+                }
+            }
+
+            return false;
+        }
+
+        // Must be called while holding m_lock
+        private void CloseLogFile()
+        {
+            if (m_provider != null)
+            {
+                // Close the current log file
+                try
+                {
+                    Flush();
+
+                    // Dispose
+                    m_provider.Dispose();
+                    m_provider = null;
+                }
+                catch
+                {
+                    return;
+                }
+            }
+        }
+
+        private void OpenLogFile()
+        {
+            try
+            {
+                // Open log file
+                m_provider = new StreamWriter(new FileStream(m_baseFilePath, FileMode.Append, FileAccess.Write, FileShare.ReadWrite));
+            }
+            catch
+            {
+            }
+        }
+
+        private void ReopenLogFile()
+        {
+            if (m_provider != null)
+            {
+                // Close the current log file
+                CloseLogFile();
+
+                ArchiveLogFile();
+            }
+
+            OpenLogFile();
+
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        private void Dispose(bool disposing)
+        {
+            if (!m_disposed)
+            {
+                if (disposing)
+                {
+                    lock (m_lock)
+                    {
+                        if (m_flushTimer != null)
+                        {
+                            m_flushTimer.Dispose();
+                            m_flushTimer = null;
+                        }
+
+                        if (m_provider != null)
+                        {
+                            CloseLogFile();
+                            m_provider = null;
+                        }
+                    }
+                }
+                m_disposed = true;
+            }
+        }
+
+        public void WriteEvent(ref DryadEventDescriptor dryadEventDescriptor, string ModuleName, string Source, DateTime TimeStamp, string OperationContext, string Message)
+        {
+            try
+            {
+                StringBuilder newMessage = new StringBuilder();
+                if (!String.IsNullOrEmpty(OperationContext))
+                {
+                    newMessage.Append(OperationContext);
+                    newMessage.Append(": ");
+                }
+                newMessage.Append(Message);
+
+                WriteEvent(ref dryadEventDescriptor, ModuleName, Source, TimeStamp, newMessage.ToString());
+            }
+            catch
+            {
+            }
+        }
+
+        public void WriteEvent(ref DryadEventDescriptor dryadEventDescriptor, string ModuleName, string Source, DateTime TimeStamp, string OperationContext, string MessageFormat, params object[] MessageParameters)
+        {
+            try
+            {
+                StringBuilder newMessage = new StringBuilder();
+                if (!String.IsNullOrEmpty(OperationContext))
+                {
+                    newMessage.Append(OperationContext);
+                    newMessage.Append(": ");
+                }
+
+                newMessage.Append(String.Format(MessageFormat, MessageParameters));
+
+                WriteEvent(ref dryadEventDescriptor, ModuleName, Source, TimeStamp, newMessage.ToString());
+            }
+            catch
+            {
+            }
+
+        }
+
+        public void WriteEvent(ref DryadEventDescriptor dryadEventDescriptor, string ModuleName, string Source, DateTime TimeStamp, string Message)
+        {
+            try
+            {
+                lock (m_lock)
+                {
+                    if (m_provider != null)
+                    {
+                        m_provider.WriteLine(
+                            String.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\"",
+                            dryadEventDescriptor.Message,
+                            TimeStamp.ToString("yyyy/MM/dd HH:mm:ss.fff", System.Globalization.CultureInfo.InvariantCulture),
+                            System.Diagnostics.Process.GetCurrentProcess().Id,
+                            System.Threading.Thread.CurrentThread.ManagedThreadId,
+                            ModuleName,
+                            Source,
+                            String.IsNullOrEmpty(Message) ? String.Empty : Message
+                            ));
+
+                        if (m_provider.BaseStream.Position > m_maxFileBytes)
+                        {
+                            m_rolloverCount++;
+                            ReopenLogFile();
+                        }
+                    }
+                }
+            }
+            catch
+            {
+            }
+        }
+
+        public void WriteEvent(ref DryadEventDescriptor dryadEventDescriptor, string ModuleName, string Source, DateTime TimeStamp, Exception Exception, string Message)
+        {
+            try
+            {
+                lock (m_lock)
+                {
+                    if (m_provider != null)
+                    {
+                        m_provider.WriteLine(
+                            String.Format("\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\",\"{7}\"",
+                            dryadEventDescriptor.Message,
+                            TimeStamp.ToString("yyyy/MM/dd HH:mm:ss.fff", System.Globalization.CultureInfo.InvariantCulture),
+                            System.Diagnostics.Process.GetCurrentProcess().Id,
+                            System.Threading.Thread.CurrentThread.ManagedThreadId,
+                            ModuleName,
+                            Source,
+                            String.IsNullOrEmpty(Message) ? String.Empty : Message,
+                            (Exception == null) ? String.Empty : Exception.ToString()
+                            ));
+
+                        if (m_provider.BaseStream.Position > m_maxFileBytes)
+                        {
+                            m_rolloverCount++;
+                            ReopenLogFile();
+                        }
+                    }
+                }
+            }
+            catch
+            {
+            }
+        }
+
+        public void WriteEvent(ref DryadEventDescriptor dryadEventDescriptor, string ModuleName, string Source, DateTime TimeStamp, Exception Exception, string MessageFormat, params object[] MessageParameters)
+        {
+            try
+            {
+                WriteEvent(ref dryadEventDescriptor, ModuleName, Source, TimeStamp, Exception, String.Format(MessageFormat, MessageParameters));
+            }
+            catch
+            {
+            }
+        }
+
+        public void Flush()
+        {
+            try
+            {
+                if (Monitor.TryEnter(m_lock, m_flushTimeout))
+                {
+                    try
+                    {
+                        if (m_provider != null)
+                        {
+                            m_provider.Flush();
+                        }
+                    }
+                    catch
+                    {
+                    }
+                    finally
+                    {
+                        Monitor.Exit(m_lock);
+                    }
+                }
+            }
+            catch
+            {
+            }
+        }
+
+    }
+
+    public sealed class DryadLogger
+    {
+        private static DISCTextProvider s_discTracer;
+        // TODO: Change the default once there is better support for setting per job
+        private static int s_traceLevel = Constants.traceVerboseNum;
+        private static object s_syncRoot = new object();
+        private static bool s_initialized = false;
+
+        private static DryadEventDescriptor DryadMethodEntry = new DryadEventDescriptor(Constants.traceVerboseNum, "MethodEntry");
+        private static DryadEventDescriptor DryadMethodExit = new DryadEventDescriptor(Constants.traceVerboseNum, "MethodExit");
+        private static DryadEventDescriptor DryadError = new DryadEventDescriptor(Constants.traceErrorNum, "Error");
+        private static DryadEventDescriptor DryadCritical = new DryadEventDescriptor(Constants.traceErrorNum, "Critical");
+        private static DryadEventDescriptor DryadWarning = new DryadEventDescriptor(Constants.traceWarningNum, "Warning");
+        private static DryadEventDescriptor DryadInformational = new DryadEventDescriptor(Constants.traceInfoNum, "Info");
+        private static DryadEventDescriptor DryadVerbose = new DryadEventDescriptor(Constants.traceVerboseNum, "Verbose");
+
+
+        private DryadLogger()
+        {
+        }
+
+        static DryadLogger()
+        {
+            try
+            {
+                string debugLevel = Environment.GetEnvironmentVariable(Constants.traceLevelEnvVar);
+
+                if (!String.IsNullOrEmpty(debugLevel))
+                {
+                    s_traceLevel = HpcQueryUtility.GetTraceLevelFromString(debugLevel);
+                }
+
+                Console.Out.WriteLine("Trace level set to {0}", HpcQueryUtility.ConvertTraceLevelToString(s_traceLevel));
+            }
+
+            catch (Exception e)
+            {
+                Console.Error.WriteLine("Failed to get tracing level: {0}", e);
+            }
+        }
+
+        public static int TraceLevel
+        {
+            get { return s_traceLevel; }
+            set { s_traceLevel = value; }
+        }
+
+        public static bool Start(string path)
+        {
+            try
+            {
+                if (!s_initialized)
+                {
+                    lock (s_syncRoot)
+                    {
+                        if (!s_initialized)
+                        {
+                            s_discTracer = new DISCTextProvider(path);
+                            if (s_discTracer != null)
+                            {
+                                s_initialized = true;
+                            }
+                            else
+                            {
+                                Console.Error.WriteLine("Tracing initialization failed: failed to get intance of tracing provider");
+                            }
+                        }
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                Console.Error.WriteLine("Tracing initialization failed: {0}", e);
+            }
+
+            return s_initialized;
+        }
+
+        public static void Stop()
+        {
+            if (s_initialized)
+            {
+                lock (s_syncRoot)
+                {
+                    if (s_initialized)
+                    {
+                        s_initialized = false;
+                        s_discTracer.Flush();
+                        s_discTracer.Dispose();
+                        s_discTracer = null;
+                    }
+                }
+            }
+        }
+
+        private static bool IsEnabled(int level)
+        {
+            return (s_initialized) && ((level & s_traceLevel) == level);
+        }
+
+        private static string GetModuleName(StackTrace inputStack)
+        {
+            if (inputStack.FrameCount > 0)
+            {
+                return (String.Format("{0}!{1}!{2}", inputStack.GetFrame(0).GetMethod().Module, inputStack.GetFrame(0).GetMethod().ReflectedType.Name, inputStack.GetFrame(0).GetMethod().Name));
+            }
+            else
+            {
+                return String.Empty;
+            }
+        }
+
+        public static void LogMethodEntry(params object[] methodParameters)
+        {
+            if (!IsEnabled(Constants.traceVerboseNum)) return;
+
+            StringBuilder parameterString = new StringBuilder();
+
+            if (methodParameters != null && methodParameters.Length > 0)
+            {
+                Int32 parameterCount = 0;
+
+                foreach (object methodParameter in methodParameters)
+                {
+                    if (parameterCount == methodParameters.Length - 1)
+                    {
+                        parameterString.Append(methodParameter);
+                    }
+                    else
+                    {
+                        parameterString.Append(methodParameter);
+                        parameterString.Append(", ");
+                    }
+
+                    parameterCount++;
+                }
+            }
+
+            s_discTracer.WriteEvent(ref DryadMethodEntry, Process.GetCurrentProcess().ProcessName, GetModuleName(new StackTrace(1)), DateTime.Now, parameterString.ToString());
+            
+            return;
+        }
+
+        public static void LogMethodExit(params object[] methodParameters)
+        {
+            if (!IsEnabled(Constants.traceVerboseNum)) return;
+
+            StringBuilder parameterString = new StringBuilder();
+
+            if (methodParameters != null && methodParameters.Length > 0)
+            {
+                Int32 parameterCount = 0;
+
+                foreach (object methodParameter in methodParameters)
+                {
+                    if (parameterCount == methodParameters.Length - 1)
+                    {
+                        parameterString.Append(methodParameter);
+                    }
+                    else
+                    {
+                        parameterString.Append(methodParameter);
+                        parameterString.Append(", ");
+                    }
+
+                    parameterCount++;
+                }
+            }
+
+            s_discTracer.WriteEvent(ref DryadMethodExit, Process.GetCurrentProcess().ProcessName, GetModuleName(new StackTrace(1)), DateTime.Now, parameterString.ToString());
+           
+
+            return;
+        }
+
+        public static void LogError(Int32 errorCode, Exception discException)
+        {
+            if (!IsEnabled(Constants.traceErrorNum)) return;
+
+            StackTrace currentStack;
+
+            if (discException != null)
+            {
+                currentStack = new StackTrace(discException);
+            }
+            else
+            {
+                currentStack = new StackTrace(1);
+            }
+
+            s_discTracer.WriteEvent(ref DryadError, Process.GetCurrentProcess().ProcessName, GetModuleName(currentStack), DateTime.Now, discException, String.Empty);
+            
+            return;
+        }
+
+        public static void LogError(Int32 errorCode, Exception discException, string messageFormat, params object[] parameterValues)
+        {
+            if (!IsEnabled(Constants.traceErrorNum)) return;
+
+            StackTrace currentStack;
+
+            if (discException != null)
+            {
+                currentStack = new StackTrace(discException);
+            }
+            else
+            {
+                currentStack = new StackTrace(1);
+            }
+
+            s_discTracer.WriteEvent(ref DryadError, Process.GetCurrentProcess().ProcessName, GetModuleName(currentStack), DateTime.Now, discException, messageFormat, parameterValues);
+            
+            return;
+        }
+
+        private static void LogCriticalToConsole(Int32 errorCode, Exception discException, string messageFormat, params object[] parameterValues)
+        {
+            if (discException == null && String.IsNullOrEmpty(messageFormat))
+            {
+                // Sadly, nothing to log
+                return;
+            }
+
+            StringBuilder message = new StringBuilder();
+
+            if (discException != null)
+            {
+                message.Append("Critical Exception occurred: ");
+                int hr = System.Runtime.InteropServices.Marshal.GetHRForException(discException);
+                if (hr != 0)
+                {
+                    message.Append("0x");
+                    message.Append(hr.ToString("X8"));
+                }
+                Console.Error.WriteLine(message.ToString());
+                Console.Error.WriteLine(discException.ToString());
+            }
+            else if (errorCode != 0)
+            {
+                message.Append("Critical error occured: code = ");
+                message.Append(errorCode);
+                Console.Error.WriteLine(message.ToString());
+            }
+
+            if (!String.IsNullOrEmpty(messageFormat))
+            {
+                try
+                {
+                    Console.Error.WriteLine(messageFormat, parameterValues);
+                }
+                catch
+                {
+                }
+            }
+            Console.Error.Flush();
+        }
+
+        public static void LogCritical(Int32 errorCode, Exception discException)
+        {
+            //
+            // For LogCritical only, write message to Console.Error so that it shows up in task's output
+            //
+            LogCriticalToConsole(errorCode, discException, String.Empty);
+            
+            if (!IsEnabled(Constants.traceCriticalNum)) return;
+
+            StackTrace currentStack;
+
+            if (discException != null)
+            {
+                currentStack = new StackTrace(discException);
+            }
+            else
+            {
+                currentStack = new StackTrace(1);
+            }
+
+            s_discTracer.WriteEvent(ref DryadCritical, Process.GetCurrentProcess().ProcessName, GetModuleName(currentStack), DateTime.Now, discException, String.Empty);
+            s_discTracer.Flush();
+            
+            return;
+        }
+
+        public static void LogCritical(Int32 errorCode, Exception discException, string messageFormat, params object[] parameterValues)
+        {
+            //
+            // For LogCritical only, write message to Console.Error so that it shows up in task's output
+            //
+            LogCriticalToConsole(errorCode, discException, messageFormat, parameterValues);
+
+            if (!IsEnabled(Constants.traceCriticalNum)) return;
+
+            StackTrace currentStack;
+
+            if (discException != null)
+            {
+                currentStack = new StackTrace(discException);
+            }
+            else
+            {
+                currentStack = new StackTrace(1);
+            }
+
+            s_discTracer.WriteEvent(ref DryadCritical, Process.GetCurrentProcess().ProcessName, GetModuleName(currentStack), DateTime.Now, discException, messageFormat, parameterValues);
+            s_discTracer.Flush();
+            
+            return;
+        }
+
+        public static void LogWarning(string operationContext, string warningMessage)
+        {
+            if (!IsEnabled(Constants.traceWarningNum)) return;
+
+            s_discTracer.WriteEvent(ref DryadWarning, Process.GetCurrentProcess().ProcessName, GetModuleName(new StackTrace(1)), DateTime.Now, operationContext, warningMessage);
+            
+            return;
+        }
+
+        public static void LogWarning(string operationContext, string warningMessageFormat, params object[] parameterValues)
+        {
+            if (!IsEnabled(Constants.traceWarningNum)) return;
+
+            s_discTracer.WriteEvent(ref DryadWarning, Process.GetCurrentProcess().ProcessName, GetModuleName(new StackTrace(1)), DateTime.Now, operationContext, warningMessageFormat, parameterValues);
+            
+            return;
+        }
+
+        public static void LogInformation(string operationContext, string operationalMessage)
+        {
+            if (!IsEnabled(Constants.traceInfoNum)) return;
+
+            s_discTracer.WriteEvent(ref DryadInformational, Process.GetCurrentProcess().ProcessName, GetModuleName(new StackTrace(1)), DateTime.Now, operationContext, operationalMessage);
+            
+            return;
+        }
+
+        public static void LogInformation(string operationContext, string operationalMessageFormat, params object[] parameterValues)
+        {
+            if (!IsEnabled(Constants.traceInfoNum)) return;
+
+            s_discTracer.WriteEvent(ref DryadInformational, Process.GetCurrentProcess().ProcessName, GetModuleName(new StackTrace(1)), DateTime.Now, operationContext, operationalMessageFormat, parameterValues);
+          
+            return;
+        }
+
+        public static void LogDebug(string operationContext, string debugMessage)
+        {
+            if (!IsEnabled(Constants.traceVerboseNum)) return;
+
+            s_discTracer.WriteEvent(ref DryadVerbose, Process.GetCurrentProcess().ProcessName, GetModuleName(new StackTrace(1)), DateTime.Now, operationContext, debugMessage);
+
+            return;
+        }
+
+        public static void LogDebug(string operationContext, string debugMessageFormat, params object[] parameterValues)
+        {
+            if (!IsEnabled(Constants.traceVerboseNum)) return;
+
+            s_discTracer.WriteEvent(ref DryadVerbose, Process.GetCurrentProcess().ProcessName, GetModuleName(new StackTrace(1)), DateTime.Now, operationContext, debugMessageFormat, parameterValues);
+            
+            return;
+        }  
+    }
+}

CommonCode/DryadVertexServiceAuthorizationManager.cs

@@ -0,0 +1,118 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//------------------------------------------------------------------------------
+// <summary>
+//      Auth manager used for checking if the caller identity matches with the 
+//      current identity.
+// </summary>
+//------------------------------------------------------------------------------
+
+namespace Microsoft.Research.Dryad
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Security.Principal;
+    using System.ServiceModel;
+    using System.ServiceModel.Channels;
+    using System.Net;
+    using System.Text;
+
+    /// <summary>
+    /// Auth manager used for checking if the caller identity matches with the current identity.
+    /// </summary>
+    public class DryadVertexServiceAuthorizationManager : ServiceAuthorizationManager
+    {
+        /// <summary>
+        /// Reference Identity
+        /// </summary>
+        WindowsIdentity currentIdentity;
+
+        /// <summary>
+        /// Creates an instance of the DryadVertexServiceAuthorizationManager class.
+        /// </summary>
+        public DryadVertexServiceAuthorizationManager()
+        {
+            this.currentIdentity = WindowsIdentity.GetCurrent();
+        }
+
+        /// <summary>
+        /// Check whether current operation context should be allowed access
+        /// </summary>
+        /// <param name="operationContext">Current operation context</param>
+        /// <returns>true = allowed</returns>
+        protected override bool CheckAccessCore(OperationContext operationContext)
+        {
+            //TODO: Put logging information to appropriate channels when available.
+
+            //
+            // Fail if context is annonymous
+            //
+            if (operationContext.ServiceSecurityContext.IsAnonymous)
+            {
+                DryadLogger.LogError(0, null, "Vertex authentication failed : Service security context is anonymous.");
+                return false;
+            }
+
+            //
+            // Get identity used in current context
+            //
+            WindowsIdentity callerIdentity = operationContext.ServiceSecurityContext.WindowsIdentity;
+            if (callerIdentity == null)
+            {
+                //
+                // Fail if identity is not set
+                //
+                DryadLogger.LogError(0, null, "Vertex authentication failed : Caller identity is null.");
+                return false;
+            }
+            else if (callerIdentity.IsAnonymous)
+            {
+                //
+                // Fail if identity is anonymous
+                //
+                DryadLogger.LogError(0, null, "Vertex authentication failed : Caller identity is anonymous.");
+                return false;
+            }
+            else if (!callerIdentity.IsAuthenticated)
+            {
+                //
+                // Fail if identity is not authenticated
+                //
+                DryadLogger.LogError(0, null, "Vertex authentication failed : Caller identity is not authenticated.");
+                return false;
+            }
+
+            //
+            // If operation context has same user as vertex service, then allow, otherwise fail.
+            //
+            if (this.currentIdentity.User == callerIdentity.User)
+            {
+                return true;
+            }
+            else
+            {
+                DryadLogger.LogError(0, null, "Vertex authentication failed : Current identity is {0}, caller identity is {1}", this.currentIdentity.Name, callerIdentity.Name);
+            }
+
+            return false;
+        }
+    }
+}

CommonCode/ExecutionHelper.cs

@@ -0,0 +1,297 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+namespace Microsoft.Research.Dryad
+{
+    using System;
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Linq;
+    using System.Net;
+    using System.Security.Principal;
+    using System.Security.AccessControl;
+    using Microsoft.Research.Dryad;
+    
+    internal static class ExecutionHelper
+    {
+        /// <summary>
+        /// List of known files in bin directory
+        /// </summary>
+        private static List<string> binFileList = new List<string>(10);
+
+        /// <summary>
+        /// %DRYAD_HOME% bin directory where installed binaries can be found
+        /// </summary>
+        private static string dryadHome = Environment.GetEnvironmentVariable("DRYAD_HOME");
+
+        /// <summary>
+        /// Lockable object used to help make this class thread safe.
+        /// </summary>
+        private static object initializeLock = new object();
+
+        /// <summary>
+        /// Used to initialize known file list
+        /// </summary>
+        private static void InitializeBinFileList()
+        {
+            if (binFileList.Count == 0)
+            {
+                lock (initializeLock)
+                {
+                    if (binFileList.Count == 0)
+                    {
+                        binFileList.Add("VertexHost.exe");
+                        binFileList.Add("Microsoft.Research.Dryad.dll");
+                        binFileList.Add("DryadLINQNativeChannels.dll");
+                        binFileList.Add("YarnQueryNativeClusterAdapter.dll");
+                        binFileList.Add("Microsoft.Research.Dryad.dll");
+                        binFileList.Add("Microsoft.Research.DryadLINQ.dll");
+                        binFileList.Add("Microsoft.Research.Dryad.Hdfs.dll");
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Check for azure execution
+        /// </summary>
+        private static bool AzureExecution
+        {
+            get
+            {
+                string debugAzure = Environment.GetEnvironmentVariable(Constants.debugAzure);
+                if (!String.IsNullOrEmpty(debugAzure))
+                {
+                    return true;
+                }
+
+                string schedulerType = Environment.GetEnvironmentVariable(Constants.schedulerTypeEnvVar);
+                return (!String.IsNullOrEmpty(schedulerType) && schedulerType == Constants.schedulerTypeAzure);
+            }
+        }
+
+        /// <summary>
+        /// Copy local resources from %ccp_home%bin to working directory
+        /// todo: Post SP2, we should run directly from %ccp_home%bin rather than performing a local copy.
+        /// </summary>
+        /// <returns>success = true</returns>
+        private static bool CopyLocalBinaries()
+        {
+            //Console.Error.WriteLine("Copying source files from {0}", dryadHome); //DEBUG
+            foreach (string localFile in binFileList)
+            {
+                // Get path to job working directory in \\hpctemp
+                string jobFilePath = Path.Combine(ProcessPathHelper.JobPath, localFile);
+
+                // Only copy files that do not already exist
+                //  Avoids overwriting files when a vertex service task fails or finishes on one node 
+                //  and a new vertex service task is scheduled on the same node within the same job
+                if (!File.Exists(jobFilePath))
+                {
+                    // Get path to file source in CCP_HOME\bin
+                    string sourceFilePath = Path.Combine(dryadHome, localFile);
+
+                    try
+                    {
+                        File.Copy(sourceFilePath, jobFilePath, true);
+                    }
+                    catch (Exception e)
+                    {
+                        Console.Error.WriteLine("[ExecutionHelper.CopyLocalResources] Exception copying '{0}' to '{1}': {2}", sourceFilePath, jobFilePath, e.Message);
+                        return false;
+                    }
+                }
+            }
+
+            return true;
+        }
+
+        private static void GetHdfsFile(string hdfsDir, string fileName, string destFileName)
+        {
+            if(!hdfsDir.EndsWith("/"))
+            {
+                hdfsDir = hdfsDir + "/";
+            }
+            var hdfsDirUri = new Uri(hdfsDir, UriKind.Absolute);
+            var hdfsFileUri = new Uri(hdfsDirUri, fileName);
+            var builder = new UriBuilder();
+            builder.Host = hdfsFileUri.DnsSafeHost;
+            builder.Port = Constants.HdfsServiceDefaultHttpPort;
+            builder.Path = "webhdfs/v1/" + hdfsFileUri.AbsolutePath.TrimStart('/');
+            builder.Query = "op=OPEN";
+            Console.WriteLine(builder.Uri);
+            var wc = new WebClient();
+            wc.DownloadFile(builder.Uri, destFileName);
+        }
+
+        /// <summary>
+        /// Copy the resources from staging dir to working dir
+        /// </summary>
+        /// <param name="resources">list of resources supplied by dryadlinq</param>
+        /// <returns>success = true</returns>
+        private static bool CopyStagedJobResources(string resources)
+        {
+            if (resources != null)
+            {
+                if (resources[0] == '@')
+                {
+                    resources = File.ReadAllText(resources.Substring(1));
+                }
+
+                if (resources.EndsWith(","))
+                {
+                    resources = resources.Substring(0, resources.Length - 1);
+                }
+                string[] files = resources.Split(',');
+                DryadLogger.LogInformation("CopyStagedJobResources", string.Format("Will copy {0} resource files.", files.Length));
+
+                if (files.Length > 1)
+                {
+                    string source = files[0];
+                    for (int i = 1; i < files.Length; i++)
+                    {
+                        string jobFilePath = Path.Combine(ProcessPathHelper.JobPath, files[i]);
+
+                        //
+                        // File may already exist due to local resource copying
+                        //
+                        if (File.Exists(jobFilePath) == false)
+                        {
+                            //
+                            // If file doesn't exist today, get it from staging location
+                            //
+                            if(source.StartsWith("hdfs://", StringComparison.InvariantCultureIgnoreCase))
+                            {
+                                // copy from HDFS
+                                DryadLogger.LogDebug("CopyStagedJobResources", string.Format(
+                                       "[ExecutionHelper.CopyJobResources] Copying '{0}' to '{1}' from HDFS dir {2}",
+                                       files[i], jobFilePath, source));
+                                GetHdfsFile(source, files[i], jobFilePath);
+                            }
+                            else
+                            {
+                                string sourceFile = Path.Combine(source, files[i]);
+                                try
+                                {
+                                    DryadLogger.LogDebug("CopyStagedJobResources", string.Format(
+                                        "[ExecutionHelper.CopyJobResources] Copying '{0}' to '{1}'",
+                                        sourceFile, jobFilePath));
+                                    File.Copy(sourceFile, jobFilePath);
+                                }
+                                catch (Exception e)
+                                {
+                                    DryadLogger.LogInformation("CopyStagedJobResources", string.Format(
+                                        "[ExecutionHelper.CopyJobResources] Exception copying '{0}' to '{1}': {2}",
+                                        sourceFile, jobFilePath, e.Message));
+                                    return false;
+                                }
+                            }
+                        }
+                    }
+                }
+                else
+                {
+                    Console.Error.WriteLine("[ExecutionHelper.CopyJobResources] invalid XC_RESOURCEFILES length = {0}", files.Length);
+                    return false;
+                }
+            }
+            else
+            {
+                Console.Error.WriteLine("[ExecutionHelper.CopyJobResources] resources = null");
+                return false;                 
+            }
+            return true;
+        }
+
+        /// <summary>
+        /// Create working directory for vertex
+        /// </summary>
+        /// <param name="id"></param>
+        /// <returns></returns>
+        public static bool InitializeForProcessExecution(int id, string resources)
+        {
+            try
+            {
+                Directory.CreateDirectory(ProcessPathHelper.ProcessPath(id));
+                Console.Error.WriteLine("Created directory: " + ProcessPathHelper.ProcessPath(id));
+                return true;
+            }
+            catch (Exception e)
+            {
+                Console.Error.WriteLine("[ExecutionHelper.InitializeForProcessExecution] Exception: {0}", e.Message);
+                Console.Error.WriteLine(e.StackTrace);
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Initialize the job directory for vertex execution
+        /// </summary>
+        /// <param name="resources">list of DryadLINQ-requested resources</param>
+        /// <returns>success/failure</returns>
+        public static bool InitializeForJobExecution(string resources)
+        {
+            try
+            {
+                //
+                // Update list of known local binaries if needed
+                //
+                InitializeBinFileList();
+
+                ProcessPathHelper.CreateUserWorkingDirectory();
+
+                Directory.CreateDirectory(ProcessPathHelper.JobPath);
+
+                //
+                // copy any files that already live locally and may be needed for the job
+                //
+                bool success = CopyLocalBinaries();
+
+                //
+                // copy any user-specified files that haven't already been copied
+                //
+                success &= CopyStagedJobResources(resources);
+                return success;
+            }
+            catch (Exception e)
+            {
+                //
+                // Write out any errors and return false on exception
+                //
+                Console.Error.WriteLine("[ExecutionHelper.InitializeForJobExecution] Exception: {0}", e.Message);
+                Console.Error.WriteLine(e.StackTrace);
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Checks if resource is one of the binaries already on the compute node
+        /// </summary>
+        /// <param name="resourceName">name of resource to check</param>
+        public static bool IsLocalResource(string resourceName)
+        {
+            InitializeBinFileList();
+
+            return ((from myfile in binFileList
+                    where string.Compare(resourceName, myfile, StringComparison.OrdinalIgnoreCase) == 0
+                    select myfile).Count() == 1);
+        }
+    }
+}

CommonCode/IDryadVertexCallback.cs

@@ -0,0 +1,80 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+namespace Microsoft.Research.Dryad
+{
+    using System;
+    using System.Collections.Generic;
+    using System.ServiceModel;
+    using System.Runtime.Serialization;
+
+    [DataContract]
+    public class ProcessStatistics
+    {
+        [DataMember]
+        public uint flags;
+        [DataMember]
+        public long processUserTime;
+        [DataMember]
+        public long processKernelTime;
+        [DataMember]
+        public int pageFaults;
+        [DataMember]
+        public int totalProcessesCreated;
+        [DataMember]
+        public ulong peakVMUsage;
+        [DataMember]
+        public ulong peakMemUsage;
+        [DataMember]
+        public ulong memUsageSeconds;
+        [DataMember]
+        public ulong totalIo;    
+    };
+
+    [DataContract]
+    public class ProcessInfo
+    {
+        [DataMember]
+        public uint flags;
+        [DataMember]
+        public ProcessState processState;
+        [DataMember]
+        public uint processStatus;
+        [DataMember]
+        public uint exitCode;
+        [DataMember]
+        public ProcessPropertyInfo[] propertyInfos;
+        [DataMember]
+        public ProcessStatistics processStatistics;
+    };
+            
+    [ServiceContract(SessionMode = SessionMode.Allowed)]
+    public interface IDryadVertexCallback
+    {
+        [OperationContract]
+        void FireStateChange(int processId, ProcessState newState);
+
+        [OperationContract]
+        void SetGetPropsComplete(int processId, ProcessInfo info, string[] propertyLabels, ulong[] propertyVersions);        
+       
+        [OperationContract]
+        void ProcessExited(int processId, int exitCode);
+    }
+}

CommonCode/IDryadVertexService.cs

@@ -0,0 +1,204 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//------------------------------------------------------------------------------
+// <summary>
+//      Vertex Service contracts
+// </summary>
+//------------------------------------------------------------------------------
+
+namespace Microsoft.Research.Dryad
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Collections.Specialized;
+    using System.Runtime.Serialization;
+    using System.ServiceModel;
+    using System.Diagnostics;
+
+    /// <summary>
+    /// Holds property information
+    /// </summary>
+    [DataContract]
+    public class ProcessPropertyInfo
+    {
+        [DataMember]
+        public string propertyLabel;
+        [DataMember]
+        public ulong propertyVersion;
+        [DataMember]
+        public string propertyString;
+        [DataMember]
+        public byte[] propertyBlock;
+    }
+
+    /// <summary>
+    /// Keeps track of vertex host process information
+    /// </summary>
+    [DataContract]
+    public class VertexProcessInfo
+    {
+        [DataMember]
+        public int DryadId;
+        [DataMember]
+        public string commandLine;
+        [DataMember]
+        public ProcessState State;        
+    }
+    
+    /// <summary>
+    /// Keeps track of CPU and memory info for vertex host process
+    /// </summary>
+    [DataContract]
+    public class VertexStatus
+    {
+        [DataMember]
+        public bool serviceIsAlive = false;
+        [DataMember]
+        public Dictionary<string,ulong> freeDiskSpaces = new Dictionary<string,ulong>();
+        [DataMember]
+        public ulong freeVirtualMemory = 0;
+        [DataMember]
+        public ulong freePhysicalMemory = 0;
+        [DataMember]
+        public uint runningProcessCount = 0;
+        [DataMember]
+        public List<VertexProcessInfo> vps = new List<VertexProcessInfo>();
+    }
+
+    [DataContract(Namespace = "http://hpc.microsoft.com/dryadvertex/")]
+    [Serializable]
+    public class VertexServiceError
+    {
+        public const string Action = "http://hpc.microsoft.com/dryadvertex/VertexServiceError";
+
+        /// <summary>
+        /// Stores the Operation
+        /// </summary>
+        [DataMember]
+        private string operation;
+
+        /// <summary>
+        /// Stores the Reason
+        /// </summary>
+        [DataMember]
+        private string reason;
+
+        /// <summary>
+        /// Initializes a new instance of the VertexServiceError class
+        /// </summary>
+        /// <param name="operation">The operation that failed</param>
+        /// <param name="reason">The detailed reason for the failure (exception.ToString())</param>
+        public VertexServiceError(string operation, string reason)
+        {
+            this.operation = operation;
+            this.reason = reason;
+        }
+
+        /// <summary>
+        /// The detailed reason for the failure
+        /// </summary>
+        public string Reason
+        {
+            get
+            {
+                return this.reason;
+            }
+        }
+
+        /// <summary>
+        /// The operation that failed
+        /// </summary>
+        public string Operation
+        {
+            get
+            {
+                return this.operation;
+            }
+        }
+    }
+
+    [DataContract(Namespace = "http://hpc.microsoft.com/dryadvertex/")]
+    [Serializable]
+    public class UnknownProcessError
+    {
+        public const string Action = "http://hpc.microsoft.com/dryadvertex/UnknownProcessError";
+
+        /// <summary>
+        /// Stores the ProcessId
+        /// </summary>
+        [DataMember]
+        private int processId;
+
+        /// <summary>
+        /// Initializes a new instance of the UnknownProcessError class
+        /// </summary>
+        /// <param name="id">Id of the unknown process</param>
+        public UnknownProcessError(int id)
+        {
+            this.processId = id;
+        }
+
+        /// <summary>
+        /// The process id which was not found
+        /// </summary>
+        public int Processid
+        {
+            get
+            {
+                return this.processId;
+            }
+        }
+
+    }
+
+    /// <summary>
+    /// Dryad Vertex Service Contract - allows GM to schedule vertices and VH to report status
+    /// </summary>
+    [ServiceContract(Name = "IDryadVertexService", Namespace = "http://hpc.microsoft.com/dryadvertex/", SessionMode = SessionMode.Allowed)]
+    public partial interface IDryadVertexService
+    {
+        [OperationContract(IsOneWay=true, Action = "http://hpc.microsoft.com/dryadvertex/cancelscheduleprocess")]
+        void CancelScheduleProcess(int processId);
+
+        // TODO: Deprecated.
+        [OperationContract(Action = "http://hpc.microsoft.com/dryadvertex/checkstatus")]
+        VertexStatus CheckStatus();
+
+        [OperationContract(IsOneWay = true, Action = "http://hpc.microsoft.com/dryadvertex/initialize")]
+        void Initialize(StringDictionary vertexEndpointAddresses);
+
+        [OperationContract(IsOneWay=true, Action = "http://hpc.microsoft.com/dryadvertex/releaseprocess")]
+        void ReleaseProcess(int processId);
+
+        [OperationContract(Action = "http://hpc.microsoft.com/dryadvertex/scheduleprocess")]
+        [FaultContract(typeof(VertexServiceError), Action = VertexServiceError.Action)]
+        bool ScheduleProcess(string replyUri, int processId, string commandLine, StringDictionary environment);
+
+        [OperationContract(Action = "http://hpc.microsoft.com/dryadvertex/setgetprops")]
+        [FaultContract(typeof(VertexServiceError), Action = VertexServiceError.Action)]
+        [FaultContract(typeof(UnknownProcessError), Action = UnknownProcessError.Action)]
+        bool SetGetProps(string replyUri, int processId, ProcessPropertyInfo[] infos, string blockOnLabel, ulong blockOnVersion, long maxBlockTime, string getPropLabel, bool ProcessStatistics);        
+
+        [OperationContract(IsOneWay=true, Action = "http://hpc.microsoft.com/dryadvertex/shutdown")]
+        void Shutdown(uint ShutdownCode);
+    }
+
+}

CommonCode/NativeMethods.cs

@@ -0,0 +1,453 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//------------------------------------------------------------------------------
+// <summary>
+//      Wrapped native methods
+// </summary>
+//------------------------------------------------------------------------------
+namespace Microsoft.Research.Dryad
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Text;
+    using System.Runtime.InteropServices;
+    using System.Runtime.ConstrainedExecution;
+    using System.Security;
+    using System.Security.Permissions;
+    using Microsoft.Win32.SafeHandles;
+
+    [SecurityPermission(SecurityAction.InheritanceDemand, UnmanagedCode = true)]
+    [SecurityPermission(SecurityAction.Demand, UnmanagedCode = true)]
+    public sealed class SafeThreadHandle : SafeHandleZeroOrMinusOneIsInvalid
+    {
+        private SafeThreadHandle() : base(true) { }
+
+        public SafeThreadHandle(IntPtr handle)
+            : base(false)
+        {
+            this.SetHandle(handle);
+        }
+
+        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
+        protected override bool ReleaseHandle()
+        {
+            return NativeMethods.CloseHandle(this.handle);
+        }
+    }
+
+
+    [SecurityPermission(SecurityAction.InheritanceDemand, UnmanagedCode = true)]
+    [SecurityPermission(SecurityAction.Demand, UnmanagedCode = true)]
+    public sealed class SafeProcessHandle : SafeHandleZeroOrMinusOneIsInvalid
+    {
+        private SafeProcessHandle() : base(true) { }
+
+        public SafeProcessHandle(IntPtr handle)
+            : base(false)
+        {
+            this.SetHandle(handle);
+        }
+
+        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
+        protected override bool ReleaseHandle()
+        {
+            return NativeMethods.CloseHandle(this.handle);
+        }
+    }
+    
+    [SecurityPermission(SecurityAction.InheritanceDemand, UnmanagedCode = true)]
+    [SecurityPermission(SecurityAction.Demand, UnmanagedCode = true)]
+    public sealed class SafeImpersonationToken : SafeHandleZeroOrMinusOneIsInvalid
+    {
+        private SafeImpersonationToken() : base(true) { }
+
+        public SafeImpersonationToken(IntPtr token)
+            : base(false)
+        {
+            this.SetHandle(token);
+        }
+
+        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
+        protected override bool ReleaseHandle()
+        {
+            return NativeMethods.CloseHandle(this.handle);
+        }
+
+    }
+
+    /// <summary>
+    /// Wrapped native methods
+    /// </summary>
+    [SuppressUnmanagedCodeSecurity]
+    public static class NativeMethods
+    {
+        public static readonly IntPtr INVALID_HANDLE_VALUE = new IntPtr(-1);
+
+        // Create process
+        public const uint CREATE_UNICODE_ENVIRONMENT = 0x00000400;
+        public const uint CREATE_SUSPENDED = 0x00000004;
+        public const uint CREATE_BREAKAWAY_FROM_JOB = 0x01000000;
+        public const uint CREATE_NO_WINDOW = 0x08000000;
+        
+        // LogonUser
+        public const uint LOGON32_LOGON_INTERACTIVE = 0x00000002;
+        public const uint LOGON32_LOGON_NETWORK = 0x00000003;
+        public const uint LOGON32_LOGON_BATCH = 0x00000004;
+        public const uint LOGON32_LOGON_NETWORK_CLEARTEXT = 0x00000008;
+        public const uint LOGON32_PROVIDER_DEFAULT = 0x00000000;
+
+
+
+        /// <summary>
+        /// Error flag for "no error"
+        /// </summary>
+        public const int ERROR_OK = 0;
+
+        /// <summary>
+        /// Error flag for insufficient buffer
+        /// </summary>
+        public const int ERROR_INSUFFICIENT_BUFFER = 122;
+
+
+        // Job object
+        public const int JobObjectExtendedLimitInformationClass = 9;
+        public const uint JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000;
+
+        [StructLayout(LayoutKind.Sequential)]
+        public struct PROCESS_INFORMATION
+        {
+            public IntPtr hProcess;
+            public IntPtr hThread;
+            public int dwProcessId;
+            public int dwThreadId;
+        }
+
+        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
+        public struct STARTUPINFO
+        {
+            public Int32 cb;
+            public string lpReserved;
+            public string lpDesktop;
+            public string lpTitle;
+            public Int32 dwX;
+            public Int32 dwY;
+            public Int32 dwXSize;
+            public Int32 dwYSize;
+            public Int32 dwXCountChars;
+            public Int32 dwYCountChars;
+            public Int32 dwFillAttribute;
+            public Int32 dwFlags;
+            public Int16 wShowWindow;
+            public Int16 cbReserved2;
+            public IntPtr lpReserved2;
+            public IntPtr hStdInput;
+            public IntPtr hStdOutput;
+            public IntPtr hStdError;
+        }
+
+        [StructLayout(LayoutKind.Sequential)]
+        public struct JobObjectExtendedLimitInformation
+        {
+            public Int64 PerProcessUserTimeLimit;
+            public Int64 PerJobUserTimeLimit;
+            public UInt32 LimitFlags;
+            public UIntPtr MinimumWorkingSetSize;
+            public UIntPtr MaximumWorkingSetSize;
+            public UInt32 ActiveProcessLimit;
+            public IntPtr Affinity;
+            public UInt32 PriorityClass;
+            public UInt32 SchedulingClass;
+            public UInt64 ReadOperationCount;
+            public UInt64 WriteOperationCount;
+            public UInt64 OtherOperationCount;
+            public UInt64 ReadTransferCount;
+            public UInt64 WriteTransferCount;
+            public UInt64 OtherTransferCount;
+            public UIntPtr ProcessMemoryLimit;
+            public UIntPtr JobMemoryLimit;
+            public UIntPtr PeakProcessMemoryUsed;
+            public UIntPtr PeakJobMemoryUsed;
+        }
+
+        [StructLayout(LayoutKind.Sequential)]
+        public struct SECURITY_ATTRIBUTES
+        {
+            public UInt32 nLength;
+            public UIntPtr lpSecurityAttributes;
+        }
+
+        [DllImport("Kernel32.dll", CallingConvention = CallingConvention.Winapi, SetLastError = true, CharSet = CharSet.Unicode)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        public static extern bool CreateProcess([MarshalAs(UnmanagedType.LPTStr)]string lpApplicationName,
+           StringBuilder lpCommandLine, IntPtr lpProcessAttributes,
+           IntPtr lpThreadAttributes, bool bInheritHandles,
+           uint dwCreationFlags, IntPtr lpEnvironment, string lpCurrentDirectory,
+           [In] ref STARTUPINFO lpStartupInfo,
+           out PROCESS_INFORMATION lpProcessInformation);
+
+        [DllImport("Advapi32.dll", CallingConvention = CallingConvention.Winapi, SetLastError = true, CharSet = CharSet.Unicode)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        public static extern bool CreateProcessAsUser(SafeImpersonationToken hToken, [MarshalAs(UnmanagedType.LPTStr)]string lpApplicationName,
+           StringBuilder lpCommandLine, IntPtr lpProcessAttributes,
+           IntPtr lpThreadAttributes, bool bInheritHandles,
+           uint dwCreationFlags, IntPtr lpEnvironment, string lpCurrentDirectory,
+           [In] ref STARTUPINFO lpStartupInfo,
+           out PROCESS_INFORMATION lpProcessInformation);
+
+        [DllImport("Advapi32.dll", SetLastError = true)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        public extern static bool LogonUser(string lpszUserName, string lpszDomain, string lpszPassword,
+            uint dwLogonType, uint dwLogonProvider, out SafeImpersonationToken phToken
+            );
+
+        [DllImport("Kernel32.dll", CallingConvention = CallingConvention.Winapi, SetLastError = true)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        public static extern bool GetExitCodeProcess(SafeProcessHandle hProcess, out uint lpExitCode);
+
+        [DllImport("Kernel32.dll", CallingConvention = CallingConvention.Winapi, SetLastError = true)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        public static extern bool TerminateProcess(SafeProcessHandle hProcess, int uExitCode);
+
+        [DllImport("Kernel32.dll", CallingConvention = CallingConvention.Winapi, SetLastError = true)]
+        public static extern uint ResumeThread(SafeThreadHandle hThread);
+
+        [DllImport("kernel32.dll", CallingConvention = CallingConvention.Winapi, SetLastError = true, CharSet = CharSet.Unicode)]
+        public static extern IntPtr CreateJobObject(IntPtr lpJobAttributes, string lpName);
+
+        public const int JobObjectExtendedLimitInformationQuery = 9;
+        public const int JobObjectExtendedLimitInformationSet = 9;
+
+        public const int QueryJobObjectBasicProcessIdList = 3;
+        [StructLayout(LayoutKind.Sequential)]
+        public struct JobObjectBasicProcessIdListHeader
+        {
+            public UInt32 NumberOfAssignedProcesses;
+            public UInt32 NumberOfProcessIdsInList;
+        }
+        
+        [DllImport("kernel32.dll", CharSet = System.Runtime.InteropServices.CharSet.Auto, SetLastError = true)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        public extern static bool QueryInformationJobObject(
+            IntPtr hJob,
+            int query,
+            out JobObjectExtendedLimitInformation info,
+            int size,
+            out int returnedSize
+            );
+        
+        [DllImport("kernel32.dll", CallingConvention = CallingConvention.Winapi, SetLastError = true)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        public extern static bool SetInformationJobObject(IntPtr hJob, int informationClass, [In] ref JobObjectExtendedLimitInformation info, int size);
+
+        [DllImport("kernel32.dll", CallingConvention = CallingConvention.Winapi, SetLastError = true)]
+        [return: MarshalAs(UnmanagedType.Bool)]
+        public extern static bool AssignProcessToJobObject(IntPtr hJob, SafeProcessHandle hProcess);
+
+        [DllImport("kernel32.dll", SetLastError = true)]
+        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)]
+        public extern static bool CloseHandle(IntPtr handle);
+
+        [DllImport("kernel32.dll", SetLastError = true)]
+        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)]
+        public extern static bool CloseHandle(HandleRef handleRef);
+
+        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success)]
+        public static void SafeCloseValidHandle(HandleRef handleRef)
+        {
+            if (handleRef.Handle != IntPtr.Zero && handleRef.Handle != INVALID_HANDLE_VALUE)
+            {
+                try
+                {
+                    CloseHandle(handleRef);
+                }
+                catch
+                {
+                    // Swallow exception
+                }
+            }
+        }
+
+
+        /// <summary>
+        /// contains information about the current state of both physical and virtual memory, including extended memory
+        /// </summary>
+        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)]
+        public class MEMORYSTATUSEX
+        {
+            /// <summary>
+            /// Size of the structure, in bytes. You must set this member before calling GlobalMemoryStatusEx. 
+            /// </summary>
+            public uint dwLength;
+
+            /// <summary>
+            /// Number between 0 and 100 that specifies the approximate percentage of physical memory that is in use (0 indicates no memory use and 100 indicates full memory use). 
+            /// </summary>
+            public uint dwMemoryLoad;
+
+            /// <summary>
+            /// Total size of physical memory, in bytes.
+            /// </summary>
+            public ulong ullTotalPhys;
+
+            /// <summary>
+            /// Size of physical memory available, in bytes. 
+            /// </summary>
+            public ulong ullAvailPhys;
+
+            /// <summary>
+            /// Size of the committed memory limit, in bytes. This is physical memory plus the size of the page file, minus a small overhead. 
+            /// </summary>
+            public ulong ullTotalPageFile;
+
+
+
+            /// <summary>
+            /// Size of available memory to commit, in bytes. The limit is ullTotalPageFile. 
+            /// </summary>
+            public ulong ullAvailPageFile;
+
+            /// <summary>
+            /// Total size of the user mode portion of the virtual address space of the calling process, in bytes. 
+            /// </summary>
+            public ulong ullTotalVirtual;
+
+            /// <summary>
+            /// Size of unreserved and uncommitted memory in the user mode portion of the virtual address space of the calling process, in bytes. 
+            /// </summary>
+            public ulong ullAvailVirtual;
+
+            /// <summary>
+            /// Size of unreserved and uncommitted memory in the extended portion of the virtual address space of the calling process, in bytes. 
+            /// </summary>
+            public ulong ullAvailExtendedVirtual;
+
+            /// <summary>
+            /// Initializes a new instance of the <see cref="T:MEMORYSTATUSEX"/> class.
+            /// </summary>
+            public MEMORYSTATUSEX()
+            {
+                this.dwLength = (uint)Marshal.SizeOf(typeof(NativeMethods.MEMORYSTATUSEX));
+            }
+        }
+
+        /// <summary>
+        /// Retrieves information about the system's current usage of both physical and virtual memory.
+        /// </summary>
+        /// <param name="lpBuffer">A pointer to a MEMORYSTATUSEX structure that receives information about current memory availability</param>
+        /// <returns>If the function succeeds, the return value is nonzero. Error code otherwise.</returns>
+        [return: MarshalAs(UnmanagedType.Bool)]
+        [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
+        public static extern bool GlobalMemoryStatusEx([In, Out] MEMORYSTATUSEX lpBuffer);
+
+        /// <summary>
+        /// Retrieves information about the amount of space that is available on a disk volume, which is the total amount of space, 
+        /// the total amount of free space, and the total amount of free space available to the user that is associated with the calling thread.
+        /// </summary>
+        /// <param name="lpDirectoryName">A directory on the disk.</param>
+        /// <param name="lpFreeBytesAvailable">A pointer to a variable that receives the total number of free bytes on a disk that are available to the user who is associated with the calling thread.</param>
+        /// <param name="lpTotalNumberOfBytes">A pointer to a variable that receives the total number of bytes on a disk that are available to the user who is associated with the calling thread.</param>
+        /// <param name="lpTotalNumberOfFreeBytes">A pointer to a variable that receives the total number of free bytes on a disk.</param>
+        /// <returns></returns>
+        [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)]
+        public static extern bool GetDiskFreeSpaceEx(string lpDirectoryName,
+           out ulong lpFreeBytesAvailable,
+           out ulong lpTotalNumberOfBytes,
+           out ulong lpTotalNumberOfFreeBytes);
+
+        /// <summary>
+        /// SID Usage Enum
+        /// </summary>
+        public enum SID_NAME_USE
+        {
+            SidTypeUser = 1,
+            SidTypeGroup,
+            SidTypeDomain,
+            SidTypeAlias,
+            SidTypeWellKnownGroup,
+            SidTypeDeletedAccount,
+            SidTypeInvalid,
+            SidTypeUnknown,
+            SidTypeComputer
+        }
+
+        /// <summary>
+        /// Get SID for account name
+        /// </summary>
+        /// <param name="lpSystemName">Compute name</param>
+        /// <param name="lpAccountName">Account name</param>
+        /// <param name="Sid">Security ID</param>
+        /// <param name="cbSid">Number of bytes needed to hold the SID</param>
+        /// <param name="ReferencedDomainName">Domain name reference by SID</param>
+        /// <param name="cchReferencedDomainName">Number of bytes needed to hold the domain</param>
+        /// <param name="peUse">Account type</param>
+        /// <returns>error flag</returns>
+        [DllImport("advapi32.dll", CharSet = CharSet.Auto, SetLastError = true)]
+        public static extern bool LookupAccountName(
+            string lpSystemName,
+            string lpAccountName,
+            [MarshalAs(UnmanagedType.LPArray)] byte[] Sid,
+            ref uint cbSid,
+            StringBuilder ReferencedDomainName,
+            ref uint cchReferencedDomainName,
+            out SID_NAME_USE peUse);
+
+        /// <summary>
+        /// Retrieves the name of the account for this SID and the name of the first domain on which this SID is found
+        /// </summary>
+        /// <param name="lpSystemName">string that specifies the target computer</param>
+        /// <param name="Sid">the SID to look up</param>
+        /// <param name="lpName">buffer that receives the account name that corresponds to the Sid parameter</param>
+        /// <param name="cchName">On input, specifies the size of the lpName buffer. If the function fails because 
+        ///                       the buffer is too small or if cchName is zero, cchName receives the required buffer size</param>
+        /// <param name="ReferencedDomainName">buffer that receives the name of the domain where the account name was found.</param>
+        /// <param name="cchReferencedDomainName">Same as cchName, but for the domain string buffer</param>
+        /// <param name="peUse">pointer to a variable that receives a SID_NAME_USE value that indicates the type of the account</param>
+        /// <returns>If the function succeeds, the function returns nonzero.If the function fails, it returns zero</returns>
+        [DllImport("advapi32.dll", CharSet = CharSet.Auto, SetLastError = true)]
+        public static extern bool LookupAccountSid(
+            string lpSystemName,
+            [MarshalAs(UnmanagedType.LPArray)] byte[] Sid,
+            StringBuilder lpName,
+            ref uint cchName,
+            StringBuilder ReferencedDomainName,
+            ref uint cchReferencedDomainName,
+            out SID_NAME_USE peUse);  
+
+        /// <summary>
+        /// Converts a security ID pointer to the string value
+        /// </summary>
+        /// <param name="pSID">pointer to SID</param>
+        /// <param name="ptrSid">string value</param>
+        /// <returns>error flag</returns>
+        [DllImport("advapi32", CharSet = CharSet.Auto, SetLastError = true)]
+        public static extern bool ConvertSidToStringSid(
+            [MarshalAs(UnmanagedType.LPArray)] byte[] pSID,
+            out IntPtr ptrSid);
+
+        /// <summary>
+        /// Frees a pointer
+        /// </summary>
+        /// <param name="hMem">pointer to free</param>
+        /// <returns>error flag</returns>
+        [DllImport("kernel32.dll")]
+        public static extern IntPtr LocalFree(IntPtr hMem);
+    }
+}

CommonCode/NetShareWrapper.cs

@@ -0,0 +1,519 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//------------------------------------------------------------------------------
+// <summary>
+//      Provide access to native APIs in Netapi32.dll
+// </summary>
+//------------------------------------------------------------------------------
+
+namespace Microsoft.Hpc.Dsc.Internal
+{
+    using System;
+    using System.Collections;
+    using System.Collections.Generic;
+    using System.Globalization;
+    using System.Runtime.InteropServices;
+    using System.Text;
+
+
+    /// <summary>
+    /// Return codes from the Netapi32.dll pinvoke calls
+    /// </summary>
+    internal enum NetShareError
+    {
+        /// <summary>
+        /// Success
+        /// </summary>
+        NERR_Success = 0,
+
+        /// <summary>
+        /// The user does not have access to the requested information.
+        /// </summary>
+        ERROR_ACCESS_DENIED = 5,
+
+        /// <summary>
+        /// Not enough storage is available to process this command
+        /// </summary>
+        ERROR_NOT_ENOUGH_MEMORY = 8,
+
+        /// <summary>
+        /// The network path was not found
+        /// </summary>
+        ERROR_BAD_NETPATH = 53,
+
+        /// <summary>
+        /// The specified parameter is not valid.
+        /// </summary>
+        ERROR_INVALID_PARAMETER = 87,
+
+        /// <summary>
+        /// The value specified for the level parameter is not valid.
+        /// </summary>
+        ERROR_INVALID_LEVEL = 124,
+
+        /// <summary>
+        /// The provided buffer was too small to hold the entire value
+        /// </summary>
+        ERROR_MORE_DATA = 234,
+
+        /// <summary>
+        /// The filename, directory name, or volume label syntax is incorrect
+        /// </summary>
+        ERROR_INVALID_NAME = 123,
+
+        /// <summary>
+        /// The device or directory does not exist
+        /// </summary>
+        NERR_UnknownDevDir = 2116,
+
+        /// <summary>
+        /// The share name is already in use on this server
+        /// </summary>
+        NERR_DuplicateShare = 2118,
+
+        /// <summary>
+        /// The client request succeeded. More entries are available. The buffer size that is specified by PreferedMaximumLength was too small to fit even a single entry
+        /// </summary>
+        NERR_BufTooSmall = 2123,
+
+        /// <summary>
+        /// The share name does not exist
+        /// </summary>
+        NERR_NetNameNotFound = 2310,
+
+        /// <summary>
+        /// The operation is not valid for a redirected resource. The specified device name is assigned to a shared resource
+        /// </summary>
+        NERR_RedirectedPath = 2117
+    }
+
+    //
+    // Provide access to native APIs in Netapi32.dll
+    //
+    internal static class NetShareWrapper
+    {
+        /// <summary>
+        /// Specifies information about the shared resource, including the name of the resource, 
+        /// type and permissions, and number of connections. 
+        /// The buf parameter points to a SHARE_INFO_2 structure.
+        /// </summary>
+        private static UInt32 _InfoLevel = 2;
+
+        /// <summary>
+        /// Disk drive
+        /// </summary>
+        private static UInt32 _STYPE_DISKTREE = 0;
+
+        /// <summary>
+        /// Permission to read a resource and, by default, execute the resource.
+        /// </summary>
+        private static UInt32 _PERM_FILE_READ = 1;
+
+        /// <summary>
+        /// Permission to write to a resource.
+        /// </summary>
+        private static UInt32 _PERM_FILE_WRITE = 2;
+
+        /// <summary>
+        /// Permission to create a resource; data can be written when creating the resource.
+        /// </summary>
+        private static UInt32 _PERM_FILE_CREATE = 4;
+
+        /// <summary>
+        /// Contains information about the shared resource, including name of the resource, type and permissions, and the number of current connections
+        /// </summary>
+        [StructLayout(LayoutKind.Sequential)]
+        private struct _SHARE_INFO_2
+        {
+            /// <summary>
+            /// Pointer to a Unicode string specifying the share name of a resource
+            /// </summary>
+            [MarshalAs(UnmanagedType.LPWStr)]
+            public string shi2_netname;
+
+            /// <summary>
+            /// A bitmask of flags that specify the type of the shared resource
+            /// </summary>
+            [MarshalAs(UnmanagedType.U4)]
+            public UInt32 shi2_type;
+
+            /// <summary>
+            /// Pointer to a Unicode string that contains an optional comment about the shared resource
+            /// </summary>
+            [MarshalAs(UnmanagedType.LPWStr)]
+            public string shi2_remark;
+
+            /// <summary>
+            /// Specifies a DWORD value that indicates the shared resource's permissions for servers running with share-level security
+            /// </summary>
+            [MarshalAs(UnmanagedType.U4)]
+            public UInt32 shi2_permissions;
+
+            /// <summary>
+            /// Specifies a DWORD value that indicates the maximum number of concurrent connections that the shared resource can accommodate.
+            /// The number of connections is unlimited if the value specified in this member is <20>1.
+            /// </summary>
+            [MarshalAs(UnmanagedType.U4)]
+            public UInt32 shi2_max_uses;
+
+            /// <summary>
+            /// Specifies a DWORD value that indicates the number of current connections to the resource
+            /// </summary>
+            [MarshalAs(UnmanagedType.U4)]
+            public UInt32 shi2_current_uses;
+
+            /// <summary>
+            /// Pointer to a Unicode string specifying the local path for the shared resource
+            /// </summary>
+            [MarshalAs(UnmanagedType.LPWStr)]
+            public string shi2_path;
+
+            /// <summary>
+            /// Pointer to a Unicode string that specifies the share's password when the server is running with share-level security
+            /// </summary>
+            [MarshalAs(UnmanagedType.LPWStr)]
+            public string shi2_passwd;
+        }
+
+        /// <summary>
+        /// Shares a server resource.
+        /// </summary>
+        /// <param name="ServerName">Pointer to a string that specifies the DNS or NetBIOS name of the remote server on which the function is to execute. If this parameter is NULL, the local computer is used.</param>
+        /// <param name="InfoLevel">Specifies the information level of the data</param>
+        /// <param name="ShareInfo">Pointer to the buffer that specifies the data</param>
+        /// <param name="OutputBuffer">Pointer to a value that receives the index of the first member of the share information structure that causes the ERROR_INVALID_PARAMETER error. If this parameter is NULL, the index is not returned on error</param>
+        /// <returns></returns>
+        [DllImport("Netapi32.dll")]
+        private static extern Int32 NetShareAdd(
+            [MarshalAs(UnmanagedType.LPWStr)] string ServerName,
+            [MarshalAs(UnmanagedType.U4)] UInt32 InfoLevel,
+            [MarshalAs(UnmanagedType.Struct)] ref _SHARE_INFO_2 ShareInfo,
+            ref IntPtr OutputBuffer);
+
+        /// <summary>
+        /// Deletes a share name from a server's list of shared resources, disconnecting all connections to the shared resource.
+        /// </summary>
+        /// <param name="ServerName">Pointer to a string that specifies the DNS or NetBIOS name of the remote server on which the function is to execute. If this parameter is NULL, the local computer is used</param>
+        /// <param name="ShareName">Pointer to a string that specifies the name of the share to delete.</param>
+        /// <param name="ParameterReserved">Reserved, must be zero.</param>
+        /// <returns>If the function succeeds, the return value is NERR_Success</returns>
+        [DllImport("Netapi32.dll")]
+        private static extern Int32 NetShareDel(
+            [MarshalAs(UnmanagedType.LPWStr)] string ServerName,
+            [MarshalAs(UnmanagedType.LPWStr)] string ShareName,
+            UInt32 ParameterReserved);
+
+        /// <summary>
+        /// Retrieves information about a particular shared resource on a server.
+        /// </summary>
+        /// <param name="ServerName">Pointer to a string that specifies the DNS or NetBIOS name of the remote server on which the function is to execute. If this parameter is NULL, the local computer is used</param>
+        /// <param name="ShareName">Pointer to a string that specifies the name of the share for which to return information</param>
+        /// <param name="InfoLevel">Specifies the information level of the data</param>
+        /// <param name="OutputBuffer">Pointer to the buffer that receives the data</param>
+        /// <returns>If the function succeeds, the return value is NERR_Success.</returns>
+        [DllImport("Netapi32.dll")]
+        private static extern Int32 NetShareGetInfo(
+            [MarshalAs(UnmanagedType.LPWStr)] string ServerName,
+            [MarshalAs(UnmanagedType.LPWStr)] string ShareName,
+            [MarshalAs(UnmanagedType.U4)] UInt32 InfoLevel,
+            ref IntPtr OutputBuffer);
+
+        /// <summary>
+        /// The NetApiBufferFree function frees the memory that the NetApiBufferAllocate function allocates
+        /// </summary>
+        /// <param name="InputBuffer">A pointer to a buffer returned previously by another network management function or memory allocated by calling the NetApiBufferAllocate function</param>
+        /// <returns></returns>
+        [DllImport("Netapi32", CharSet = CharSet.Auto)]
+        private static extern Int32 NetApiBufferFree(IntPtr InputBuffer);
+
+        /// <summary>
+        /// Prints out the error based on the error code
+        /// </summary>
+        /// <param name="ErrorCode">Error code returned by pinvoke call</param>
+        private static void PrintError(Int32 ErrorCode)
+        {
+            switch ((NetShareError)ErrorCode)
+            {
+                case NetShareError.ERROR_ACCESS_DENIED:
+                    Console.Error.WriteLine("Access to share denied.");
+                    break;
+
+                case NetShareError.ERROR_NOT_ENOUGH_MEMORY:
+                    Console.Error.WriteLine("Not enough memory available.");
+                    break;
+
+                case NetShareError.ERROR_INVALID_PARAMETER:
+                    Console.Error.WriteLine("Invalid parameter specified.");
+                    break;
+
+                case NetShareError.ERROR_INVALID_LEVEL:
+                    Console.Error.WriteLine("Invalid level specified.");
+                    break;
+
+                case NetShareError.ERROR_MORE_DATA:
+                    Console.Error.WriteLine("More data available and not large enough buffer specified.");
+                    break;
+
+                case NetShareError.ERROR_INVALID_NAME:
+                    Console.Error.WriteLine("The filename, directory name, or volume label syntax is incorrect");
+                    break;
+
+                case NetShareError.NERR_UnknownDevDir:
+                    Console.Error.WriteLine("Unknown device specified.");
+                    break;
+
+                case NetShareError.NERR_DuplicateShare:
+                    Console.Error.WriteLine("Duplicate share specified.");
+                    break;
+
+                case NetShareError.NERR_BufTooSmall:
+                    Console.Error.WriteLine("Not large enough buffer specified.");
+                    break;
+
+                case NetShareError.NERR_NetNameNotFound:
+                    Console.Error.WriteLine("Share name not found.");
+                    break;
+
+                case NetShareError.NERR_RedirectedPath:
+                    Console.Error.WriteLine("The operation is not valid for a redirected resource. The specified device name is assigned to a shared resource.");
+                    break;
+
+                case NetShareError.ERROR_BAD_NETPATH:
+                    Console.Error.WriteLine("The network path was not found.");
+                    break;
+
+                default:
+                    Console.Error.WriteLine(String.Format(CultureInfo.CurrentCulture, "Unknown error occured (Error Code {0}).", ErrorCode));
+                    break;
+            }
+        }
+
+        /// <summary>
+        /// Create a net share
+        /// </summary>
+        /// <param name="ServerName">Server hosting share</param>
+        /// <param name="ShareName">name of share</param>
+        /// <param name="SharePath">path to share</param>
+        /// <param name="ShareDescription">description of share</param>
+        /// <returns>success/failure</returns>
+        internal static int CreateShare(string ServerName, string ShareName, string SharePath, string ShareDescription)
+        {
+            //
+            // Ensure UNC path formatting at start
+            //
+            if (!ServerName.StartsWith(@"\\"))
+            {
+                if (!ServerName.StartsWith(@"\"))
+                {
+                    ServerName = @"\\" + ServerName;
+                }
+                else
+                {
+                    ServerName = @"\" + ServerName;
+                }
+            }
+
+            //
+            // Build share information
+            //
+            _SHARE_INFO_2 ShareInfo = new _SHARE_INFO_2();
+            ShareInfo.shi2_netname = ShareName;
+            ShareInfo.shi2_type = _STYPE_DISKTREE;
+            ShareInfo.shi2_remark = ShareDescription;
+            ShareInfo.shi2_permissions = _PERM_FILE_READ | _PERM_FILE_WRITE | _PERM_FILE_CREATE;
+            ShareInfo.shi2_max_uses = UInt32.MaxValue;
+            ShareInfo.shi2_current_uses = 0;
+            ShareInfo.shi2_path = SharePath;
+            ShareInfo.shi2_passwd = String.Empty;
+
+            IntPtr OutputBuffer = IntPtr.Zero;
+
+            //
+            // Create the share and report success or failure
+            //
+            Int32 ErrorCode = NetShareAdd(ServerName, _InfoLevel, ref ShareInfo, ref OutputBuffer);
+
+
+            if (ErrorCode != (Int32)NetShareError.NERR_Success)
+            {
+                PrintError(ErrorCode);
+            }
+
+            return ErrorCode;
+        }
+
+        /// <summary>
+        /// Deletes an existing share
+        /// </summary>
+        /// <param name="ServerName">Server hosting share</param>
+        /// <param name="ShareName">name of share</param>
+        /// <returns>success/failure</returns>
+        internal static bool DeleteShare(string ServerName, string ShareName)
+        {
+            //
+            // Ensure UNC path formatting at start
+            //
+            if (!ServerName.StartsWith(@"\\"))
+            {
+                if (!ServerName.StartsWith(@"\"))
+                {
+                    ServerName = @"\\" + ServerName;
+                }
+                else
+                {
+                    ServerName = @"\" + ServerName;
+                }
+            }
+
+            //
+            // Attempt to delete the share and report success/failure
+            //
+            Int32 ErrorCode = NetShareDel(ServerName, ShareName, 0);
+            if (ErrorCode == (Int32)NetShareError.NERR_Success)
+            {
+                return (true);
+            }
+            else
+            {
+                PrintError(ErrorCode);
+            }
+
+            return (false);
+        }
+
+        /// <summary>
+        /// Get the drive where a share is hosted
+        /// </summary>
+        /// <param name="ServerName">Server hosting share</param>
+        /// <param name="ShareName">Name of share</param>
+        /// <returns>drive letter (empty string if unsuccessful)</returns>
+        internal static string GetLocalDrive(string ServerName, string ShareName)
+        {
+            //
+            // Ensure UNC path formatting at start
+            //
+            if (!ServerName.StartsWith(@"\\"))
+            {
+                if (!ServerName.StartsWith(@"\"))
+                {
+                    ServerName = @"\\" + ServerName;
+                }
+                else
+                {
+                    ServerName = @"\" + ServerName;
+                }
+            }
+
+            string ShareDrive = String.Empty;
+            string SharePath = String.Empty;
+            IntPtr OutputBuffer = IntPtr.Zero;
+
+            //
+            // Attempt to get the share information. Report error if unsuccessful
+            //
+            Int32 ErrorCode = NetShareGetInfo(ServerName, ShareName, _InfoLevel, ref OutputBuffer);
+            if (ErrorCode == (Int32)NetShareError.NERR_Success)
+            {
+                _SHARE_INFO_2 ShareInfo = (_SHARE_INFO_2)Marshal.PtrToStructure(OutputBuffer, typeof(_SHARE_INFO_2));
+                SharePath = ShareInfo.shi2_path;
+                NetApiBufferFree(OutputBuffer);
+            }
+            else
+            {
+                PrintError(ErrorCode);
+                return (ShareDrive);
+            }
+
+            if (!String.IsNullOrEmpty(SharePath))
+            {
+                //
+                // If a share path was returned, attempt to parse out the drive letter
+                //
+                int Index = SharePath.IndexOf(':');
+                if (Index > 0)
+                {
+                    ShareDrive = SharePath.Substring(0, Index + 1);
+                }
+            }
+
+            return (ShareDrive);
+        }
+
+        // TODO: Sync Dryad GM and vertex with new version that returns error code
+        internal static string GetLocalPath(string ServerName, string ShareName)
+        {
+            string SharePath;
+
+            int err = GetLocalPath(ServerName, ShareName, out SharePath);
+            if (err != (int)NetShareError.NERR_Success)
+            {
+                Console.Error.WriteLine("GetLocalPath failed: server {0}, share {1}", ServerName, ShareName);
+                PrintError(err);
+            }
+
+            return SharePath;
+        }
+
+        /// <summary>
+        /// Returns the local path to a share on the specified server
+        /// </summary>
+        /// <param name="ServerName">Server the path resides on</param>
+        /// <param name="ShareName">Share to find path to</param>
+        /// <returns>Local path to share or empty string if failure</returns>
+        internal static int GetLocalPath(string ServerName, string ShareName, out string SharePath)
+        {
+            //
+            // Ensure UNC path formatting at start
+            //
+            if (!String.IsNullOrEmpty(ServerName) && !ServerName.StartsWith(@"\\"))
+            {
+                if (!ServerName.StartsWith(@"\"))
+                {
+                    ServerName = @"\\" + ServerName;
+                }
+                else
+                {
+                    ServerName = @"\" + ServerName;
+                }
+            }
+
+            IntPtr OutputBuffer = IntPtr.Zero;
+            SharePath = String.Empty;
+
+            //
+            // Get share info structure
+            //
+            Int32 ErrorCode = NetShareGetInfo(ServerName, ShareName, _InfoLevel, ref OutputBuffer);
+
+            if (ErrorCode == (Int32)NetShareError.NERR_Success)
+            {
+                //
+                // If successful, get the local path to the resource and free the buffer
+                //
+                _SHARE_INFO_2 ShareInfo = (_SHARE_INFO_2)Marshal.PtrToStructure(OutputBuffer, typeof(_SHARE_INFO_2));
+                SharePath = ShareInfo.shi2_path;
+                NetApiBufferFree(OutputBuffer);
+            }
+
+            return ErrorCode;
+        }
+    }
+}

CommonCode/ProcessPathHelper.cs

@@ -0,0 +1,172 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//------------------------------------------------------------------------------
+// <summary>
+//      Helper class that builds working directories for dryad jobs
+// </summary>
+//------------------------------------------------------------------------------
+
+namespace Microsoft.Research.Dryad
+{
+    using System;
+    using System.IO;
+    using System.Security.Principal;
+    using System.Security.AccessControl;
+    using Microsoft.Hpc.Dsc.Internal;
+
+    /// <summary>
+    /// Helper class that builds working directories for dryad jobs
+    /// </summary>
+    public static class ProcessPathHelper
+    {
+        /// <summary>
+        /// Current job directory path
+        /// </summary>
+        private static string jobPath = string.Empty;
+
+        /// <summary>
+        /// Lockable object used to synchronize job path creation
+        /// </summary>
+        private static object jobPathLock = new object();
+
+        /// <summary>
+        /// Build the current job directory path
+        /// </summary>
+        private static void BuildJobPath()
+        {
+            //
+            // Get the path to the HpcTemp share on the local machine and append the current user name to build the job path.
+            // If unable to get local path to HpcTemp share, or necessary environment variables, leave jobPath as string.empty.
+            //
+            string rootDir = RootWorkingDirectory;
+
+            if (!string.IsNullOrEmpty(rootDir))
+            {
+                try
+                {
+                    string userName = Environment.GetEnvironmentVariable("USERNAME");
+                    if (!string.IsNullOrEmpty(userName))
+                    {
+                        //
+                        // If all goes well, assign the path to jobPath, so it can accessed.
+                        //
+                        ProcessPathHelper.jobPath = Path.Combine(Path.Combine(rootDir, userName), Environment.GetEnvironmentVariable("CCP_JOBID"));
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Console.Error.WriteLine("ProcessPathHelper.BuildJobPath: Unable to get environment variable: {0}", ex.Message);
+                }
+            }
+            else
+            {
+                Console.Error.WriteLine("ProcessPathHelper.BuildJobPath: Unable to get local path to {0} share.", Constants.DscTempShare);
+            }
+        }
+        
+        /// <summary>
+        /// Returns the path to the specified vertex dir
+        /// </summary>
+        /// <param name="id">Vertex id</param>
+        /// <returns>path to vertex directory</returns>
+        public static string ProcessPath(int id)
+        {
+            return Path.Combine(JobPath, id.ToString());
+        }
+
+        /// <summary>
+        /// Returns path to the specified vertex's working dir
+        /// </summary>
+        /// <param name="id">Vertex ID</param>
+        /// <returns>path to the vertex working directory</returns>
+        public static string ProcessWorkingDirectory(int id)
+        {
+            return Path.Combine(ProcessPath(id), "WD");
+        }
+
+        /// <summary>
+        /// Gets path to directory for the current job. 
+        /// If job path cannot be retrieved, string.empty may be returned.
+        /// </summary>
+        public static string JobPath
+        {
+            get
+            {
+                //
+                // If job path hasn't been built, build it.
+                // Lock to allow multiple vertices to access this property without issue.
+                //
+                if (string.IsNullOrEmpty(ProcessPathHelper.jobPath))
+                {
+                    lock (jobPathLock)
+                    {
+                        if (string.IsNullOrEmpty(ProcessPathHelper.jobPath))
+                        {
+                            ProcessPathHelper.BuildJobPath();
+                        }
+                    }
+                }
+
+                return ProcessPathHelper.jobPath;
+            }
+        }
+
+        /// <summary>
+        /// Get the root working directory
+        /// </summary>
+        public static string RootWorkingDirectory
+        {
+            get
+            {
+                return (NetShareWrapper.GetLocalPath("localhost", Constants.DscTempShare));
+            }
+        }
+
+
+        /// <summary>
+        /// 
+        /// </summary>
+        public static void CreateUserWorkingDirectory()
+        {
+            //Create user specific working directory if does not exist and set full control to the user and administrators only.
+
+            string userWorkingDirectory = String.Format(@"{0}\{1}", ProcessPathHelper.RootWorkingDirectory, Environment.UserName);
+
+            if (!Directory.Exists(userWorkingDirectory))
+            {
+                DirectorySecurity directorySecurity = new DirectorySecurity();
+
+                //Add full control to user
+
+                directorySecurity.AddAccessRule(new FileSystemAccessRule(WindowsIdentity.GetCurrent().Name, FileSystemRights.FullControl, InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit, PropagationFlags.None, AccessControlType.Allow));
+
+                //Add full control to administrators
+
+                SecurityIdentifier administratorGroup = new SecurityIdentifier(WellKnownSidType.BuiltinAdministratorsSid, null);
+
+                directorySecurity.AddAccessRule(new FileSystemAccessRule(administratorGroup, FileSystemRights.FullControl, InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit, PropagationFlags.None, AccessControlType.Allow));
+
+                Directory.CreateDirectory(userWorkingDirectory, directorySecurity);
+            }
+
+        }
+    }
+}

CommonCode/ProcessState.cs

@@ -0,0 +1,38 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+namespace Microsoft.Research.Dryad
+{
+    [System.Flags]
+    public enum ProcessState
+    {
+        // Initial states
+        Uninitialized       = 0x00,
+        Unscheduled         = 0x01,
+
+        // "Running" states
+        AssignedToNode      = 0x10,
+        Running             = 0x11,
+
+        // Terminal states
+        SchedulingFailed    = 0x20,
+        Completed           = 0x21
+    }
+}

CommonCode/QueryUtility.cs

@@ -0,0 +1,85 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+using System;
+
+namespace Microsoft.Research.Dryad
+{
+    class HpcQueryUtility
+    {
+        /// <summary>
+        /// Converts numeric trace level to string representation
+        /// </summary>
+        /// <returns></returns>
+        internal static string ConvertTraceLevelToString(int runtimeTraceLevel)
+        {
+            switch (runtimeTraceLevel)
+            {
+                case Constants.traceOffNum:
+                    return Constants.traceOff;
+
+                case Constants.traceCriticalNum:
+                    return Constants.traceCritical;
+
+                case Constants.traceErrorNum:
+                    return Constants.traceError;
+
+                case Constants.traceWarningNum:
+                    return Constants.traceWarning;
+
+                case Constants.traceInfoNum:
+                    return Constants.traceInfo;
+
+                case Constants.traceVerboseNum:
+                    return Constants.traceVerbose;
+
+                default:
+                    return Constants.traceError;
+            };
+        }
+
+        internal static int GetTraceLevelFromString(string runtimeTraceLevel)
+        {
+            switch (runtimeTraceLevel)
+            {
+                case Constants.traceOff:
+                    return Constants.traceOffNum;
+
+                case Constants.traceCritical:
+                    return Constants.traceCriticalNum;
+
+                case Constants.traceError:
+                    return Constants.traceErrorNum;
+
+                case Constants.traceWarning:
+                    return Constants.traceWarningNum;
+
+                case Constants.traceInfo:
+                    return Constants.traceInfoNum;
+
+                case Constants.traceVerbose:
+                    return Constants.traceVerboseNum;
+
+                default:
+                    return Constants.traceErrorNum;
+            };
+        }
+    }
+}

CommonCode/RetryFramework.cs

@@ -0,0 +1,358 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//------------------------------------------------------------------------------
+// Security review: nzeng 01-11-06
+//------------------------------------------------------------------------------
+
+#region Using directives
+
+using System;
+using System.Text;
+using System.Diagnostics;
+using System.Threading;
+
+#endregion
+
+namespace Microsoft.Hpc
+{
+    internal class RetryManager
+    {
+        public const int InfiniteRetries = -1;
+
+        RetryWaitTimer _waitTimer;
+
+        int _maxRetries;
+        int _totalTimeLimit = Timeout.Infinite;
+
+        int _retryCount = 0;
+        int _totalWaitTime = 0;
+        int _currentWaitTime = 0;
+
+        public RetryManager(RetryWaitTimer waitTimer) : this(waitTimer, InfiniteRetries) { }
+        public RetryManager(RetryWaitTimer waitTimer, int maxRetries) : this(waitTimer, maxRetries, Timeout.Infinite) { }
+
+        public RetryManager(RetryWaitTimer waitTimer, int maxRetries, int totalTimeLimit)
+        {
+            if (waitTimer == null)
+            {
+                throw new ArgumentNullException("wait");
+            }
+            _waitTimer = waitTimer;
+
+            SetMaxRetries(maxRetries);
+            SetTotalTimeLimit(totalTimeLimit);
+        }
+
+
+        /// <summary>
+        /// Gets the number of retries attempted thus far
+        /// </summary>
+        public int RetryCount       { get { return _retryCount;  } }
+
+        /// <summary>
+        /// Get the total spent waiting between retries
+        /// </summary>
+        public int ElaspsedWaitTime { get { return _totalWaitTime; } }
+
+        /// <summary>
+        /// Gets or sets the maximum number of retries
+        /// </summary>
+        public int MaxRetryCount
+        {
+            get { return _maxRetries; }
+            set { SetMaxRetries(value); }
+        }
+
+        /// <summary>
+        /// Gets or sets the total amount of time that may be spend waiting for retries.        
+        /// </summary>
+        public int TotalTimeLimit
+        {
+            get { return _totalTimeLimit; }
+            set { SetTotalTimeLimit(value); }
+        }
+
+        void SetMaxRetries(int n)
+        {
+            if (n <= 0 && n != RetryManager.InfiniteRetries)
+            {
+                throw new ArgumentException("The maximum number of retries must be greater than zero, or RetryOperator.InfiniteRetries");
+            }
+            _maxRetries = n;
+        }
+
+        void SetTotalTimeLimit(int t)
+        {
+            if (t <= 0 && t != Timeout.Infinite)
+            {
+                throw new ArgumentException("The specified time must be greater than zero, or Timeout.Infinite");
+            }
+            _totalTimeLimit = t;
+        }
+
+
+        /// <summary>
+        /// Returns true if there are more retries left
+        /// </summary>
+        public bool HasAttemptsLeft
+        {
+            get
+            {
+                return ((_maxRetries == RetryManager.InfiniteRetries || _retryCount < _maxRetries)
+                        && (_totalTimeLimit == Timeout.Infinite || _totalWaitTime < _totalTimeLimit));
+            }
+        }
+
+        /// <summary>
+        /// Get the next wait time
+        /// </summary>
+        public int NextWaitTime
+        {
+            get
+            {
+                int waitTime = _waitTimer.GetNextWaitTime(_retryCount, _currentWaitTime);
+                if (_totalTimeLimit != Timeout.Infinite && (_totalWaitTime + waitTime > _totalTimeLimit))
+                {
+                    waitTime = _totalTimeLimit - _totalWaitTime;
+                }
+                return waitTime;
+            }
+        }
+
+        /// <summary>
+        /// Increment the retry count and advance the total wait time without actually waiting
+        /// </summary>
+        public void SimulateNextAttempt()
+        {
+            WaitForNextAttempt(false);
+        }
+
+
+        /// <summary>
+        /// Wait until the next retry by making the current thread sleep for the appropriate amount of time.
+        /// May return immediately if the wait is zero.
+        /// </summary>
+        public void WaitForNextAttempt()
+        {
+            WaitForNextAttempt(true);
+        }
+
+       
+        void WaitForNextAttempt(bool doSleep)
+        {
+            if (!HasAttemptsLeft)
+            {
+                throw new InvalidOperationException("There are no more retry attempts remaining");
+            }
+
+            _currentWaitTime = NextWaitTime;
+            _retryCount++;
+
+            Debug.Assert(_currentWaitTime >= 0);
+            if (_currentWaitTime > 0)
+            {
+                if (doSleep)
+                {
+                    Thread.Sleep(_currentWaitTime);
+                }
+                _totalWaitTime += _currentWaitTime;
+            }            
+        }
+
+        /// <summary>
+        /// Resets the retry manager's retry count
+        /// </summary>
+        public void Reset()
+        {
+            _retryCount = 0;
+            _totalWaitTime = 0;
+            _currentWaitTime = 0;
+        }
+    }
+
+
+    /// <summary>
+    /// Defines how long a retry manager will wait between sub-sequent retries
+    /// </summary>
+    internal abstract class RetryWaitTimer
+    {
+        internal abstract int GetNextWaitTime(int retryCount, int currentWaitTime);
+    }
+
+
+    /// <summary>
+    /// Instantly returns without waiting
+    /// </summary>
+    internal class InstantRetryTimer : RetryWaitTimer
+    {
+        internal override int GetNextWaitTime(int retryCount, int currentWaitTime)
+        {
+            return 0;
+        }
+
+        // This class should be a singleton
+        private InstantRetryTimer() { }        
+        
+        static InstantRetryTimer _instance = new InstantRetryTimer();
+        public static InstantRetryTimer Instance 
+        { 
+            get { return _instance; } 
+        }
+    }
+
+    /// <summary>
+    /// Waits a constant time between subsequent retries
+    /// </summary>
+    internal class PeriodicRetryTimer : RetryWaitTimer
+    {
+        int _period;
+
+        public PeriodicRetryTimer(int period)
+        {
+            if (period < 0)
+            {
+                throw new ArgumentOutOfRangeException("period", "The period must be a non-negative integer (in milliseconds)");
+            }
+            _period = period;
+        }
+
+        internal override int GetNextWaitTime(int retryCount, int currentWaitTime)
+        {
+            return _period;
+        }
+    }
+
+    /// <summary>
+    /// A retry timer where wait time at retry n depends on the wait at retry n-1.
+    /// </summary>
+    internal abstract class BoundedBackoffRetryTimer : RetryWaitTimer
+    {
+        int _initialWait;
+        int _waitUpperBound;
+
+        protected BoundedBackoffRetryTimer(int initialWait, int waitUpperBound)
+        {
+            if (initialWait <= 0)
+            {
+                throw new ArgumentOutOfRangeException("initialWait", "Initial value must be a positive integer (in milliseconds)");
+            }
+            if (waitUpperBound <= 0 && waitUpperBound != Timeout.Infinite)
+            {
+                throw new ArgumentOutOfRangeException("waitCap", "The wait cap must be greater than zero, or Timeout.Infinite");
+            }
+
+            _initialWait = initialWait;            
+            _waitUpperBound = waitUpperBound;
+        }
+
+        internal override int GetNextWaitTime(int retryCount, int currentWaitTime)
+        {
+            if (retryCount == 0)
+            {
+                return _initialWait;
+            }
+
+            int nextWaitTime = GetBackOffValue(currentWaitTime);
+            if (nextWaitTime < 0)
+            {
+                return 0;
+            }
+            if (_waitUpperBound != Timeout.Infinite && nextWaitTime > _waitUpperBound)
+            {
+                return _waitUpperBound;
+            }
+            return nextWaitTime;
+        }
+
+        protected abstract int GetBackOffValue(int currentValue);
+    }
+
+
+    /// <summary>
+    /// Wait times will increase exponentially
+    /// </summary>
+    internal class ExponentialBackoffRetryTimer : BoundedBackoffRetryTimer
+    {
+        double _growthFactor;        
+
+        public ExponentialBackoffRetryTimer(int initialWait) : this(initialWait, Timeout.Infinite, 2) { }
+        public ExponentialBackoffRetryTimer(int initialWait, int waitUpperBound) : this(initialWait, waitUpperBound, 2) { }
+
+        public ExponentialBackoffRetryTimer(int initialWait, int waitUpperBound, double growthFactor)
+            : base(initialWait, waitUpperBound)
+        {
+            if (growthFactor <= 0)
+            {
+                throw new ArgumentOutOfRangeException("growthFactor", "The growth factor must be a positive value");
+            }
+            _growthFactor = growthFactor;
+        }
+
+        protected override int GetBackOffValue(int currentValue)
+        {
+ 	        return (int)Math.Round(currentValue * _growthFactor);
+        }
+    }
+
+    /// <summary>
+    /// Wait times will increase exponentially and also vary a bit randomly
+    /// </summary>
+    internal class ExponentialRandomBackoffRetryTimer : ExponentialBackoffRetryTimer
+    {
+        Random _rand = null;
+        public ExponentialRandomBackoffRetryTimer(int initialWait) : this(initialWait, Timeout.Infinite, 2) { }
+        public ExponentialRandomBackoffRetryTimer(int initialWait, int waitUpperBound) : this(initialWait, waitUpperBound, 2) { }
+
+        public ExponentialRandomBackoffRetryTimer(int initialWait, int waitUpperBound, double growthFactor)
+            : base(initialWait, waitUpperBound,growthFactor)
+        {
+            _rand = new Random();
+        }
+
+        protected override int GetBackOffValue(int currentValue)
+        {
+            return ((int)base.GetBackOffValue(currentValue)) + _rand.Next(0, currentValue);
+        }
+    }
+
+
+    /// <summary>
+    /// Wait times will increase linearly
+    /// </summary>
+    internal class LinearBackoffRetryTimer : BoundedBackoffRetryTimer
+    {
+        int _increment;
+
+        public LinearBackoffRetryTimer(int initialWait) : this(initialWait, Timeout.Infinite, initialWait) { }
+        public LinearBackoffRetryTimer(int initialWait, int waitUpperBound) : this(initialWait, waitUpperBound, initialWait) { }
+
+        public LinearBackoffRetryTimer(int initialWait, int waitUpperBound, int increment)
+            : base(initialWait, waitUpperBound)
+        {
+            _increment = increment;
+        }
+
+        protected override int GetBackOffValue(int currentValue)
+        {
+            return currentValue + _increment;
+        }
+    }
+}

CommonCode/SchedulerHelper.cs

@@ -0,0 +1,753 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+namespace Microsoft.Research.Dryad
+{
+    using System;
+    using System.Globalization;
+    using System.ServiceModel;
+    using System.ServiceModel.Channels;
+    using System.Threading;
+    using System.Collections.Generic;
+    using System.Collections.Specialized;
+    using System.Collections.Concurrent;
+    using System.Reflection;
+    using System.Net.Security;
+    using System.Security;
+    using System.Security.AccessControl;
+    using System.Security.Principal;
+    using System.Text;
+    using System.Xml;
+    using System.Xml.Serialization;
+    using Microsoft.Hpc;
+    using Microsoft.Win32;
+    using Microsoft.Research.Dryad.YarnBridge;
+
+    [Flags]
+    public enum VertexTaskState
+    {
+        NA = 0x00000000,
+        Waiting = 0x00000010,
+        Running = 0x00000020,
+        Finished = 0x00000030,
+        Failed = 0x00000031,
+        Canceled = 0x00000032
+    }
+
+    public class VertexChangeEventArgs : EventArgs
+    {
+        public VertexChangeEventArgs(int id)
+        {
+            Id = id;
+            OldState = VertexTaskState.NA;
+            OldNode = String.Empty;
+            OldRequeueCount = 0;
+        }
+
+        public int Id
+        {
+            get;
+            private set;
+        }
+
+        public VertexTaskState OldState
+        {
+            get;
+            set;
+        }
+
+        public VertexTaskState NewState
+        {
+            get;
+            set;
+        }
+
+        public string OldNode
+        {
+            get;
+            set;
+        }
+
+        public string NewNode
+        {
+            get;
+            set;
+        }
+
+        public int OldRequeueCount
+        {
+            get;
+            set;
+        }
+
+        public int NewRequeueCount
+        {
+            get;
+            set;
+        }
+    }
+
+    public class VertexComputeNode
+    {
+        public string ComputeNode;
+        public int instanceId;
+        public VertexTaskState State;
+    }
+
+    public delegate void VertexChangeEventHandler(object sender, VertexChangeEventArgs e);
+
+    public interface ISchedulerHelper : IDisposable
+    {
+        event VertexChangeEventHandler OnVertexChange;
+
+        void FinishJob();
+
+        string GetVertexServiceBaseAddress(string nodename, int instanceId);
+
+        NetTcpBinding GetVertexServiceBinding();
+
+        void SetJobProgress(int n, string message);
+
+        bool StartTaskMonitorThread();
+
+        void StopTaskMonitorThread();
+
+        bool WaitForTasksReady();
+
+    }
+
+    public class SchedulerHelperFactory
+    {
+        private static ISchedulerHelper m_instance = null;
+        private static object m_lock = new object();
+
+        public static ISchedulerHelper GetInstance()
+        {
+            if (m_instance == null)
+            {
+                lock (m_lock)
+                {
+                    if (m_instance == null)
+                    {
+                        string schedulerType = System.Environment.GetEnvironmentVariable(Constants.schedulerTypeEnvVar);
+                        if (String.IsNullOrEmpty(schedulerType) || schedulerType == Constants.schedulerTypeYarn)
+                        {
+                            m_instance = new YarnSchedulerHelper();
+                        }
+                        else if (schedulerType == Constants.schedulerTypeLocal)
+                        {
+                            m_instance = new LocalSchedulerHelper();
+                        }
+                        else
+                        {
+                            throw new InvalidOperationException(String.Format("Scheduler type {0} is not supported", schedulerType));
+                        }
+                    }
+                }
+            }
+            return m_instance;
+       }
+    }
+
+    public class LocalSchedulerHelper : ISchedulerHelper
+    {
+        protected string m_EnvCcpLocalProcessComputeNodes = System.Environment.GetEnvironmentVariable(Constants.localProcessComputeNodesEnvVar);
+        protected string[] m_LocalProcessComputeNodes = new string[0];
+        private bool m_disposed = false;
+
+
+        private VertexChangeEventHandler m_vertexChangeEvent;
+
+        event VertexChangeEventHandler ISchedulerHelper.OnVertexChange
+        {
+            add
+            {
+                lock (m_vertexChangeEvent)
+                {
+                    m_vertexChangeEvent += value;
+                }
+            }
+            remove
+            {
+                lock (m_vertexChangeEvent)
+                {
+                    m_vertexChangeEvent -= value;
+                }
+            }
+        }
+
+        public LocalSchedulerHelper()
+        {
+            if (!String.IsNullOrEmpty(m_EnvCcpLocalProcessComputeNodes))
+            {
+                m_LocalProcessComputeNodes = m_EnvCcpLocalProcessComputeNodes.Split(',');
+            }
+        }
+
+        void IDisposable.Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        private void Dispose(bool disposing)
+        {
+            if (!m_disposed)
+            {
+                if (disposing)
+                {
+                }
+                m_disposed = true;
+            }
+        }
+
+        void ISchedulerHelper.FinishJob()
+        {
+            
+        }
+
+        string ISchedulerHelper.GetVertexServiceBaseAddress(string nodename, int instanceId)
+        {
+            return String.Format(Constants.vertexAddrFormat, "localhost", nodename);
+        }
+
+        NetTcpBinding ISchedulerHelper.GetVertexServiceBinding()
+        {
+            NetTcpBinding binding = new NetTcpBinding(SecurityMode.Transport, false);
+            binding.PortSharingEnabled = true;
+            binding.Security.Transport.ClientCredentialType = TcpClientCredentialType.Windows;
+            binding.Security.Transport.ProtectionLevel = ProtectionLevel.None;
+            binding.SendTimeout = Constants.SendTimeout;
+            binding.ReceiveTimeout = Constants.ReceiveTimeout;
+            binding.MaxReceivedMessageSize = Constants.MaxReceivedMessageSize;
+            binding.MaxBufferPoolSize = Constants.MaxBufferPoolSize;
+            binding.MaxConnections = Constants.MaxConnections;
+            binding.ListenBacklog = Constants.ListenBacklog;
+            binding.ReaderQuotas = System.Xml.XmlDictionaryReaderQuotas.Max;
+
+            return binding;
+
+        }
+
+        void ISchedulerHelper.SetJobProgress(int n, string message)
+        {
+        }
+
+        bool ISchedulerHelper.StartTaskMonitorThread()
+        {
+            return true;
+        }
+
+        void ISchedulerHelper.StopTaskMonitorThread()
+        {
+            return;
+        }
+
+        bool ISchedulerHelper.WaitForTasksReady()
+        {
+            return true;
+        }
+
+    }
+
+    public class YarnSchedulerHelper : ISchedulerHelper
+    {
+        public enum YarnTaskState
+        {
+            NA,
+            Scheduling,
+            Running,
+            Completed,
+            Failed
+        }
+
+        public class VertexTask
+        {
+            public VertexTask(int id, string node, YarnTaskState state, int requeueCount, DateTime changeTime)
+            {
+                Id = id;
+                Node = node;
+                State = state;
+                RequeueCount = requeueCount;
+                ChangeTime = changeTime;
+            }
+
+            public int Id
+            {
+                get;
+                set;
+            }
+
+            public string Node
+            {
+                get;
+                set;
+            }
+
+            public YarnTaskState State
+            {
+                get;
+                set;
+            }
+
+            public int RequeueCount
+            {
+                get;
+                set;
+            }
+
+            public DateTime ChangeTime
+            {
+                get;
+                set;
+            }
+        }
+
+        private VertexTask[] m_vertices = null;
+
+        protected string m_EnvCcpClusterName = System.Environment.GetEnvironmentVariable(Constants.clusterNameEnvVar);
+        protected int m_EnvCcpJobId = Convert.ToInt32(System.Environment.GetEnvironmentVariable(Constants.jobIdEnvVar));
+
+        private object m_eventLock = new object();
+        private VertexChangeEventHandler m_vertexChangeEvent;
+
+        private int m_minNodes = -1;
+        private int m_maxNodes = -1;
+        private int m_startNodes = -1;
+        private int m_runningTasks = 0;  // No longer start at 1 since the GM is not running under a task
+        private int m_finishedTasks = 0;
+        private object m_lock = new object();
+        private bool m_disposed = false;
+        private AutoResetEvent m_taskChangeEvt = new AutoResetEvent(false);
+
+        private bool m_taskMonitorThreadRunning = false;
+        private ManualResetEvent m_threadStopEvt = new ManualResetEvent(false);
+        private Thread m_taskMonitorThread = null;
+        private BlockingCollection<VertexTask> m_taskUpdateQueue;
+
+        public AMInstance m_appMaster;  
+
+        private const int GM_EXITCODE_CANNOT_ACCESS_SCHEDULER = 1000;
+
+        #region Events
+
+        event VertexChangeEventHandler ISchedulerHelper.OnVertexChange
+        {
+            add
+            {
+                lock (m_eventLock)
+                {
+                    m_vertexChangeEvent += value;
+                }
+            }
+            remove
+            {
+                lock (m_eventLock)
+                {
+                    m_vertexChangeEvent -= value;
+                }
+            }
+        }
+
+        private AMInstance GetScheduler()
+        {
+            return m_appMaster;
+        }
+
+        #endregion
+
+        #region Properties
+
+        private int JobMinNodes
+        {
+            get
+            {
+                return m_minNodes;
+            }
+        }
+
+        private int JobMaxNodes
+        {
+            get
+            {
+                return m_maxNodes;
+            }
+        }
+
+        private int JobStartNodes
+        {
+            get
+            {
+                return m_startNodes;
+            }
+        }
+
+        #endregion
+
+        public YarnSchedulerHelper()
+        {
+            // init the DryadLogger, just to make sure
+            DryadLogger.Start("xcompute.log");
+            m_taskUpdateQueue = new BlockingCollection<VertexTask>();
+
+            // if we are not running in a vertex, then init the GM
+            string jmString = Environment.GetEnvironmentVariable(Constants.jobManager);
+            if (String.IsNullOrEmpty(jmString))
+            {
+                m_minNodes = int.Parse(Environment.GetEnvironmentVariable("MINIMUM_COMPUTE_NODES"));
+                m_maxNodes = int.Parse(Environment.GetEnvironmentVariable("MAXIMUM_COMPUTE_NODES"));
+                m_startNodes = m_minNodes;
+                
+                m_vertices = new VertexTask[JobMaxNodes + 2];
+                DryadLogger.LogInformation("YarnSchedulerHelper()", "Initializing JAVA GM");
+                DryadLogger.LogInformation("YarnSchedulerHelper()", "m_maxNodes: {0}", m_maxNodes);
+                AMInstance.RegisterGMCallback(new UpdateProcessState(QueueYarnUpdate));
+                ((ISchedulerHelper)this).OnVertexChange += new VertexChangeEventHandler(OnVertexChangeHandler);
+                m_appMaster = new AMInstance();
+
+            }
+            else
+            {
+                m_vertices = new VertexTask[JobMaxNodes + 2];
+                DryadLogger.LogInformation("YarnSchedulerHelper()", "Not initializing JAVA GM");
+            }
+            
+            
+        }
+
+        #region Methods
+
+        void IDisposable.Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        private void Dispose(bool disposing)
+        {
+            if (!m_disposed)
+            {
+                if (disposing)
+                {
+                    if (m_appMaster != null)
+                    {
+                        m_appMaster.Close();
+                    }
+                    this.m_taskChangeEvt.Close();
+                }
+
+                m_disposed = true;
+            }
+        }
+
+        void ISchedulerHelper.FinishJob()
+        {
+            m_appMaster.Finish();
+        }
+
+        string ISchedulerHelper.GetVertexServiceBaseAddress(string nodename, int instanceId)
+        {
+            return String.Format(Constants.vertexAddrFormat, nodename, instanceId);
+        }
+
+        NetTcpBinding ISchedulerHelper.GetVertexServiceBinding()
+        {
+            NetTcpBinding binding = new NetTcpBinding(SecurityMode.Transport, false);
+            binding.Security.Transport.ClientCredentialType = TcpClientCredentialType.Windows;
+            binding.Security.Transport.ProtectionLevel = ProtectionLevel.None;
+            binding.SendTimeout = Constants.VertexSendTimeout;
+            binding.ReceiveTimeout = Constants.ReceiveTimeout;
+            binding.MaxReceivedMessageSize = Constants.MaxReceivedMessageSize;
+            binding.MaxBufferPoolSize = Constants.MaxBufferPoolSize;
+            binding.MaxConnections = Constants.MaxConnections;
+            binding.ListenBacklog = Constants.ListenBacklog;
+            binding.ReaderQuotas = System.Xml.XmlDictionaryReaderQuotas.Max;
+
+            return binding;
+
+        }
+
+        void ISchedulerHelper.SetJobProgress(int n, string message)
+        {
+            DryadLogger.LogWarning("SetJobProgress", "n: {0} message: {1}", n, message);
+        }
+
+
+
+        bool ISchedulerHelper.StartTaskMonitorThread()
+        {
+            // We only want to have one of these threads running, in case we get called more than once
+            if (m_taskMonitorThreadRunning == false)
+            {
+                lock (m_lock)
+                {
+                    if (m_taskMonitorThreadRunning == false)
+                    {
+                        ((ISchedulerHelper)this).OnVertexChange += new VertexChangeEventHandler(OnVertexChangeHandler);
+                        try
+                        {
+                            m_taskMonitorThread = new Thread(new ThreadStart(TaskMonitorThread));
+                            m_taskMonitorThread.Start();
+                            m_taskMonitorThreadRunning = true;
+                            return true;
+                        }
+                        catch (Exception e)
+                        {
+                            DryadLogger.LogCritical(0, e, "Failed to start task monitoring thread");
+                            return false;
+                        }
+                    }
+                }
+            }
+            return true;
+        }
+
+        void ISchedulerHelper.StopTaskMonitorThread()
+        {
+            DryadLogger.LogMethodEntry();
+            bool wait = false;
+            if (m_taskMonitorThreadRunning)
+            {
+                lock (m_lock)
+                {
+                    if (m_taskMonitorThreadRunning)
+                    {
+                        m_threadStopEvt.Set();
+                        wait = true;
+                    }
+                }
+            }
+
+            if (wait)
+            {
+                try
+                {
+                    m_taskMonitorThread.Join();
+                }
+                catch (Exception e)
+                {
+                    DryadLogger.LogError(0, e, "Failed to wait for task monitor thread to stop.");
+                }
+            }
+            DryadLogger.LogMethodExit();
+        }
+
+        private VertexTaskState YarnTaskStateToVertexTaskState(YarnTaskState ts)
+        {
+            VertexTaskState vts = VertexTaskState.NA;
+            if (ts == YarnTaskState.NA)
+            {
+                vts = VertexTaskState.NA;
+            }
+            else if (ts < YarnTaskState.Running)
+            {
+                vts = VertexTaskState.Waiting;
+            }
+            else if (ts == YarnTaskState.Running)
+            {
+                vts = VertexTaskState.Running;
+            }
+            else
+            {
+                switch (ts)
+                {
+                    case YarnTaskState.Completed:
+                        vts = VertexTaskState.Finished;
+                        break;
+                    case YarnTaskState.Failed:
+                        vts = VertexTaskState.Failed;
+                        break;
+                    //case TaskState.Canceled:
+                    //case TaskState.Canceling:
+                    //    vts = VertexTaskState.Canceled;
+                    //    break;
+                }
+            }
+            DryadLogger.LogDebug("Task State", "Mapped ts: {0} to vts: {1}", ts, vts);
+            return vts;
+        }
+
+        bool ISchedulerHelper.WaitForTasksReady()
+        {
+            // The basic strategy is to wait for the maximum number of vertex tasks which is 
+            // practical. Start by waiting for AllocatedNodes.Count.  As tasks fail or are cancelled,
+            // decrement the number of tasks to wait for until we drop below Min at which time the
+            // scheduler will end the job. Also, if tasks are rerun, increment the number of tasks to wait for.
+            do
+            {
+                // Event set by the Task Monitor Thread when it finishes processes a batch of changes.
+                m_taskChangeEvt.WaitOne();
+
+                // Don't want OnVertexChangeHandler updating these counts while we're checking them
+                lock (this)
+                {
+                    DryadLogger.LogInformation("Wait for vertex tasks",
+                                               "{0} tasks are running, waiting for at least {1} before starting",
+                                               m_runningTasks, m_startNodes);
+                    if (m_runningTasks >= m_startNodes)
+                    {
+                        // We have enough running tasks to start
+                        DryadLogger.LogDebug("Wait for vertex tasks", 
+                            "Sufficient number of tasks transitioned to running to begin: {0} running tasks", 
+                            m_runningTasks);
+                        return true;
+                    }
+                }
+
+            } while (true);
+        }
+
+        public void QueueYarnUpdate(int taskId, int taskState, string nodeName)
+        {
+            DryadLogger.LogInformation("QueueYarnUpdate", "Task {0} on node {2} is in state {3}", taskId, nodeName,
+                                       taskState);
+            // Set change event arguments
+            YarnTaskState yTaskState = (YarnTaskState)taskState;
+            VertexTask v = new VertexTask(taskId, nodeName, yTaskState, int.MaxValue, DateTime.UtcNow);
+            m_taskUpdateQueue.Add(v);
+        }
+
+        public void ProcessYarnUpdate(VertexTask v)
+        {
+            DryadLogger.LogInformation("ProcessYarnUpdate", "Task {0} on node {1} is in state {2}", v.Id, v.Node,
+                                       v.State);
+            VertexChangeEventArgs e = new VertexChangeEventArgs(v.Id);
+
+            e.NewNode = v.Node;
+            e.NewState = YarnTaskStateToVertexTaskState(v.State);
+            e.NewRequeueCount = v.RequeueCount;
+
+            if (m_vertices[v.Id] != null)
+            {
+                e.OldNode = m_vertices[v.Id].Node;
+                e.OldState = YarnTaskStateToVertexTaskState(m_vertices[v.Id].State);
+                e.OldRequeueCount = m_vertices[v.Id].RequeueCount;
+            }
+
+            if (e.NewRequeueCount != e.OldRequeueCount)
+            {
+                DryadLogger.LogInformation("ProcessYarnUpdate", "Task {0} requeue count changed from {1} to {2}",
+                                           v.Id, e.OldRequeueCount, e.NewRequeueCount);
+            }
+
+            // Update current vertex state
+            m_vertices[v.Id] = v;
+            m_vertexChangeEvent(this, e);
+            //m_taskChangeEvt.Set();
+        }
+
+         private void TaskMonitorThread()
+        {
+            TimeSpan pollInterval = TimeSpan.FromSeconds(1);
+            TimeSpan maxPollInterval = TimeSpan.FromSeconds(16);
+
+            // The main loop.  Each iteration polls for task changes.
+            while (true)
+            {
+                bool foundUpdate = false;
+                DateTime loopStartTime = DateTime.Now;
+                //
+                // Process change results from blocking queue
+                //
+                do
+                {
+                    VertexTask v = null;
+                    if (m_taskUpdateQueue.TryTake(out v, pollInterval))
+                    {
+                        foundUpdate = true;
+                        ProcessYarnUpdate(v);
+                    }
+
+                } while ((DateTime.Now - loopStartTime) < pollInterval);
+
+                if (foundUpdate)
+                {
+                    // Notify WaitForTasksReady once for each polling cycle
+                    // so that it gets all the changes in one batch
+                    m_taskChangeEvt.Set();
+                }
+
+                // Check to see if we've been told to stop.
+                // Timeout after pollInterval.
+                // TODO: For better shutdown perf, we may want to check this at other places
+                // or just kill the thread - but this provides a more graceful exit.
+                if (m_threadStopEvt.WaitOne(pollInterval, true))
+                {
+                    m_taskMonitorThreadRunning = false;
+                    DryadLogger.LogInformation("Task Monitoring Thread", "Received shutdown event");
+                    return;
+                }
+
+                // Double the polling interval each iteration up to maxPollInterval
+                if (pollInterval < maxPollInterval)
+                {
+                    double newSeconds = 2 * pollInterval.TotalSeconds;
+                    if (newSeconds < maxPollInterval.TotalSeconds)
+                    {
+                        pollInterval = TimeSpan.FromSeconds(newSeconds);
+                    }
+                    else
+                    {
+                        pollInterval = maxPollInterval;
+                    }
+                }
+                
+            }
+        }
+
+        private void OnVertexChangeHandler(object sender, VertexChangeEventArgs ve)
+        {
+            if (ve.OldState != ve.NewState)
+            {
+                // Don't want to update counts while WaitForTasksReady is checking them
+                lock (this)
+                {
+                    if (ve.OldState == VertexTaskState.Running)
+                    {
+                        m_runningTasks--;
+                    }
+                    else if (ve.OldState > VertexTaskState.Running)
+                    {
+                        m_finishedTasks--;
+                        // Task transitioning from a completed state so we can increment
+                        // the number of tasks to wait for at startup
+                        m_startNodes++;
+                    }
+
+                    if (ve.NewState == VertexTaskState.Running)
+                    {
+                        m_runningTasks++;
+                    }
+                    else if (ve.NewState > VertexTaskState.Running)
+                    {
+                        m_finishedTasks++;
+                        // Task transitioning to a completed state so we need to
+                        // decrement the number of tasks to wait for at startup.
+                        m_startNodes--;
+                    }
+                }
+            }
+        }
+
+        #endregion
+    }
+
+
+}

CommonCode/SoftAffinity.cs

@@ -0,0 +1,79 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+namespace Microsoft.Research.Dryad
+{
+    using System;
+    using System.ServiceModel.Channels;
+    using System.Runtime.Serialization;
+
+    [DataContract]
+    public class SoftAffinity : IComparable
+    {
+        [DataMember]
+        private string node = null;
+        [DataMember]
+        private ulong weight = ulong.MinValue;
+
+        public SoftAffinity(string n, ulong w)
+        {
+            node = n;
+            weight = w;
+        }
+
+        // Compare SoftAffinities using their weights
+        int IComparable.CompareTo(object obj)
+        {
+            SoftAffinity other = obj as SoftAffinity;
+
+            if (other == null)
+            {
+                return -1;
+            }
+
+            if (this.Weight < other.Weight)
+            {
+                return -1;
+            }
+            else if (this.Weight > other.Weight)
+            {
+                return 1;
+            }
+            else
+            {
+                return 0;
+            }
+        }
+
+        [DataMember]
+        public string Node
+        {
+            get { return node; }
+            set { node = value; }
+        }
+
+        [DataMember]
+        public ulong Weight
+        {
+            get { return weight; }
+            set { weight = value; }
+        }
+    }
+}

Dryad.sln

@@ -0,0 +1,359 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2012
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DryadVertex", "DryadVertex", "{99F5E7FE-ADD4-4E9D-BA90-5D3D3409BEB3}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "VertexHost", "VertexHost", "{37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "System", "System", "{D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Vertex", "Vertex", "{DC277807-506B-4F7C-BECD-345079B91044}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CommonCode", "CommonCode", "{387FAB85-B968-48F6-A2F7-79A1F2CF2F54}"
+	ProjectSection(SolutionItems) = preProject
+		CommonCode\AzureUtils.cs = CommonCode\AzureUtils.cs
+		CommonCode\Constants.cs = CommonCode\Constants.cs
+		CommonCode\DiscLocalMonitor.cs = CommonCode\DiscLocalMonitor.cs
+		CommonCode\DryadTracing.cs = CommonCode\DryadTracing.cs
+		CommonCode\DryadVertexServiceAuthorizationManager.cs = CommonCode\DryadVertexServiceAuthorizationManager.cs
+		CommonCode\ExecutionHelper.cs = CommonCode\ExecutionHelper.cs
+		CommonCode\IDryadVertexCallback.cs = CommonCode\IDryadVertexCallback.cs
+		CommonCode\IDryadVertexService.cs = CommonCode\IDryadVertexService.cs
+		CommonCode\NativeMethods.cs = CommonCode\NativeMethods.cs
+		CommonCode\NetShareWrapper.cs = CommonCode\NetShareWrapper.cs
+		CommonCode\ProcessPathHelper.cs = CommonCode\ProcessPathHelper.cs
+		CommonCode\ProcessState.cs = CommonCode\ProcessState.cs
+		CommonCode\QueryUtility.cs = CommonCode\QueryUtility.cs
+		CommonCode\RetryFramework.cs = CommonCode\RetryFramework.cs
+		CommonCode\SchedulerHelper.cs = CommonCode\SchedulerHelper.cs
+		CommonCode\SoftAffinity.cs = CommonCode\SoftAffinity.cs
+	EndProjectSection
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GraphManager", "GraphManager\GraphManager.vcxproj", "{8E30F4A4-603B-4799-A473-6EF5388661BA}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "DryadYarnBridge", "DryadYarnBridge\DryadYarnBridge.vcxproj", "{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "linqtodryadjm_managed", "linqtodryadjm_managed_yarn\linqtodryadjm_managed.csproj", "{1311809B-306E-44A4-9D69-8A7BD15123C5}"
+	ProjectSection(ProjectDependencies) = postProject
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2} = {09FB27C7-D1A5-4A59-B010-67D5886DD9A2}
+	EndProjectSection
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Research.Dryad.ClusterAdapter", "xcompute_managed\Microsoft.Research.Dryad.ClusterAdapter.csproj", "{F4B04940-67CF-4796-B6D3-3CFD38FB988A}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "YarnQueryNativeClusterAdapter", "xcompute_native\YarnQueryNativeClusterAdapter.vcxproj", "{E092E2B9-D3C9-4CE2-8201-BDA442574C97}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DryadVertexService", "DryadVertex\service\DryadVertexService.csproj", "{27D89037-8934-45BE-8A44-2561F9330EB7}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "channel", "DryadVertex\VertexHost\system\channel\channel.vcxproj", "{482E0741-E244-4974-97D4-3A7167581E91}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "classlib", "DryadVertex\VertexHost\system\classlib\classlib.vcxproj", "{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "common", "DryadVertex\VertexHost\system\common\common.vcxproj", "{57663B94-E11B-431E-BE4B-E2C61112DEC5}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dprocess", "DryadVertex\VertexHost\system\dprocess\dprocess.vcxproj", "{AA529122-F51C-48D7-A8C1-C0B24F570885}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ManagedWrapperVertex", "DryadVertex\VertexHost\vertex\managedwrappervertex\ManagedWrapperVertex.vcxproj", "{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "VertexHost", "DryadVertex\VertexHost\vertex\vertexHost\VertexHost.vcxproj", "{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "WrapperNativeInfo", "DryadVertex\VertexHost\vertex\WrapperNativeInfo\WrapperNativeInfo.vcxproj", "{AB9EA66C-5811-49A7-B002-24203AEB9083}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "WrapperNativeInfoDll", "DryadVertex\VertexHost\vertex\WrapperNativeInfoDll\WrapperNativeInfoDll.vcxproj", "{3EE0920C-0607-4569-9EC3-5C12BB6EF244}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LinqToDryad", "LinqToDryad\LinqToDryad.csproj", "{D33C34CC-6DB2-417C-88B7-299830711774}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Hdfs", "Hdfs", "{6003A98A-82CB-4385-8B29-70BC28F19076}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "HdfsBridgeManaged", "Hdfs\HdfsBridgeManaged\HdfsBridgeManaged.vcxproj", "{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "HdfsBridgeNative", "Hdfs\HdfsBridgeNative\HdfsBridgeNative.vcxproj", "{95FBF9B7-9407-4554-A74A-3527839BD1B6}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "include", "include", "{89C5654B-02E4-478D-A7E6-50D79F638B4F}"
+	ProjectSection(SolutionItems) = preProject
+		DryadVertex\VertexHost\vertex\include\ChannelTransform.h = DryadVertex\VertexHost\vertex\include\ChannelTransform.h
+		DryadVertex\VertexHost\vertex\include\CompressionVertex.h = DryadVertex\VertexHost\vertex\include\CompressionVertex.h
+		DryadVertex\VertexHost\vertex\include\DataBlockItem.h = DryadVertex\VertexHost\vertex\include\DataBlockItem.h
+		DryadVertex\VertexHost\vertex\include\FifoChannel.h = DryadVertex\VertexHost\vertex\include\FifoChannel.h
+		DryadVertex\VertexHost\vertex\include\FifoInputChannel.h = DryadVertex\VertexHost\vertex\include\FifoInputChannel.h
+		DryadVertex\VertexHost\vertex\include\FifoOutputChannel.h = DryadVertex\VertexHost\vertex\include\FifoOutputChannel.h
+		DryadVertex\VertexHost\vertex\include\GzipCompressionChannelTransform.h = DryadVertex\VertexHost\vertex\include\GzipCompressionChannelTransform.h
+		DryadVertex\VertexHost\vertex\include\GzipDecompressionChannelTransform.h = DryadVertex\VertexHost\vertex\include\GzipDecompressionChannelTransform.h
+		DryadVertex\VertexHost\vertex\include\InputChannel.h = DryadVertex\VertexHost\vertex\include\InputChannel.h
+		DryadVertex\VertexHost\vertex\include\ManagedWrapper.h = DryadVertex\VertexHost\vertex\include\ManagedWrapper.h
+		DryadVertex\VertexHost\vertex\include\NullChannelTransform.h = DryadVertex\VertexHost\vertex\include\NullChannelTransform.h
+		DryadVertex\VertexHost\vertex\include\OutputChannel.h = DryadVertex\VertexHost\vertex\include\OutputChannel.h
+		DryadVertex\VertexHost\vertex\include\wrappernativeinfo.h = DryadVertex\VertexHost\vertex\include\wrappernativeinfo.h
+	EndProjectSection
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|Mixed Platforms = Debug|Mixed Platforms
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release|Any CPU = Release|Any CPU
+		Release|Mixed Platforms = Release|Mixed Platforms
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Debug|Win32.ActiveCfg = Debug|Win32
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Debug|Win32.Build.0 = Debug|Win32
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Debug|x64.ActiveCfg = Debug|x64
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Debug|x64.Build.0 = Debug|x64
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Release|Any CPU.ActiveCfg = Release|x64
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Release|Mixed Platforms.ActiveCfg = Release|x64
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Release|Mixed Platforms.Build.0 = Release|x64
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Release|Win32.ActiveCfg = Release|Win32
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Release|Win32.Build.0 = Release|Win32
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Release|x64.ActiveCfg = Release|x64
+		{8E30F4A4-603B-4799-A473-6EF5388661BA}.Release|x64.Build.0 = Release|x64
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Debug|Win32.ActiveCfg = Debug|Win32
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Debug|Win32.Build.0 = Debug|Win32
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Debug|x64.ActiveCfg = Debug|x64
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Debug|x64.Build.0 = Debug|x64
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Release|Any CPU.ActiveCfg = Release|x64
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Release|Mixed Platforms.ActiveCfg = Release|x64
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Release|Mixed Platforms.Build.0 = Release|x64
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Release|Win32.ActiveCfg = Release|Win32
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Release|Win32.Build.0 = Release|Win32
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Release|x64.ActiveCfg = Release|x64
+		{09FB27C7-D1A5-4A59-B010-67D5886DD9A2}.Release|x64.Build.0 = Release|x64
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Debug|Win32.ActiveCfg = Debug|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Debug|x64.Build.0 = Debug|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Release|Win32.ActiveCfg = Release|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Release|x64.ActiveCfg = Release|Any CPU
+		{1311809B-306E-44A4-9D69-8A7BD15123C5}.Release|x64.Build.0 = Release|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Debug|Win32.ActiveCfg = Debug|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Debug|x64.Build.0 = Debug|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Release|Win32.ActiveCfg = Release|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Release|x64.ActiveCfg = Release|Any CPU
+		{F4B04940-67CF-4796-B6D3-3CFD38FB988A}.Release|x64.Build.0 = Release|Any CPU
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Debug|Win32.ActiveCfg = Debug|Win32
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Debug|Win32.Build.0 = Debug|Win32
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Debug|x64.ActiveCfg = Debug|x64
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Debug|x64.Build.0 = Debug|x64
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Release|Any CPU.ActiveCfg = Release|Win32
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Release|Win32.ActiveCfg = Release|Win32
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Release|Win32.Build.0 = Release|Win32
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Release|x64.ActiveCfg = Release|x64
+		{E092E2B9-D3C9-4CE2-8201-BDA442574C97}.Release|x64.Build.0 = Release|x64
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Debug|Win32.ActiveCfg = Debug|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Debug|x64.Build.0 = Debug|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Release|Win32.ActiveCfg = Release|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Release|x64.ActiveCfg = Release|Any CPU
+		{27D89037-8934-45BE-8A44-2561F9330EB7}.Release|x64.Build.0 = Release|Any CPU
+		{482E0741-E244-4974-97D4-3A7167581E91}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{482E0741-E244-4974-97D4-3A7167581E91}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{482E0741-E244-4974-97D4-3A7167581E91}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{482E0741-E244-4974-97D4-3A7167581E91}.Debug|Win32.ActiveCfg = Debug|Win32
+		{482E0741-E244-4974-97D4-3A7167581E91}.Debug|Win32.Build.0 = Debug|Win32
+		{482E0741-E244-4974-97D4-3A7167581E91}.Debug|x64.ActiveCfg = Debug|x64
+		{482E0741-E244-4974-97D4-3A7167581E91}.Debug|x64.Build.0 = Debug|x64
+		{482E0741-E244-4974-97D4-3A7167581E91}.Release|Any CPU.ActiveCfg = Release|Win32
+		{482E0741-E244-4974-97D4-3A7167581E91}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{482E0741-E244-4974-97D4-3A7167581E91}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{482E0741-E244-4974-97D4-3A7167581E91}.Release|Win32.ActiveCfg = Release|Win32
+		{482E0741-E244-4974-97D4-3A7167581E91}.Release|Win32.Build.0 = Release|Win32
+		{482E0741-E244-4974-97D4-3A7167581E91}.Release|x64.ActiveCfg = Release|x64
+		{482E0741-E244-4974-97D4-3A7167581E91}.Release|x64.Build.0 = Release|x64
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Debug|Win32.ActiveCfg = Debug|Win32
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Debug|Win32.Build.0 = Debug|Win32
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Debug|x64.ActiveCfg = Debug|x64
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Debug|x64.Build.0 = Debug|x64
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Release|Any CPU.ActiveCfg = Release|Win32
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Release|Win32.ActiveCfg = Release|Win32
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Release|Win32.Build.0 = Release|Win32
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Release|x64.ActiveCfg = Release|x64
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}.Release|x64.Build.0 = Release|x64
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Debug|Win32.ActiveCfg = Debug|Win32
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Debug|Win32.Build.0 = Debug|Win32
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Debug|x64.ActiveCfg = Debug|x64
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Debug|x64.Build.0 = Debug|x64
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Release|Any CPU.ActiveCfg = Release|Win32
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Release|Win32.ActiveCfg = Release|Win32
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Release|Win32.Build.0 = Release|Win32
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Release|x64.ActiveCfg = Release|x64
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5}.Release|x64.Build.0 = Release|x64
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Debug|Win32.ActiveCfg = Debug|Win32
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Debug|Win32.Build.0 = Debug|Win32
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Debug|x64.ActiveCfg = Debug|x64
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Debug|x64.Build.0 = Debug|x64
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Release|Any CPU.ActiveCfg = Release|Win32
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Release|Win32.ActiveCfg = Release|Win32
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Release|Win32.Build.0 = Release|Win32
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Release|x64.ActiveCfg = Release|x64
+		{AA529122-F51C-48D7-A8C1-C0B24F570885}.Release|x64.Build.0 = Release|x64
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Debug|Win32.ActiveCfg = Debug|Win32
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Debug|Win32.Build.0 = Debug|Win32
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Debug|x64.ActiveCfg = Debug|x64
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Debug|x64.Build.0 = Debug|x64
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Release|Any CPU.ActiveCfg = Release|Win32
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Release|Win32.ActiveCfg = Release|Win32
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Release|Win32.Build.0 = Release|Win32
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Release|x64.ActiveCfg = Release|x64
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C}.Release|x64.Build.0 = Release|x64
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Debug|Win32.ActiveCfg = Debug|Win32
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Debug|Win32.Build.0 = Debug|Win32
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Debug|x64.ActiveCfg = Debug|x64
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Debug|x64.Build.0 = Debug|x64
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Release|Any CPU.ActiveCfg = Release|Win32
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Release|Win32.ActiveCfg = Release|Win32
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Release|Win32.Build.0 = Release|Win32
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Release|x64.ActiveCfg = Release|x64
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37}.Release|x64.Build.0 = Release|x64
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Debug|Win32.ActiveCfg = Debug|Win32
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Debug|Win32.Build.0 = Debug|Win32
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Debug|x64.ActiveCfg = Debug|x64
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Debug|x64.Build.0 = Debug|x64
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Release|Any CPU.ActiveCfg = Release|Win32
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Release|Win32.ActiveCfg = Release|Win32
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Release|Win32.Build.0 = Release|Win32
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Release|x64.ActiveCfg = Release|x64
+		{AB9EA66C-5811-49A7-B002-24203AEB9083}.Release|x64.Build.0 = Release|x64
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Debug|Any CPU.ActiveCfg = Debug|Win32
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Debug|Win32.ActiveCfg = Debug|Win32
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Debug|Win32.Build.0 = Debug|Win32
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Debug|x64.ActiveCfg = Debug|x64
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Debug|x64.Build.0 = Debug|x64
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Release|Any CPU.ActiveCfg = Release|Win32
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Release|Mixed Platforms.ActiveCfg = Release|Win32
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Release|Mixed Platforms.Build.0 = Release|Win32
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Release|Win32.ActiveCfg = Release|Win32
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Release|Win32.Build.0 = Release|Win32
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Release|x64.ActiveCfg = Release|x64
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244}.Release|x64.Build.0 = Release|x64
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Debug|Win32.ActiveCfg = Debug|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Debug|x64.Build.0 = Debug|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Release|Win32.ActiveCfg = Release|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Release|x64.ActiveCfg = Release|Any CPU
+		{D33C34CC-6DB2-417C-88B7-299830711774}.Release|x64.Build.0 = Release|Any CPU
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Debug|Win32.ActiveCfg = Debug|Win32
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Debug|Win32.Build.0 = Debug|Win32
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Debug|x64.ActiveCfg = Debug|x64
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Debug|x64.Build.0 = Debug|x64
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Release|Any CPU.ActiveCfg = Release|x64
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Release|Mixed Platforms.ActiveCfg = Release|x64
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Release|Mixed Platforms.Build.0 = Release|x64
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Release|Win32.ActiveCfg = Release|Win32
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Release|Win32.Build.0 = Release|Win32
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Release|x64.ActiveCfg = Release|x64
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5}.Release|x64.Build.0 = Release|x64
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Debug|Mixed Platforms.Build.0 = Debug|x64
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Debug|Win32.ActiveCfg = Debug|Win32
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Debug|Win32.Build.0 = Debug|Win32
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Debug|x64.ActiveCfg = Debug|x64
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Debug|x64.Build.0 = Debug|x64
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Release|Any CPU.ActiveCfg = Release|x64
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Release|Mixed Platforms.ActiveCfg = Release|x64
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Release|Mixed Platforms.Build.0 = Release|x64
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Release|Win32.ActiveCfg = Release|Win32
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Release|Win32.Build.0 = Release|Win32
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Release|x64.ActiveCfg = Release|x64
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6}.Release|x64.Build.0 = Release|x64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D} = {99F5E7FE-ADD4-4E9D-BA90-5D3D3409BEB3}
+		{27D89037-8934-45BE-8A44-2561F9330EB7} = {99F5E7FE-ADD4-4E9D-BA90-5D3D3409BEB3}
+		{D8B4F38E-2BF7-44A0-BDBF-025B46501DDE} = {37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D}
+		{DC277807-506B-4F7C-BECD-345079B91044} = {37D9C01A-94F3-4B9E-9AA6-C4D3A2B5AD0D}
+		{482E0741-E244-4974-97D4-3A7167581E91} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
+		{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
+		{57663B94-E11B-431E-BE4B-E2C61112DEC5} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
+		{AA529122-F51C-48D7-A8C1-C0B24F570885} = {D8B4F38E-2BF7-44A0-BDBF-025B46501DDE}
+		{BDEDD3BB-C7E2-498F-A212-F99786C8E23C} = {DC277807-506B-4F7C-BECD-345079B91044}
+		{0CF3D1D5-9BBE-4175-979B-EC6138EF4F37} = {DC277807-506B-4F7C-BECD-345079B91044}
+		{AB9EA66C-5811-49A7-B002-24203AEB9083} = {DC277807-506B-4F7C-BECD-345079B91044}
+		{3EE0920C-0607-4569-9EC3-5C12BB6EF244} = {DC277807-506B-4F7C-BECD-345079B91044}
+		{89C5654B-02E4-478D-A7E6-50D79F638B4F} = {DC277807-506B-4F7C-BECD-345079B91044}
+		{C0F4C1E3-1F9E-4C55-BD6A-0241D35425F5} = {6003A98A-82CB-4385-8B29-70BC28F19076}
+		{95FBF9B7-9407-4554-A74A-3527839BD1B6} = {6003A98A-82CB-4385-8B29-70BC28F19076}
+	EndGlobalSection
+EndGlobal

DryadVertex/VertexHost/system/channel/channel.vcxproj

@@ -0,0 +1,186 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{482E0741-E244-4974-97D4-3A7167581E91}</ProjectGuid>
+    <RootNamespace>channel</RootNamespace>
+    <Keyword>Win32Proj</Keyword>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+    <CLRSupport>true</CLRSupport>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+    <CLRSupport>true</CLRSupport>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <_ProjectFileVersion>10.0.40219.1</_ProjectFileVersion>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Debug\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Debug\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(Platform)\$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(Platform)\$(Configuration)\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Release\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Release\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(Platform)\$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(Platform)\$(Configuration)\</IntDir>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\Hdfs\HdfsBridgeNative;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;WIN32_LEAN_AND_MEAN;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>false</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+    <Lib>
+      <AdditionalDependencies>hdfsbridgenative.lib</AdditionalDependencies>
+      <AdditionalLibraryDirectories>..\..\..\..\bin\$(Configuration)</AdditionalLibraryDirectories>
+    </Lib>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;WIN32_LEAN_AND_MEAN;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+      <AdditionalIncludeDirectories>include;..\common\include;..\classlib\include;src;..\..\..\..\Hdfs\HdfsBridge;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClInclude Include="include\channelbuffer.h" />
+    <ClInclude Include="src\channelbufferhdfs.h" />
+    <ClInclude Include="src\channelbuffernativereader.h" />
+    <ClInclude Include="src\channelbuffernativewriter.h" />
+    <ClInclude Include="src\channelbufferqueue.h" />
+    <ClInclude Include="src\channelfifo.h" />
+    <ClInclude Include="src\channelhelpers.h" />
+    <ClInclude Include="include\channelinterface.h" />
+    <ClInclude Include="include\channelitem.h" />
+    <ClInclude Include="include\channelmarshaler.h" />
+    <ClInclude Include="include\channelmemorybuffers.h" />
+    <ClInclude Include="include\channelparser.h" />
+    <ClInclude Include="src\channelreader.h" />
+    <ClInclude Include="src\channelwriter.h" />
+    <ClInclude Include="include\concreterchannel.h" />
+    <ClInclude Include="src\concreterchannelhelpers.h" />
+    <ClInclude Include="include\recordarray.h" />
+    <ClInclude Include="include\recorditem.h" />
+    <ClInclude Include="include\recordparser.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="src\channelbuffer.cpp" />
+    <ClCompile Include="src\channelbufferhdfs.cpp" />
+    <ClCompile Include="src\channelbuffernativereader.cpp" />
+    <ClCompile Include="src\channelbuffernativewriter.cpp" />
+    <ClCompile Include="src\channelbufferqueue.cpp" />
+    <ClCompile Include="src\channelfifo.cpp" />
+    <ClCompile Include="src\channelhelpers.cpp" />
+    <ClCompile Include="src\channelitem.cpp" />
+    <ClCompile Include="src\channelmarshaler.cpp" />
+    <ClCompile Include="src\channelparser.cpp" />
+    <ClCompile Include="src\channelreader.cpp" />
+    <ClCompile Include="src\channelwriter.cpp" />
+    <ClCompile Include="src\concreterchannel.cpp" />
+    <ClCompile Include="src\memorybuffers.cpp" />
+    <ClCompile Include="src\recorditem.cpp" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\..\xcompute_native\YarnQueryNativeClusterAdapter.vcxproj">
+      <Project>{e092e2b9-d3c9-4ce2-8201-bda442574c97}</Project>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

DryadVertex/VertexHost/system/channel/include/channelbuffer.h

@@ -0,0 +1,275 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include <DrCommon.h>
+#include <channelitem.h>
+#include <channelmemorybuffers.h>
+#include <dvertexcommand.h>
+
+class RChannelBufferReader;
+
+#define RCHANNEL_BUFFER_OFFSET_UNDEFINED ((UInt64) -1)
+
+/* this is a dummy class which may be extended in future to allow item
+   parsers to communicate with the buffer reader to influence i/o
+   buffer sizes, prefetching strategies, etc. Nobody uses it yet and
+   NULL should be passed wherever there is a prefetchCookie argument
+   for now. */
+class RChannelBufferPrefetchInfo
+{
+public:
+    virtual ~RChannelBufferPrefetchInfo();
+};
+
+
+/* buffers read from a channel are typed with one of the following: */
+enum RChannelBufferType {
+    /* RChannelBuffer_Data: this is a data buffer which can be cast to
+       an RChannelBufferData object to retrieve the payload. */
+    RChannelBuffer_Data,
+
+    /* the following are all marker buffers which can be cast to an
+       RChannelBufferMarker object to retrieve the item describing the
+       marker. */
+
+    /* RChannelBuffer_Hole: this describes a "hole" in the
+         underlying data stream. */
+    RChannelBuffer_Hole,
+
+    /* RChannelBuffer_EndOfStream: this is a marker indicating that
+       the end of the stream has been succesfully reached. */
+    RChannelBuffer_EndOfStream,
+
+    /* RChannelBuffer_Restart: this is a marker indicating that the
+       remote end has requested a restart. */
+    RChannelBuffer_Restart,
+
+    /* RChannelBuffer_Abort: this is a marker indicating that the
+       underlying stream has suffered an unrecoverable error, restart
+       is impossible, and no more data is forthcoming. */
+    RChannelBuffer_Abort
+};
+
+/* base class for data buffers generated by the byte-oriented read
+   layer of a channel */
+class RChannelBuffer : public DrRefCounter
+{
+public:
+    static bool IsTerminationBuffer(RChannelBufferType type);
+
+    /* When a consumer has finished using the data in a buffer it
+       should call this completion handler returning the
+       buffer.
+
+       prefetchCookie is an optional hint which may be used to
+       influence subsequent read buffer sizes or prefetching
+       behaviour. It is dependent on the implementation of the
+       underlying buffer-oriented i/o class and should be NULL for
+       now.
+
+       The completion callback mechanism is used to implement flow
+       control, as the buffer-oriented i/o will only allow the
+       consumer to hold a bounded number of outstanding buffers before
+       blocking further reads.
+    */
+    virtual void
+        ProcessingComplete(RChannelBufferPrefetchInfo* prefetchCookie) = 0;
+
+    /* get the buffer's type.
+     */
+    RChannelBufferType GetType();
+
+    /* this returns a description of the buffer, used for debugging
+       and monitoring purposes.
+    */
+    DryadMetaData* GetMetaData();
+
+    /* this replaces the item's current metadata with a new object */
+    void ReplaceMetaData(DryadMetaData* metaData);
+
+protected:
+    RChannelBuffer(RChannelBufferType type);
+    virtual ~RChannelBuffer();
+
+private:
+    RChannelBufferType    m_type;
+    DryadMetaDataRef      m_metaData;
+    DrBListEntry          m_listPtr;
+    friend class DryadBList<RChannelBuffer>;
+};
+
+typedef class DryadBList<RChannelBuffer> ChannelBufferList;
+
+/* this is a default completion handler used to return buffers of type
+   RChannelBufferDataDefault and RChannelBufferMarkerDefault. It is
+   called from the ProcessingComplete methods of each of those
+   classes. */
+class RChannelBufferDefaultHandler
+{
+public:
+    /* the callee owns a reference to buffer after this call */
+    virtual void ReturnBuffer(RChannelBuffer* buffer) = 0;
+};
+
+class ChannelDataBufferList;
+
+class RChannelBufferData : public RChannelBuffer
+{
+public:
+    /* The payload of a data buffer is a DryadLockedMemoryBuffer.
+       This cannot be grown, nor can its available size be
+       modified. It is derived from a DryadFixedMemoryBuffer and hence
+       it is guaranteed to wrap a single contiguous memory region. It
+       has the same thread-safety properties as the underlying memory
+       region, e.g. it can be read concurrently from multiple threads,
+       and if the threads co-operate they can safely concurrently
+       write to non-overlapping regions.
+
+       The caller must increment the reference count of the returned
+       DryadLockedMemoryBuffer if it wishes to use it after the
+       RChannelBufferData goes out of scope.
+    */
+    virtual DryadLockedMemoryBuffer* GetData() = 0;
+
+    /* this returns a metadata description of an offset in the buffer
+       (which may include a description of the buffer, along with
+       e.g. the offset's overall position in the stream), used for
+       debugging and monitoring purposes. The caller owns a reference
+       to the returned metadata. If isStart is true the returned
+       metadata writes the offset using the ItemStreamStartOffset and
+       ItemBufferStartOffset elements, otherwise it uses the
+       ItemStreamEndOffset and ItemBufferEndOffset elements.
+     */
+    virtual void GetOffsetMetaData(bool isStart,
+                                   UInt64 offset,
+                                   DryadMetaDataRef* dstMetaData) = 0;
+
+protected:
+    RChannelBufferData();
+    virtual ~RChannelBufferData();
+    DrBListEntry   m_dataListPtr;
+    friend class ChannelDataBufferList;
+};
+
+class ChannelDataBufferList : public DrBList
+{
+public:
+    static RChannelBufferData* CastOut(DrBListEntry* item);
+    static DrBListEntry* CastIn(RChannelBufferData* item);
+};
+
+class RChannelBufferDataDefault : public RChannelBufferData
+{
+public:
+    static RChannelBufferDataDefault*
+        Create(DryadLockedMemoryBuffer* dataBuffer,
+               UInt64 startOffset,
+               RChannelBufferDefaultHandler* parent);
+
+    /* The payload of a data buffer is a DryadLockedMemoryBuffer.
+       This cannot be grown, nor can its available size be
+       modified. It is derived from a DryadFixedMemoryBuffer and hence
+       it is guaranteed to wrap a single contiguous memory region. It
+       has the same thread-safety properties as the underlying memory
+       region, e.g. it can be read concurrently from multiple threads,
+       and if the threads co-operate they can safely concurrently
+       write to non-overlapping regions.
+
+       The caller must increment the reference count of the returned
+       DryadLockedMemoryBuffer if it wishes to use it after the
+       RChannelBufferData goes out of scope.
+    */
+    DryadLockedMemoryBuffer* GetData();
+
+    /* this returns a metadata description of an offset in the buffer
+       (which may include a description of the buffer, along with
+       e.g. the offset's overall position in the stream), used for
+       debugging and monitoring purposes. The caller owns a reference
+       to the returned metadata. If isStart is true the returned
+       metadata writes the offset using the ItemStreamStartOffset and
+       ItemBufferStartOffset elements, otherwise it uses the
+       ItemStreamEndOffset and ItemBufferEndOffset elements.
+     */
+    void GetOffsetMetaData(bool isStart, UInt64 offset,
+                           DryadMetaDataRef* dstMetaData);
+
+    void ProcessingComplete(RChannelBufferPrefetchInfo* prefetchCookie);
+
+protected:
+    RChannelBufferDataDefault(DryadLockedMemoryBuffer* dataBuffer,
+                              UInt64 startOffset,
+                              RChannelBufferDefaultHandler* parent);
+    virtual ~RChannelBufferDataDefault();
+
+private:
+    DryadLockedMemoryBuffer*        m_dataBuffer;
+    UInt64                          m_startOffset;
+    RChannelBufferDefaultHandler*   m_parent;
+};
+
+class RChannelBufferMarker : public RChannelBuffer
+{
+public:
+    /* the caller's referenece to item is transferred to the newly
+       created buffer */
+    static RChannelBufferMarker*
+        Create(RChannelBufferType type,
+               RChannelItem* item);
+
+    /* Get an item describing the marker event signified by this
+       buffer, which can be passed to the application. The reference
+       count of the item is increased before it is returned, so the
+       caller must decrease it before letting the item go out of
+       scope.
+     */
+    RChannelItem* GetItem();
+
+protected:
+    RChannelBufferMarker(RChannelBufferType type,
+                         RChannelItem* item);
+    virtual ~RChannelBufferMarker();
+
+private:
+    RChannelItem*                   m_item;
+};
+
+class RChannelBufferMarkerDefault : public RChannelBufferMarker
+{
+public:
+    /* the caller's referenece to item is transferred to the newly
+       created buffer */
+    static RChannelBufferMarkerDefault*
+        Create(RChannelBufferType type,
+               RChannelItem* item,
+               RChannelBufferDefaultHandler* parent);
+
+    void ProcessingComplete(RChannelBufferPrefetchInfo* prefetchCookie);
+
+protected:
+    RChannelBufferMarkerDefault(RChannelBufferType type,
+                                RChannelItem* item,
+                                RChannelBufferDefaultHandler* parent);
+    ~RChannelBufferMarkerDefault();
+
+private:
+    RChannelBufferDefaultHandler*   m_parent;
+};

DryadVertex/VertexHost/system/channel/include/channelinterface.h

@@ -0,0 +1,731 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include <channelitem.h>
+
+class RChannelBufferPrefetchInfo;
+enum TransformType;
+
+/* this is a stub object that is e.g. passed to parsers and marshalers
+   when they are initialized. For example, DryadVertexProgramBase
+   inherits from this interface and is set as the context for all
+   parsers and marshalers in that vertex so that they can access
+   configuration parameters, etc. */
+class RChannelContext : public IDrRefCounter
+{
+public:
+    virtual ~RChannelContext();
+};
+
+typedef DrRef<RChannelContext> RChannelContextRef;
+
+class RChannelInterruptHandler
+{
+public:
+    /* When the byte-oriented layer from which a channel is reading
+       signals an error condition by delivering a Restart or Abort
+       buffer, the application may wish to learn this immediately
+       rather than waiting for the error item to be delivered (for
+       example, the application may be blocking reads from the channel
+       indefinitely, yet still want timely error notifications.) In
+       this case, the application can register an interrupt handler
+       which will be called back when an error occurs, or during Drain
+       if no error occurs.
+
+       interruptItem is the terminatino item which will eventually be
+       delivered. The callee of ProcessInterrupt should call IncRef on
+       interruptItem if it wants to store a reference to the item.
+     */
+    virtual void ProcessInterrupt(RChannelItem* interruptItem) = 0;
+};
+
+
+class RChannelItemArrayReaderHandler
+{
+public:
+    RChannelItemArrayReaderHandler();
+    virtual ~RChannelItemArrayReaderHandler();
+
+    /* This sets the maximum number of items which will be delivered
+       via the ProcessItemArray callback. The default value is 1. */
+    void SetMaximumArraySize(UInt32 maximumArraySize);
+    UInt32 GetMaximumArraySize();
+
+    /* When an asynchronous item is ready to be delivered on a channel
+       this method is called on the handler object passed to the
+       associated call to RChannelReader::SupplyHandler().
+
+       If deliveredArray is empty and the asynchronous read was not
+       cancelled, then the channel has completed, i.e. a termination
+       item has been delivered to some other synchronous or
+       asynchronous read method. If there are multiple outstanding
+       reads the order of delivery is undefined so an empty array may
+       appear on one handler before the termination item has actually
+       been delivered.
+
+       If an asynchronous read is cancelled ProcessItem is called
+       before the Cancel call completes. If the array is empty in this
+       case it is impossible to determine from the ProcessItem
+       callback whether it occurred because of successful cancellation
+       or because a termination item has been delivered to the
+       application.
+     */
+    virtual void ProcessItemArray(RChannelItemArray* deliveredArray) = 0;
+
+    /* there are two types of asynchronous handler: queued and
+       immediate. If the handler is queued, then it is called on a
+       worker thread which is specifically assigned for processing
+       items, and the application should perform as much computation
+       as necessary on the calling thread before returning.
+
+       If the handler is immediate then it may be called on a variety
+       of different threads, including I/O processing threads, and the
+       application should return as soon as possible. Immediate
+       handlers are suitable, for example, for waking up a synchronous
+       thread which has been blocking on a read, or for queueing a
+       work item to an application's private thread pool if the
+       standard thread pool is for some reason inappropriate.
+
+       In general rather than implementing the ImmediateDispatch
+       method the application should just inherit from
+       RChannelItemReaderHandlerImmediate or
+       RChannelItemReaderHandlerQueued.
+    */
+    virtual bool ImmediateDispatch() = 0;
+
+private:
+    UInt32    m_maximumArraySize;
+};
+
+/* the base class for an immediate item array handler, see
+   RChannelItemArrayReaderHandler::ImmediateDispatch above. */
+class RChannelItemArrayReaderHandlerQueued :
+    public RChannelItemArrayReaderHandler
+{
+public:
+    bool ImmediateDispatch();
+};
+
+/* the base class for a queued item array handler, see
+   RChannelItemArrayReaderHandler::ImmediateDispatch above. */
+class RChannelItemArrayReaderHandlerImmediate :
+    public RChannelItemArrayReaderHandler
+{
+public:
+    bool ImmediateDispatch();
+};
+
+class RChannelItemReaderHandler :
+    public RChannelItemArrayReaderHandler
+{
+public:
+    virtual ~RChannelItemReaderHandler();
+
+    /* When an asynchronous item is ready to be delivered on a channel
+       this method is called on the handler object passed to the
+       associated call to RChannelReader::SupplyHandler().
+
+       If deliveredItem is NULL and the asynchronous read was not
+       cancelled, then the channel has completed, i.e. a termination
+       item has been delivered to some synchronous or asynchronous
+       read method. If there are multiple outstanding reads the order
+       of delivery is undefined so a NULL may appear on one handler
+       before the termination item has actually been delivered.
+
+       If an asynchronous read is cancelled ProcessItem is called with
+       NULL before the Cancel call completes. In this case it is
+       impossible to determine from the ProcessItem callback whether
+       it occurred because of successful cancellation or because a
+       termination item has been delivered to the application.
+
+       If deliveredItem is non-NULL then the callee now owns a
+       reference to deliveredItem.
+     */
+    virtual void ProcessItem(RChannelItem* deliveredItem) = 0;
+
+    /* there are two types of asynchronous handler: queued and
+       immediate. If the handler is queued, then it is called on a
+       worker thread which is specifically assigned for processing
+       items, and the application should perform as much computation
+       as necessary on the calling thread before returning.
+
+       If the handler is immediate then it may be called on a variety
+       of different threads, including I/O processing threads, and the
+       application should return as soon as possible. Immediate
+       handlers are suitable, for example, for waking up a synchronous
+       thread which has been blocking on a read, or for queueing a
+       work item to an application's private thread pool if the
+       standard thread pool is for some reason inappropriate.
+
+       In general rather than implementing the ImmediateDispatch
+       method the application should just inherit from
+       RChannelItemReaderHandlerImmediate or
+       RChannelItemReaderHandlerQueued.
+    */
+    virtual bool ImmediateDispatch() = 0;
+
+    /* implementation of RChannelItemArrayReaderHandler interface */
+    void ProcessItemArray(RChannelItemArray* deliveredArray);
+};
+
+/* the base class for an immediate item handler, see
+   RChannelItemReaderHandler::ImmediateDispatch above. */
+class RChannelItemReaderHandlerQueued : public RChannelItemReaderHandler
+{
+public:
+    bool ImmediateDispatch();
+};
+
+/* the base class for a queued item handler, see
+   RChannelItemReaderHandler::ImmediateDispatch above. */
+class RChannelItemReaderHandlerImmediate : public RChannelItemReaderHandler
+{
+public:
+    bool ImmediateDispatch();
+};
+
+class SyncItemReaderBase
+{
+public:
+    virtual DrError ReadItemSync(RChannelItemRef* item /* out */) = 0;
+};
+
+class RChannelReader : public SyncItemReaderBase
+{
+public:
+    virtual ~RChannelReader();
+
+    /* a client must call Start to cause the channel to start
+       generating items for the first time, or after a Drain has
+       completed before the channel is restarted.
+
+       prefetchCookie is passed to the byte-oriented buffer layer and
+       should be NULL for now.
+    */
+    virtual void Start(RChannelBufferPrefetchInfo* prefetchCookie) = 0;
+
+    /* SupplyHandler passes a handler which will be "returned" via a
+       matching call to handler->ProcessItemArray. Multiple handlers
+       may be outstanding at a given time. The channel blocks when
+       there are no handlers available, so this is the primary flow
+       control mechanism to allow the reader to exert back-pressure on
+       a channel.
+
+       If the channel has not been started or has delivered a
+       termination item to the application since the last call to
+       Start, handler->ProcessItemArray will be called with an empty
+       array on the calling thread before SupplyHandler returns.
+
+       If there is an item waiting to be delivered and handler is an
+       RChannelItemReaderHandlerImmediate then handler will be called
+       back with the items on the calling thread before SupplyHandler
+       returns.
+
+       If handler is an RChannelItemReaderHandlerQueued it will never
+       be called back on the calling thread. If there are items
+       waiting to be delivered, a processing request will be queued
+       for the item with handler, otherwise handler will be queued
+       waiting for the next item to be ready.
+
+       If thread B submits an RChannelItemReaderHandlerImmediate with
+       a call to SupplyHandler that is overlapped with thread A's call
+       to SupplyHandler, B's immediate handler may be called on A's
+       calling thread before A's call to SupplyHandler returns.
+
+       After a handler has been submitted to SupplyHandler with a
+       given value of cancelCookie, that handler is guaranteed to be
+       returned before any matching call to Cancel with the same value
+       of cancelCookie returns. If there is no item available at the
+       time the handler is cancelled it will be called with an empty
+       array. cancelCookie may take any value, including NULL, however
+       it is safest to use NULL or the address of an object owned by
+       the caller. The cancellation mechanism is used internally by
+       the synchronous FetchNextItem call which uses an allocated heap
+       address as its cancelCookie. Using the heap address of an
+       object which has not been freed before the call to Cancel will
+       avoid any danger of a cookie collision.
+
+       For any item A which has already been returned via an async
+       handler or call to FetchNextItem when SupplyHandler is called
+       it is guaranteed that A's sequence number is less than or equal
+       to the sequence number of the item eventually returned on
+       handler. Beyond this constraint, if multiple handlers are
+       outstanding at once, or while calls to FetchNextItem are in
+       progress, the order of delivered items is undefined.
+     */
+    virtual void SupplyHandler(RChannelItemArrayReaderHandler* handler,
+                               void* cancelCookie) = 0;
+
+    /* any handler passed to SupplyHandler with value cancelCookie is
+       guaranteed to be returned before a call to Cancel completes. */
+    virtual void Cancel(void* cancelCookie) = 0;
+
+    /* FetchNextItemArray blocks waiting until an item is available, a
+       termination item is delivered on another thread, or the timeOut
+       interval has elapsed (timeOut can be DrTimeInterval_Infinite in
+       which case FetchNextItem will block indefinitely).
+
+       If the timeout expires FetchNextItemArray returns false and
+       *pItemArray is an empty array. Otherwise FetchNextItem returns
+       true and the any returned items are stored in
+       *pItemArray. *pItemArray may be empty even if
+       FetchNextItemArray returns true: this will happen if Start has
+       not been called or a termination item has already been
+       delivered on the channel since the last call to Start. At most
+       maxArraySize items will be delivered in *pItemArray, however
+       fewer may be delivered.
+     */
+    virtual bool FetchNextItemArray(UInt32 maxArraySize,
+                                    RChannelItemArrayRef* pItemArray,
+                                    DrTimeInterval timeOut) = 0;
+
+    /* FetchNextItem blocks waiting until an item is available, a
+       termination item is delivered on another thread, or the timeOut
+       interval has elapsed (timeOut can be DrTimeInterval_Infinite in
+       which case FetchNextItem will block indefinitely).
+
+       If the timeout expires FetchNextItem returns false and *outItem
+       is NULL. Otherwise FetchNextItem returns true and the returned
+       item is stored in *outItem. *outItem may be NULL even if
+       FetchNextItem returns true: this will happen if Start has not
+       been called or a termination item has already been delivered on
+       the channel since the last call to Start.
+     */
+    bool FetchNextItem(RChannelItemRef* pOutItem,
+                       DrTimeInterval timeOut);
+    DrError ReadItemSync(RChannelItemRef* pOutItem /* out */);
+
+    /* Instruct the channel to stop reading items and prepare for
+       Drain. The interruptItem must be of type RChannelItem_Abort,
+       RChannelItem_Restart, RChannelItem_EndOfStream or
+       RChannelItem_ParseError. If the channel is an in-process FIFO
+       or a pipe which has not broken, the interruptItem will be
+       delivered to the remote end and the remote writer will
+       therefore be made aware of the channel close request. Interrupt
+       may be called multiple times after a channel has been Started
+       and before it has been Drained. The first call will have its
+       interruptItem delivered; subsequent interruptItems will be
+       discarded. If multiple calls to Interrupt are overlapped, the
+       call whose interruptItem gets delivered is undefined. The
+       caller's reference to interruptItem is not modified by this
+       call.
+     */
+    virtual void Interrupt(RChannelItem* interruptItem) = 0;
+
+    /* Instruct the channel to return all outstanding handlers via
+       their handler->ProcessItem callbacks in preparation for either
+       closing or restarting the channel. Once the Drain method
+       returns all outstanding handler callbacks will have completed
+       and all waiting calls to FetchNextItem will have been unblocked
+       (though of course they may not have returned to the calling
+       thread). For obvious reasons, Drain may not be called from a
+       handler's ProcessItem callback.
+
+       If the channel is an in-process FIFO or a pipe which has not
+       broken, and Interrupt has not previously been called, Drain
+       will send an appropriate termination item to the write end. If
+       the writer has delivered a termination item to the reader
+       implementation, an item of the same type will be returned to
+       the writer. Otherwise an item of type RChannel_Abort will be
+       returned to the writer. If the RChannelReader client has not
+       read a ternmination item, then the item delivered by a call to
+       Drain which is not preceded by Interrupt is undefined, since
+       the implementation may have received a termination item which
+       has not yet been forwarded to the client.
+     */
+    virtual void Drain() = 0;
+
+    /* If the writer has sent a termination item down the channel, it
+       is returned in pWriterDrainItem. If the reader has sent a
+       termination item to the writer (via Interrupt or drain) it is
+       returned in pReaderDrainItem. These termination items are set
+       to NULL when a channel is restarted. The returned items are
+       undefined if this call overlaps with a call to Start or Drain,
+       and it is illegal to call GetTerminationItem after Close has
+       been called. */
+    virtual void GetTerminationItems(RChannelItemRef* pWriterDrainItem,
+                                     RChannelItemRef* pReaderDrainItem) = 0;
+
+    /* After a call to Drain, this returns a status code and
+       optionally error metadata corresponding to the final status of
+       the channel. If the writer sent an error item,
+       GetTerminationStatus returns the error associated with that
+       item. Otherwise if the reader sent an Interrupt with an
+       EndOfStream item or called Drain after receiving an EndOfStream
+       item from the writer, GetTerminationStatus will return
+       DrError_EndOfStream (signalling clean termination). Otherwise
+       GetTerminationStatus returns DryadError_ProcessingInterrupted
+       which signifies that there was no error reported on the
+       channel, but the reader requested early termination. */
+    DrError GetTerminationStatus(DryadMetaDataRef* pErrorData);
+
+    /* Close may only be called if Start has never been called, or if
+       Drain has completed since the last call to Start. Close must be
+       called before the RChannelReader is destroyed. After Close has
+       been called no other methods may be called on RChannelReader.
+    */
+    virtual void Close() = 0;
+
+    /* Get the total length in bytes of the channel. This is known
+       only after the channel has been opened. */
+    virtual bool GetTotalLength(UInt64* pLen) = 0;
+
+    /* Get the total expected length in bytes of the channel. This may
+       be known even before the channel is opened. If
+       GetExpectedLength returns false, *pLen is undefined. */
+    virtual bool GetExpectedLength(UInt64* pLen) = 0;
+    virtual void SetExpectedLength(UInt64 expectedLength) = 0;
+
+    /* Get the URI of the channel. */
+    virtual const char* GetURI() = 0;
+
+    void SetTransformType(TransformType t) { m_transform = t; }
+    TransformType GetTransformType() { return m_transform; }
+
+private:
+     TransformType m_transform;
+};
+
+
+class RChannelItemArrayWriterHandler
+{
+public:
+    virtual ~RChannelItemArrayWriterHandler();
+
+    /* this interface is called whenever an async write (started by
+       RChannelWriter::WriteItemList) completes. If WriteItemList was
+       called with the flush flag set, this handler will not be
+       returned until the underlying channel has reported that the
+       write has completed. Otherwise this handler may be called
+       before the item has been written to the channel.
+
+       Normally status is RChannelItem_Data, however if the underlying
+       channel implementation has requested a restart or abort, status
+       will be RChannelItem_Restart or RChannelItem_Abort
+       respectively. If WriteItem was called before the channel was
+       started, or after a termination item had been written, then
+       status will be RChannelItem_EndOfStream. If status is not
+       RChannelItem_Data the item may or may not have actually been
+       written to the output, regardless of whether a flush was
+       requested (if status is RChannelItem_EndOfStream the item has
+       definitely not been written to the output).
+
+       If the item marshaler encountered any errors the error
+       descriptions will be returned in failureArray. This is an array
+       of items: each element is non-NULL if and only if the
+       corresponding item in the original list had a marshal error. If
+       there were no marshaling errors, failureArray will be NULL.
+    */
+    virtual void ProcessWriteArrayCompleted(RChannelItemType status,
+                                            RChannelItemArray*
+                                            failureArray) = 0;
+};
+
+class RChannelItemWriterHandler :
+    public RChannelItemArrayWriterHandler
+{
+public:
+    virtual ~RChannelItemWriterHandler();
+
+    /* this interface is called whenever an async write (started by
+       RChannelWriter::WriteItem) completes. If WriteItem was called
+       with the flush flag set, this handler will not be returned
+       until the underlying channel has reported that the write has
+       completed. Otherwise this handler may be called before the item
+       has been written to the channel.
+
+       Normally status is RChannelItem_Data, however if the underlying
+       channel implementation has requested a restart or abort, status
+       will be RChannelItem_Restart or RChannelItem_Abort
+       respectively. If WriteItem was called before the channel was
+       started, or after a termination item had been written, then
+       status will be RChannelItem_EndOfStream. If status is not
+       RChannelItem_Data the item may or may not have actually been
+       written to the output, regardless of whether a flush was
+       requested (if status is RChannelItem_EndOfStream the item has
+       definitely not been written to the output).
+
+       If the item marshaler encountered an error the error
+       description will be returned in marshalFailureItem. The callee
+       does not own a reference to marshalFailureItem.
+    */
+    virtual void ProcessWriteCompleted(RChannelItemType status,
+                                       RChannelItem* marshalFailureItem) = 0;
+
+    /* implementation of RChannelItemArrayWriterHandler interface */
+    void ProcessWriteArrayCompleted(RChannelItemType status,
+                                    RChannelItemArray* failureArray);
+};
+
+class SyncItemWriterBase
+{
+public:
+    /* A client should call WriteMapOutputItem or
+       WriteMapOutputItemConsumingReference whenever it wants to send
+       an item to the output channel. The latter consumes a reference
+       to item, the former does not. */
+    virtual void WriteItemSyncConsumingReference(RChannelItemRef& item) = 0;
+    void WriteItemSyncConsumingFreeReference(RChannelItem* item);
+    void WriteItemSync(RChannelItem* item);
+    virtual DrError GetWriterStatus() = 0;
+};
+
+/*
+  The RChannelWriter is the primary mechanism for writing
+  application-specific structured items to an underlying byte-oriented
+  channel.
+
+  The application first calls Start, then sends a series of items,
+  primarily of type RChannelItem_Data, though markers may be
+  interspersed, followed by a termination item of type
+  RChannelItem_Restart, RChannelItem_Abort or
+  RChannelItem_EndOfStream. In the case of a pipe which has not
+  broken, the termination item will be sent to the remote end so that
+  the consuming process learns the reason for the pipe closure.
+
+  After sending a termination item no further items will be sent on
+  the channel until a call to Drain has completed. After this point
+  the channel may call Start again in the case of a restart, and start
+  sending items again.
+
+  Each item is marshaled into the byte-stream using an
+  application-specific marshaler object which must co-operate with the
+  application so that bare RChannelItem objects can be cast by it into
+  objects containing meaningful data.
+ */
+class RChannelWriter : public SyncItemWriterBase
+{
+public:
+    virtual ~RChannelWriter();
+
+    /* a client must call Start before writing items to the channel
+       the first time, or after a Drain has completed before writing
+       to a restarted channel.
+    */
+    virtual void Start() = 0;
+
+    /* WriteItem queues item for async write to the channel.
+       handler->ProcessWriteCompleted will be called when the async
+       write "completes."
+
+       If flushAfter is false the handler may be returned before the
+       item has actually been written on the underlying channel. If
+       flushAfter is true the handler will not be returned until the
+       underlying channel has reported that the item has been
+       written. Setting flushAfter potentially causes the underlying
+       channel to write a partial buffer and should be used sparingly
+       where performance is important.
+
+       WriteItem transfers the caller's reference to item to the
+       channel and it is guaranteed that handler will be called before
+       the next call to Drain completes. If Start has not yet been
+       called or a termination item has been queued since the last
+       call to Start, handler->ProcessWriteCompleted will be called
+       with status RChannelItem_EndOfStream on the caller's stack
+       before WriteItem returns.
+
+       Flow control is exercised by having the channel refrain from
+       calling handler->ProcessWriteCompleted if the underlying
+       stream has blocked. Therefore the application should limit the
+       total number of outstanding calls to WriteItem in flight whose
+       handlers have not yet been returned. Async items are serialized
+       to a queue and marshaled on a worker thread, and handler is
+       returned after this marshaling completes, so in the common case
+       where the underlying channel has not blocked, the application
+       should allow enough outstanding handlers to account for this
+       delay in order to avoid "bubbles" in the processing pipeline.
+
+       Once a call to WriteItem passing item A has completed, then any
+       items which are subsequently submitted to the channel are
+       guaranteed to appear after A. Beyond this constraint, if calls
+       to WriteItem and/or WriteItemSync are overlapped the order of
+       serialization to the channel is undefined.
+    */
+    virtual void WriteItemArray(RChannelItemArrayRef& itemArray,
+                                bool flushAfter,
+                                RChannelItemArrayWriterHandler* handler) = 0;
+    void WriteItem(RChannelItem* item,
+                   bool flushAfter,
+                   RChannelItemArrayWriterHandler* handler);
+
+    /* WriteItemArraySync submits item to be written to the channel and
+       may block indefinitely if the underlying channel has blocked.
+
+       If flush is false, WriteItemArraySync may return before the array
+       has actually been sent to the underlying channel. If flush is
+       true WriteItemListSync will not return until the underlying
+       channel reports that item has been written. Setting flush
+       potentially causes the underlying channel to write a partial
+       buffer and should be used sparingly where efficiency is
+       crucial.
+
+       If the marshaler encounters an error when marshaling any item
+       in the list, and pFailureArray is non-NULL, then pFailureArray
+       will be filled in with an array of the same size as the
+       original itemList. This array will contain non-NULL elements
+       corresponding to any item in itemList which had a marshal
+       error. If pFailureArray is non-NULL and there are no marshaling
+       errors, *pFailureArray=NULL on return.
+
+       WriteItemListSync always consunes the caller's reference to
+       each item in the list and leaves itemList empty. If Start has
+       not yet been called or a termination item has been queued since
+       the last call to Start WriteItemListSync returns
+       RChannelItem_EndOfStream, no item is written to the channel,
+       and *pFailureArray is guaranteed to be NULL if pFailureArray is
+       non-NULL. Otherwise, normally status is RChannelItem_Data,
+       however if the underlying channel implementation has requested
+       a restart or abort, status will be RChannelItem_Restart or
+       RChannelItem_Abort respectively. If status is
+       RChannelItem_Restart or RChannelItem_Abort the item in the list
+       may or may not have actually been written to the output,
+       regardless of whether a flush was requested.
+
+       Once a call to WriteItemListSync passing item A has completed,
+       then any items which are subsequently submitted to the channel
+       are guaranteed to appear after A. Beyond this constraint, if
+       calls to WriteItem and/or WriteItemSync are overlapped the
+       order of serialization to the channel is undefined.
+    */
+    virtual RChannelItemType
+        WriteItemArraySync(RChannelItemArrayRef& itemArray,
+                           bool flush,
+                           RChannelItemArrayRef* pFailureArray) = 0;
+
+    /* WriteItemSync submits item to be written to the channel and may
+       block indefinitely if the underlying channel has blocked.
+
+       If flush is false, WriteItemSync may return before item has
+       actually been sent to the underlying channel. If flush is true
+       WriteItemSync will not return until the underlying channel
+       reports that item has been written. Setting flush potentially
+       causes the underlying channel to write a partial buffer and
+       should be used sparingly where efficiency is crucial.
+
+       If the marshaler encounters an error when marshaling item and
+       pMarshalFailureItem is non-NULL, *pMarshalFailureItem will
+       contain an item describing that error. In this case item has
+       not been marshaled to the channel, however *pMarshalFailureItem
+       has been. If there is no marshaling error and
+       pMarshalFailureItem is non-NULL, *pMarshalFailureItem is NULL.
+
+       WriteItemSync does not consume the caller's reference to
+       item. If Start has not yet been called or a termination item
+       has been queued since the last call to Start WriteItemSync
+       returns RChannelItem_EndOfStream, item is not written to the
+       channel, and *pMarshalFailureItem is guaranteed to be NULL if
+       pMarshalFailureItem is non-NULL. Otherwise, normally status is
+       RChannelItem_Data, however if the underlying channel
+       implementation has requested a restart or abort, status will be
+       RChannelItem_Restart or RChannelItem_Abort respectively. If
+       status is RChannelItem_Restart or RChannelItem_Abort the item
+       may or may not have actually been written to the output,
+       regardless of whether a flush was requested.
+
+       Once a call to WriteItemSync passing item A has completed, then
+       any items which are subsequently submitted to the channel are
+       guaranteed to appear after A. Beyond this constraint, if calls
+       to WriteItem and/or WriteItemSync are overlapped the order of
+       serialization to the channel is undefined.
+    */
+    RChannelItemType
+        WriteItemSync(RChannelItem* item,
+                      bool flush,
+                      RChannelItemRef* pMarshalFailureItem);
+    void WriteItemSyncConsumingReference(RChannelItemRef& item);
+
+    /* Drain may not be called until after a termination item has been
+       written to the channel using a call to WriteItemListSync,
+       WriteItemSync, WriteItemList or WriteItem which has
+       return. Drain will not return until the underlying channel has
+       drained and all outstanding async handlers submitted to
+       WriteItem or WriteItemList have been returned. At this point
+       any calls to WriteItemSync or WriteItemListSync will be
+       unblocked, though of course they may not have returned to the
+       caller.
+
+       If the application does not wish to block waiting for Drain it
+       should wait until the handler submitted with the termination
+       item has returned (or use a blocking call to WriteItemSync or
+       WriteItemListSync). This handler will not be returned until the
+       underlying channel has drained, and it is always the last
+       handler to be returned, so Drain will not block after it has
+       been returned.
+
+       The call to Drain will block waiting for the "remote end" of
+       the channel to signal that the channel has been shut down. In
+       the case of a file or stream this happens immediately. In the
+       case of a pipe or FIFO this happens only when the reader end
+       calls Interrupt or Drain, and the item sent to the reader's
+       Interrupt command is delivered by the writer's Drain and stored
+       in *pRemoteStatus if pRemoteStatus is non-NULL. The item will
+       be type RChannelItem_Abort, RChannelItem_Restart,
+       RChannelItem_ParseError or RChannelItem_EndOfStream. The
+       writer's call to Drain will block for at most a time interval
+       of timeOut before returning. If the timeout period expires
+       before the remote end's drain item is received and
+       pRemoteStatus is non-NULL, *pRemoteStatus will be set to NULL.
+    */
+    virtual void Drain(DrTimeInterval timeOut,
+                       RChannelItemRef* pRemoteStatus) = 0;
+
+    /* (*pWriterDrainItem) is set to the termination item written to
+       the channel, or NULL if no termination item has been written
+       since the most recent call to Start.  *pReaderDrainItem is set
+       to the remote status item returned by the most recent call to
+       Drain, or NULL if Drain has not been called since the last call
+       to Start. The return values are undefined if this call overlaps
+       with a call to Start or Drain, and it is illegal to call
+       GetTerminationItems after Close has been called.
+    */
+    virtual void GetTerminationItems(RChannelItemRef* pWriterDrainItem,
+                                     RChannelItemRef* pReaderDrainItem) = 0;
+
+    /* This returns an error corresponding to the most recent
+       termination item written to the channel. It is illegal to call
+       GetTerminationStatus before sending a termination item. */
+    DrError GetTerminationStatus(DryadMetaDataRef* pErrorData);
+
+    DrError GetWriterStatus();
+
+    /* Close may not be called unless Start has never been called or
+       Drain has completed since the last call to Start. After Close
+       is called the channel may not be restarted. */
+    virtual void Close() = 0;
+
+    /* Get/set a hint about the total length the channel is expected
+       to be. Some channel implementations can use this to improve
+       write performance and decrease disk fragmentation. A value of 0
+       (the default) means that the size is unknown. */
+    virtual UInt64 GetInitialSizeHint() = 0;
+    virtual void SetInitialSizeHint(UInt64 hint) = 0;
+
+    /* Get the URI of the channel. */
+    virtual const char* GetURI() = 0;
+
+    void SetTransformType(TransformType t) { m_transform = t; }
+    TransformType GetTransformType() { return m_transform; }
+
+private:
+     TransformType m_transform;
+
+};

DryadVertex/VertexHost/system/channel/include/channelitem.h

@@ -0,0 +1,294 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include <DrCommon.h>
+#include <dryadlisthelper.h>
+#include <dryadmetadata.h>
+#include <channelmemorybuffers.h>
+
+/*
+  Structured data elements passed on restartable channels are called
+  Items.
+
+  Every item is derived from the RChannelItem base class and has a
+  type, a sequence number and a metadata description. The type is
+  assigned by the creator of the item, and is discussed below. The
+  sequence number is only used for items being read from a channel and
+  is assigned automatically by the channel reader mechanism. The
+  metadata is initially populated by the item creator and may be
+  initialized from the underlying data stream on a read channel, but
+  may be added to or modified subsequently during processing.
+
+  Items fall into two fundamental classes: data items and marker
+  items. Data items have type RChannelItem_Data and carry
+  application-specific data. A data item which is read from a channel
+  is guaranteed to have been created by an application-specific parser
+  object which consumes byte-oriented buffers, and may be cast by the
+  application into a rich type. The sequence number of a data item is
+  automatically assigned by the channel reader machinery, and data
+  items read from a channel have unique, densely assigned, increasing
+  sequence numbers starting at zero.
+
+  All non-data item types are known as marker items and are used to
+  represent and serialize information such as error conditions and
+  end-of-stream information. Errors can include information about
+  "holes" in the underlying data stream (caused e.g. when a subset of
+  the input data is unavailable) and parse failures. Marker items may
+  be produced and consumed by a variety of components, and information
+  is passed between these components using the metadata. There are
+  convenience methods below for creating standard error marker
+  items. The sequence number of a marker item is automatically
+  assigned by the channel reader machinery, and it is equal to the
+  sequence number of the next data item (or, equivalently, one greater
+  than the sequence number of the preceding data item).
+
+  On write, all items are passed to an application-specific marshaling
+  object which knows how to serialize both marker items and data items
+  with a particular format into byte-oriented buffers. Not all
+  channels store or transmit marker items, so it is legal for the
+  marshaler to do nothing when presented with a marker item, but in
+  general it is expected that applications will make use of marker
+  items to convey rich monitoring and debugging information on
+  internal channels between the vertices of a distributed computation.
+
+  A valid complete channel consists of a sequence of "body" items
+  followed by a single "termination" item. A channel reader will
+  always produce such a sequence unless interrupted by the application
+  calling Drain prematurely, and a channel writer should be fed such a
+  sequence. Termination items have type EndOfStream, Abort, Restart,
+  ParseError and MarshalError: all other items are body items and may
+  appear in any sequence in a valid channel. No item may appear in a
+  channel after a termination item.
+
+  The item constructor is private to allow applications to use private
+  memory management. Each item supplies a Create method and is
+  reference-counted: a reference is released by calling DecRef on the
+  item.
+*/
+
+enum RChannelItemType {
+    /* RChannelItem_Data: a data item generated by application
+       specific code in the parser or the application body. The
+       application may cast this item to a rich derived type. */
+    RChannelItem_Data,
+
+    /* RChannelItem_BufferHole: a marker item supplied by the buffer
+       reader describing a hole in the underlying data stream. */
+    RChannelItem_BufferHole,
+
+    /* RChannelItem_ItemHole: a marker item supplied by the parser
+       describing a hole in the parsed data stream (usually occurring
+       because of malformed input data which the parser has skipped
+       over). */
+    RChannelItem_ItemHole,
+
+    /* RChannelItem_EndOfStream: a marker item appearing as the last
+       item in the stream and signaling clean completion. */
+    RChannelItem_EndOfStream,
+
+    /* RChannelItem_Restart: a marker item appearing as the last item
+       in the stream signaling an error condition and requesting that
+       the channel be restarted if possible. */
+    RChannelItem_Restart,
+
+    /* RChannelItem_Abort: a marker item appearing as the last item in
+       the stream signaling an unrecoverable error condition. */
+    RChannelItem_Abort,
+
+    /* RChannelItem_ParseError: a marker item supplied by the parser
+       signifying an unrecoverable error. */
+    RChannelItem_ParseError,
+
+    /* RChannelItem_MarshalError: a marker item supplied by the parser
+       signifying an unrecoverable error. */
+    RChannelItem_MarshalError
+};
+
+class RChannelItem;
+typedef DrRef<RChannelItem> RChannelItemRef;
+
+class DrResettableMemoryReader : public DrMemoryBufferReader
+{
+public:
+    DrResettableMemoryReader(DrMemoryBuffer *pMemoryBuffer);
+    void ResetToBufferOffset(Size_t offset);
+};
+
+class RChannelItem : public DrRefCounter
+{
+public:
+    /* returns true if the item can be used as a channel termination
+       marker, false otherwise */
+    static bool IsTerminationItem(RChannelItemType type);
+
+    /* return the type of the item, set at creation time. */
+    RChannelItemType GetType();
+
+    /* The sequence number of the item in the Channel with respect to
+       all delivered Data items. This sequence number is generated
+       locally, and so if a Data item appears after any holes in the
+       stream it may not correspond to the sequence number for this
+       item when read a second time, or from another reader.
+
+       If the item type is not RChannelItem_Data this number is the
+       sequence number of the *next* Data item, if any. Data items are
+       assigned dense, increasing sequence numbers.
+     */
+    UInt64 GetDataSequenceNumber();
+
+    /* The sequence number of the item in the Channel with respect to
+       all delivered items. This sequence number is generated locally,
+       and so if there are any markers or holes in the stream it may
+       not correspond to the sequence number for this item when read a
+       second time, or from another reader. Items are assigned dense,
+       increasing sequence numbers.
+     */
+    UInt64 GetDeliverySequenceNumber();
+
+    /* this returns a description of the item, used for debugging and
+       monitoring purposes. When the item is created the metadata is
+       NULL.
+     */
+    DryadMetaData* GetMetaData();
+
+    /* these set the item's sequence numbers and are called by the
+       channel reader */
+    void SetDataSequenceNumber(UInt64 dataSequenceNumber);
+    void SetDeliverySequenceNumber(UInt64 deliverySequenceNumber);
+
+    /* this replaces the item's current metadata with a new object */
+    void ReplaceMetaData(DryadMetaData* metaData);
+
+    /* this does a shallow copy. By default this assert-fails, and
+       concrete Data items which need to be cloned must implement
+       an item-specific method. */
+    virtual void Clone(RChannelItemRef* pClonedItem);
+
+    virtual UInt64 GetNumberOfSubItems() const;
+    virtual void TruncateSubItems(UInt64 numberOfSubItems);
+    virtual UInt64 GetItemSize() const;
+
+    virtual DrError DeSerialize(DrResettableMemoryReader* reader,
+                                Size_t availableSize);
+    virtual DrError DeSerializePartial(DrResettableMemoryReader* reader,
+                                       Size_t availableSize);
+    virtual DrError Serialize(ChannelMemoryBufferWriter* writer);
+
+    /* if the item has metadata which contains a Prop_Dryad_ErrorCode
+       then this is returned. Otherwise the error depends on the type:
+       RChannelItem_Data: DrError_OK
+       RChannelItem_BufferHole: DryadError_BufferHole
+       RChannelItem_ItemHole: DryadError_ItemHole
+       RChannelItem_EndOfStream: DrError_EndOfStream
+       RChannelItem_Restart: DryadError_ChannelRestart
+       RChannelItem_Abort: DryadError_ChannelAbort
+       RChannelItem_ParseError: DryadError_ItemParseError
+       RChannelItem_MarshalError: DryadError_ItemMarshalError
+    */
+    DrError GetErrorFromItem();
+
+    static const UInt64 s_invalidSequenceNumber = ((UInt64) -1);
+
+    static const UInt32 s_defaultItemBatchSize = 16;
+    static const UInt32 s_defaultRecordBatchSize = 256;
+
+protected:
+    RChannelItem(RChannelItemType type);
+    virtual ~RChannelItem();
+
+private:
+    RChannelItemType   m_type;
+    UInt64             m_dataSequenceNumber;
+    UInt64             m_deliverySequenceNumber;
+    DryadMetaDataRef   m_metaData;
+    DrBListEntry       m_listPtr;
+    friend class DryadBList<RChannelItem>;
+};
+
+typedef DryadBList<RChannelItem> RChannelItemList;
+
+class RChannelMarkerItem : public RChannelItem
+{
+public:
+    /* used to create a standard marker item. If withMetaData is true,
+       an empty metadata object is created with the item, otherwise
+       the item's metadata is left NULL. */
+    static RChannelMarkerItem*
+        Create(RChannelItemType type, bool withMetaData);
+
+    /* this does a shallow copy, which shares the same metadata as the
+       original (with an increased refcount). */
+    virtual void Clone(RChannelItemRef* pClonedItem);
+
+
+    /* these convenience methods create marker items with common
+       metadata fields already filled in */
+
+    /* fills in the metadata element Prop_Dryad_ErrorCode with the
+       supplied information */
+    static RChannelItem* CreateErrorItem(RChannelItemType itemType,
+                                         DrError errorCode);
+    /* fills in the metadata elements Prop_Dryad_ErrorCode and
+       Prop_Dryad_ErrorString with the supplied information. */
+    static RChannelItem*
+        CreateErrorItemWithDescription(RChannelItemType itemType,
+                                       DrError errorCode,
+                                       const char* errorDescription);
+
+protected:
+    RChannelMarkerItem(RChannelItemType type);
+    virtual ~RChannelMarkerItem();
+};
+
+class RChannelDataItem : public RChannelItem
+{
+public:
+    virtual UInt64 GetNumberOfSubItems() const;
+    virtual UInt64 GetItemSize() const;
+
+protected:
+    RChannelDataItem();
+    virtual ~RChannelDataItem();
+};
+
+class RChannelItemArray : public DrRefCounter
+{
+public:
+    RChannelItemArray();
+    ~RChannelItemArray();
+
+    void SetNumberOfItems(UInt32 numberOfItems);
+    void ExtendNumberOfItems(UInt32 numberOfItems);
+    UInt32 GetNumberOfItems();
+
+    void TruncateToSize(UInt32 prefix);
+    void DiscardPrefix(UInt32 prefix);
+
+    RChannelItemRef* GetItemArray();
+
+private:
+    UInt32            m_numberOfItems;
+    RChannelItemRef*  m_baseItemArray;
+    RChannelItemRef*  m_itemArray;
+};
+
+typedef DrRef<RChannelItemArray> RChannelItemArrayRef;

DryadVertex/VertexHost/system/channel/include/channelmarshaler.h

@@ -0,0 +1,161 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include <DrCommon.h>
+#include <channelinterface.h>
+#include <channelbuffer.h>
+#include <errorreporter.h>
+
+/* The RChannelItemMarshaler is used to supply application-specifc
+   marshaling code to convert an RChannelItem object to a byte stream.
+
+   It is expected that most marshalers will be stateless, but the
+   marshaler is called sequentially with each item in turn, so it is
+   possible to implement marshalers with state if desired.
+
+   If an RChannelItemMarshaler object is passed to only one
+   RChannelWriter object then calls to its methods will never be
+   overlapped. Methods are called on worker threads and should compute
+   for as long as necessary before returning. The worker threads are
+   used for item parsing, item marshaling, item processing and other
+   actions.
+ */
+class RChannelItemMarshalerBase : public IDrRefCounter
+{
+public:
+    RChannelItemMarshalerBase();
+    virtual ~RChannelItemMarshalerBase();
+
+    void SetMaxMarshalBatchSize(UInt32 maxMarshalBatchSize);
+    UInt32 GetMaxMarshalBatchSize();
+
+    void SetMarshalerContext(RChannelContext* context);
+    RChannelContext* GetMarshalerContext();
+
+    /* this indicates, e.g., which channel on a vertex the marshaler
+       is attached to. It may not be unique in a given process if
+       there are e.g. multiple vertices. */
+    void SetMarshalerIndex(UInt32 index);
+    UInt32 GetMarshalerIndex();
+
+    /* Reset is called before any items are marshaled to a stream. If
+       a channel is restarted, Reset will be called before starting to
+       write items again. */
+    virtual void Reset();
+
+    /* MarshalItem is called whenever a new item should be written to
+       the underlying stream. The item should be marshaled using the
+       DrMemoryWriter writer in such a way that a matching
+       RChannelItemParser object can unmarshal it subsequently.
+
+       If the marshaler succeeds in writing the item, it should return
+       DrError_OK and *pFailureItem will be ignored. It is legal to
+       write no data for some items or item types if the application
+       semantics permit this.
+
+       If the marshaler encounters an error when parsing it should
+       attempt to recover and return a descriptive item of type
+       RChannelItem_MarshalError in *pFailureItem rather than, for
+       example, asserting. In this case, any data written to writer
+       will be discarded, and self will immediately be called back
+       with the RChannelItem_MarshalError item. Any failure item
+       returned by this second call will be ignored.
+
+       If MarshalItem returns DryadError_ChannelAbort or
+       DryadError_ChannelRestart then the appropriate termination item
+       will be sent to the channel preventing subsequent items from
+       being marshaled. Any other error code will cause the channel to
+       continue marshaling items past the error.
+
+       flush is set if the channel will be flushed after this item is
+       marshaled. This can be ignored by most marshalers.
+    */
+    virtual DrError MarshalItem(ChannelMemoryBufferWriter* writer,
+                                RChannelItem* item,
+                                bool flush,
+                                RChannelItemRef* pFailureItem) = 0;
+
+private:
+    UInt32                       m_maxMarshalBatchSize;
+    UInt32                       m_index;
+    RChannelContextRef           m_context;
+};
+
+typedef DrRef<RChannelItemMarshalerBase> RChannelItemMarshalerRef;
+
+class RChannelItemMarshaler : public RChannelItemMarshalerBase
+{
+public:
+    ~RChannelItemMarshaler();
+    DRREFCOUNTIMPL
+};
+
+class RChannelStdItemMarshalerBase : public RChannelItemMarshalerBase
+{
+public:
+    virtual ~RChannelStdItemMarshalerBase();
+
+    DrError MarshalItem(ChannelMemoryBufferWriter* writer,
+                        RChannelItem* item,
+                        bool flush,
+                        RChannelItemRef* pFailureItem);
+
+    virtual DrError MarshalMarker(ChannelMemoryBufferWriter* writer,
+                                  RChannelItem* item,
+                                  bool flush,
+                                  RChannelItemRef* pFailureItem);
+};
+
+class RChannelStdItemMarshaler : public RChannelStdItemMarshalerBase
+{
+public:
+    virtual ~RChannelStdItemMarshaler();
+    DRREFCOUNTIMPL
+};
+
+class DryadMarshalerFactoryBase : public IDrRefCounter
+{
+public:
+    virtual ~DryadMarshalerFactoryBase();
+    virtual void MakeMarshaler(RChannelItemMarshalerRef* pMarshaler,
+                               DVErrorReporter* errorReporter) = 0;
+};
+
+typedef DrRef<DryadMarshalerFactoryBase> DryadMarshalerFactoryRef;
+
+class DryadMarshalerFactory : public DryadMarshalerFactoryBase
+{
+public:
+    virtual ~DryadMarshalerFactory();
+    DRREFCOUNTIMPL
+};
+
+template<class _T> class StdMarshalerFactory : public DryadMarshalerFactory
+{
+public:
+    typedef _T Marshaler;
+    void MakeMarshaler(RChannelItemMarshalerRef* pMarshaler,
+                       DVErrorReporter* errorReporter)
+    {
+        pMarshaler->Attach(new Marshaler());
+    }
+};

DryadVertex/VertexHost/system/channel/include/channelmemorybuffers.h

@@ -0,0 +1,141 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include <portmemorybuffers.h>
+
+class RChannelBufferData;
+class RChannelBufferWriter;
+
+class ChannelDataBufferList;
+
+class RChannelReaderBuffer : public DrMemoryBuffer
+{
+public:
+    RChannelReaderBuffer(DryadLockedBufferList* bufferList,
+                         Size_t startOffset, Size_t endOffset);
+    RChannelReaderBuffer(ChannelDataBufferList* bufferList,
+                         Size_t startOffset, Size_t endOffset);
+    ~RChannelReaderBuffer();
+
+    //
+    // Retrieve pointer to the data stored in memory block at uOffset
+    // and max size available in this block
+    //
+    // Returns NULL if no data at this offset, valid pointer otherwise
+    //
+    void *GetDataAddress(Size_t uOffset,
+                         Size_t *puSize,
+                         Size_t *puPriorSize);
+
+    //
+    // Preallocate enough memory buffers to fix uMaxSize bytes of data.
+    //
+    void IncreaseAllocatedSize(Size_t uSize);
+
+private:
+    class Current
+    {
+    public:
+        Current(Size_t initialStartOffset,
+                Size_t finalEndOffset,
+                DryadLockedMemoryBuffer** bufferArray,
+                size_t nBuffers,
+                Size_t offset);
+
+        void* GetDataAddress(DryadLockedMemoryBuffer** bufferArray,
+                             Size_t offset,
+                             Size_t *puSize,
+                             Size_t *puPriorSize);
+
+    private:
+        void SetTailBufferData(DryadLockedMemoryBuffer** bufferArray,
+                               size_t nBuffers,
+                               Size_t finalEndOffset);
+
+        size_t             m_currentBuffer;
+        Size_t             m_currentBufferBase;
+        Size_t             m_currentHeadCutLength;
+        Size_t             m_currentTailOffset;
+    };
+
+    void Initialise(Size_t startOffset, Size_t endOffset);
+
+    DryadLockedMemoryBuffer**     m_bufferArray;
+    size_t                        m_nBuffers;
+    Size_t                        m_initialStartOffset;
+    Size_t                        m_finalEndOffset;
+};
+
+class RChannelWriterBuffer : public DrMemoryBuffer
+{
+public:
+    RChannelWriterBuffer(RChannelBufferWriter* bufferProvider,
+                         DryadFixedBufferList* bufferList);
+    ~RChannelWriterBuffer();
+
+    //
+    // Retrieve pointer to the data stored in memory block at uOffset
+    // and max size available in this block
+    //
+    // Returns NULL if no data at this offset, valid pointer otherwise
+    //
+    void *GetDataAddress(Size_t uOffset,
+                         Size_t *puSize,
+                         Size_t *puPriorSize);
+
+    //
+    // Preallocate enough memory buffers to fix uMaxSize bytes of data.
+    //
+    void IncreaseAllocatedSize(Size_t uSize);
+
+protected:
+    void InternalSetAvailableSize(Size_t uSize);
+
+private:
+    DryadFixedBufferList*          m_bufferList;
+
+    Size_t                         m_currentBufferOffset;
+    Size_t                         m_currentBaseOffset;
+    Size_t                         m_baseBufferOffset;
+
+    Size_t                         m_availableHighWaterMark;
+    Size_t                         m_availableStartOffset;
+    Size_t                         m_availableBufferOffset;
+    DrBListEntry*                  m_lastAvailableBuffer;
+
+    RChannelBufferWriter*          m_bufferProvider;
+};
+
+class ChannelMemoryBufferWriter : public DrMemoryBufferWriter
+{
+public:
+    ChannelMemoryBufferWriter(DrMemoryBuffer* writer,
+                              DryadFixedBufferList* bufferList);
+
+    bool MarkRecordBoundary();
+    Size_t GetLastRecordBoundary();
+
+private:
+    Size_t                  m_initialBoundary;
+    Size_t                  m_lastRecordBoundary;
+    DryadFixedBufferList*   m_bufferList;
+};

DryadVertex/VertexHost/system/channel/include/channelparser.h

@@ -0,0 +1,560 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "channelbuffer.h"
+#include "dobjpool.h"
+#include <errorreporter.h>
+#include "recordarray.h"
+
+class RChannelRawItemParser;
+
+/* The classes in this header file must be overridden to implement an
+   application-specific parser which can take application-specific
+   data from a byte-oriented channel and parse it into RChannelItem
+   objects.
+
+   At the start of the header is the RChannelRawItemParser class which
+   may be overridden directly in the unlikely event that an
+   application needs full control over the parsing process. Almost all
+   applications will be able to derive from the RChannelItemParser or
+   RChannelLengthDelimitedItemParser classes which follow.
+   RChannelLengthDelimitedItemParser is somewhat simpler to use in the
+   case that the length of items in the channel can be determined by
+   reading a prefix of the item.
+
+   Assuming that a parser object is only passed to a single
+   RChannelReader object, no method calls on the parser will ever be
+   overlapped. Methods are called on worker threads and should compute
+   for as long as necessary before returning. The worker threads are
+   used for item parsing, item marshaling, item processing and other
+   actions.
+*/
+
+/* RChannelRawItemParser is the raw interface for unmarshaling
+   structured items from a byte-stream, however the convenience
+   classes below will be more suitable for most applications. */
+class RChannelItemParserBase : public IDrRefCounter
+{
+public:
+    RChannelItemParserBase();
+    virtual ~RChannelItemParserBase();
+
+    void SetMaxParseBatchSize(UInt32 maxParseBatchSize);
+    UInt32 GetMaxParseBatchSize();
+
+    void SetParserContext(RChannelContext* context);
+    RChannelContext* GetParserContext();
+
+    /* this indicates, e.g., which channel on a vertex the parser is
+       attached to. It may not be unique in a given process if there
+       are e.g. multiple vertices. */
+    void SetParserIndex(UInt32 index);
+    UInt32 GetParserIndex();
+
+    /* The Item Parser takes a sequence of data buffers from the
+       underlying Channel and returns a sequence of items parsed out
+       of the data.
+
+       A single method call, RawParseItem, implements the API between
+       the Channel and the parser. In the common case, when
+       RawParseItem is called the parser returns the next item from
+       whatever buffer data it has cached: it must maintain enough
+       state to be able to keep track of which the next item is.
+
+       If the parser does not have enough cached data to return the
+       next item (e.g. an item straddles two or more data buffers) it
+       signals this on return from the method, and the next call to
+       RawParseItem will supply the next buffer in sequence.
+
+       The buffers are supplied as a sequence of RChannelBuffer_Data
+       and RChannelBuffer_Hole buffers followed by a single
+       RChannelBuffer_EndOfStream though there may be a restart at any
+       time. The returned items are of type RChannelItem_Data,
+       RChannelItem_ItemHole, RChannelItem_BufferHole,
+       RChannelItem_EndOfStream or RChannelItem_ParseError and between
+       restarts they obey the pattern sequence
+
+         (Data | ItemHole | BufferHole)* (EndOfStream | ParseError)
+
+       The detailed method description follows:
+
+       The code calling the parser behaves as though it stores a
+       parserState variable which can take on the values NeedsData,
+       DataReady and Stopped; this variable is updated after every
+       call to RawParseItem. The rules for calling ItemParser are as
+       follows:
+
+         1) At any time RawParseItem may be called with
+         resetParser=true. This signifies that the entire channel has
+         been restarted. In this case inData contains a pointer to a
+         buffer and the parser should discard all saved state and
+         restart parsing from the beginning of inData.
+
+         2) Otherwise resetParser=false and the call to RawParseItem
+         depends on the value of parserState:
+
+           2a) parserState = NeedsData: *inData contains the next buffer
+           in sequence
+           2b) parserState = DataReady: inData=NULL
+           2c) parserState = Stopped: RawParseItem will never be called
+
+       RawParseItem must set *outPrefetchCookie before returning.
+
+       If RawParseItem returns NULL, then parserState is set to
+       NeedsData and *outPrefetchCookie is delivered to the buffer
+       reader when the current buffer (the most recent to have been
+       passed to RawParseItem) is returned. For now *outPrefetchCookie
+       should always be NULL. The return value may not be NULL if the
+       current buffer is of type RChannelBuffer_EndOfStream.
+
+       Otherwise *outPrefetchCookie must be NULL and the returned item
+       must be of type RChannelItem_Data, RChannelItem_ItemHole,
+       RChannelItem_BufferHole, RChannelItem_ParseError or
+       RChannelItem_EndOfStream.
+
+       The type of the returned item should be as follows:
+
+         RChannelItem_Data: the item is the next item successfully
+         parsed from the stream.
+
+         RChannelItem_ItemHole: the parser encountered malformed data
+         but believes it has successfully skipped over it and
+         resynchronized on the start of the next well-formed item. The
+         returned item contains metadata describing the error and
+         e.g. the amount of data skipped.
+
+         RChannelItem_BufferHole: the parser has been given a buffer
+         of type RChannelBuffer_Hole. After it has returned any
+         partial data in the preceding buffers as RChannelItem_Data or
+         RChannelItem_ItemHole items, it will return the
+         RChannelItem_BufferHole which is stored in the
+         RChannelBuffer_Hole object. This contains the buffer
+         provider's explanation for the hole.
+
+         RChannelItem_EndOfStream: all the data in the stream has been
+         consumed. The returned item may only have type
+         RChannelItem_EndOfStream if the current buffer has type
+         RChannelBuffer_EndOfStream.
+
+         RChannelItem_ParseError: the parser encountered malformed
+         data, in unable to recover, and will not be able to return
+         any more items from the stream. The returned item contains
+         metadata describing the error.
+
+       If the returned item has type RChannelItem_Data,
+       RChannelItem_BufferHole or RChannelItem_ItemHole then
+       parserState is set to DataReady before the next call to
+       RawParseItem, otherwise parserState is set to Stopped.
+    */
+    virtual RChannelItem* RawParseItem(bool restartParser,
+                                       RChannelBuffer* inData,
+                                       RChannelBufferPrefetchInfo**
+                                       outPrefetchCookie /* out */) = 0;
+
+private:
+    UInt32                       m_maxParseBatchSize;
+    UInt32                       m_index;
+    RChannelContextRef           m_context;
+};
+
+typedef DrRef<RChannelItemParserBase> RChannelItemParserRef;
+
+class RChannelRawItemParser : public RChannelItemParserBase
+{
+public:
+    virtual ~RChannelRawItemParser();
+    DRREFCOUNTIMPL
+};
+
+class RChannelItemTransformerBase : public IDrRefCounter
+{
+public:
+    virtual ~RChannelItemTransformerBase();
+
+    /* this is called once before the first use of the other
+       transformer methods. The default implementation does
+       nothing. Initialize can retrieve the parser context, etc. from
+       the base class. */
+    virtual void InitializeTransformer(RChannelItemParserBase* parent,
+                                       DVErrorReporter* errorReporter);
+
+    /* this is called once for each item in the input. It should write
+       zero or more transformed items to writer. The default
+       implementation writes each input directly back to writer. */
+    virtual void TransformItem(RChannelItemRef& inputItem,
+                               SyncItemWriterBase* writer,
+                               DVErrorReporter* errorReporter);
+
+    /* this is called when the input is exhausted. In some cases,
+       input may be broken up into fragments (e.g. a cosmos stream
+       with missing extents). In this case, Flush will be called at
+       the end of each fragment, and will be followed by another
+       sequence of Map calls to process the next fragment, followed by
+       another Flush, until the end of the input is reached. The
+       default implementation does nothing. */
+    virtual void FlushTransformer(SyncItemWriterBase* writer,
+                                  DVErrorReporter* errorReporter);
+
+    /* this is called before Flush if the input encounters an error,
+       with the item describing the error (e.g. a corrupt extent in a
+       cosmos stream). The default implementation does nothing. */
+    virtual void ReportTransformerErrorItem(RChannelItemRef& errorItem,
+                                            DVErrorReporter* errorReporter);
+};
+
+typedef DrRef<RChannelItemTransformerBase> RChannelItemTransformerRef;
+
+class RChannelItemTransformer : public RChannelItemTransformerBase
+{
+public:
+    virtual ~RChannelItemTransformer();
+
+    DRREFCOUNTIMPL
+};
+
+
+/* this record contains a single buffer. */
+class RChannelBufferRecord
+{
+public:
+    void SetData(DrMemoryBuffer* data);
+    DrMemoryBuffer* GetData() const;
+
+private:
+    DrRef<DrMemoryBuffer>  m_buffer;
+};
+
+/* this item contains a single buffer, however it is also a record
+   array that contains a single record holding the same buffer. That
+   way it can be used in both item-reading and record-reading
+   interfaces. */
+class RChannelBufferItem : public PackedRecordArray<RChannelBufferRecord>
+{
+public:
+    static RChannelBufferItem* Create(DrMemoryBuffer* buffer);
+    DrMemoryBuffer* GetData() const;
+    virtual UInt64 GetItemSize() const;
+
+private:
+    RChannelBufferItem(DrMemoryBuffer* buffer);
+};
+
+class RChannelTransformerParserBase :
+    public RChannelItemParserBase,
+    public DVErrorReporter,
+    public SyncItemWriterBase
+{
+public:
+    RChannelTransformerParserBase();
+    virtual ~RChannelTransformerParserBase();
+
+    void SetTransformer(RChannelItemTransformerBase* transformer);
+    RChannelItemTransformerBase* GetTransformer();
+
+    /* the implementation of the base parser interface */
+    RChannelItem* RawParseItem(bool restartParser,
+                               RChannelBuffer* inData,
+                               RChannelBufferPrefetchInfo**
+                               outPrefetchCookie /* out */);
+
+    /* the implementation of the item writer interface */
+    void WriteItemSyncConsumingReference(RChannelItemRef& item);
+    DrError GetWriterStatus();
+
+private:
+    RChannelItemTransformerRef        m_transformer;
+    bool                              m_transformedAny;
+    RChannelItemList                  m_itemList;
+};
+
+class RChannelTransformerParser : public RChannelTransformerParserBase
+{
+    DRREFCOUNTIMPL
+};
+
+class RChannelItemParserNoRefImpl : public RChannelItemParserBase
+{
+public:
+    RChannelItemParserNoRefImpl();
+    virtual ~RChannelItemParserNoRefImpl();
+
+    /* ResetParser is called whenever the underlying channel restarts,
+       and before the first buffer is passed to the parser. On
+       receiving a call to ResetParser the parser should discard all
+       internal state and return to the start-of-stream ready
+       condition. */
+    virtual void ResetParser();
+
+    /* ParseNextItem is called whenever another item is needed, and
+       the item should be parsed from the buffers in bufferList,
+       starting at startOffset in the first buffer in the list. The
+       list will never be empty. It may be convenient to wrap the
+       bufferList in a DrMemoryBuffer using the RChannelReaderBuffer
+       class in channelmemorybuffers.h, e.g.
+         wrapper = new RChannelReaderBuffer(bufferList,
+                                            startOffset,
+                                            bufferList->back()->GetData()->
+                                            GetAvailableSize());
+
+       If there is not enough data to parse the next item,
+       ParseNextItem should return NULL. On the next call the parser
+       is guaranteed to receive a longer list of buffers starting from
+       the same point in the stream (with bufferList as a prefix)
+       unless ResetParser or ParsePartialItem has been called, so it
+       can cache information about partially parsed items if desired.
+
+       If an item is successfully parsed from availableData it should
+       be returned with type RChannelItem_Data, and the number of
+       bytes consumed should be stored in *pOutLength which must be <=
+       the total available size of the buffers in bufferList -
+       startOffset. The parser is allowed to behave as if the
+       application now "owns" this prefix of the available data,
+       e.g. it can increase the refcounts of buffers in bufferList,
+       store them inside the returned item, and modify the data within
+       this prefix.
+
+       If a parse error occurs but the parser can resynchronize at the
+       start of the next item, ParseNextItem should return an item of
+       type RChannelItem_ItemHole describing the error and set
+       *pOutLength to the number of bytes to be skipped before the
+       start of the next item.
+
+       If a parse error occurs and the parser cannot determine the
+       start of the next item, ParseNextItem should return an item of
+       type RChannelItem_ParseError describing the error. In this case
+       ResetParser will be called before the next call to
+       ParseNextItem or ParsePartialItem.
+    */
+    virtual RChannelItem* ParseNextItem(ChannelDataBufferList* bufferList,
+                                        Size_t startOffset,
+                                        Size_t* pOutLength) = 0;
+
+    /* ParsePartialItem is called whenever a buffer of type
+       RChannelBuffer_Hole or RChannelBuffer_EndOfStream arrives on
+       the channel and there is unparsed data remaining before this
+       marker buffer. The remaining data is passed as bufferList and
+       the buffer is passed as markerBuffer. bufferList may be empty.
+
+       If there is no useful data remaining in bufferList,
+       ParsePartialItem should return NULL. If the data in bufferList
+       can be interpreted as a valid item, this item should be
+       returned as an item of type RChannelItem_Data. If the data is
+       malformed, ParsePartialItem should return an item of type
+       RChannelItem_ItemHole describing the malformed data.
+
+       After a call to ParsePartialItem, the parser should probably
+       not be holding on to any state. If the marker was a buffer
+       hole, the next call to ParseNextItem will be called with
+       bufferList starting after the hole in the stream. If the
+       marker was an end of stream buffer, ResetParser will be called
+       before the next call to ParseNextItem or ParsePartialItem.
+    */
+    virtual RChannelItem* ParsePartialItem(ChannelDataBufferList* bufferList,
+                                           Size_t startOffset,
+                                           RChannelBufferMarker*
+                                           markerBuffer);
+
+
+
+    /* this implements the RChannelRawItemParser interface */
+    RChannelItem* RawParseItem(bool restartParser,
+                               RChannelBuffer* inData,
+                               RChannelBufferPrefetchInfo** outPrefetchCookie);
+
+protected:
+    void ResetParserInternal();
+    void DiscardBufferPrefix(Size_t discardLength);
+    RChannelItem* DealWithPartialBuffer(RChannelBufferMarker* mBuffer);
+
+private:
+    bool                    m_needsReset;
+    bool                    m_needsData;
+    RChannelItemRef         m_savedItem;
+
+    ChannelDataBufferList   m_bufferList;
+
+    Size_t                  m_bufferListStartOffset;
+};
+
+class RChannelItemParser : public RChannelItemParserNoRefImpl
+{
+public:
+    virtual ~RChannelItemParser();
+    DRREFCOUNTIMPL
+};
+
+
+class RChannelStdItemParserNoRefImpl : public RChannelItemParserNoRefImpl
+{
+public:
+    RChannelStdItemParserNoRefImpl(DObjFactoryBase* factory);
+    virtual ~RChannelStdItemParserNoRefImpl();
+
+    void ResetParser();
+
+    RChannelItem* ParseNextItem(ChannelDataBufferList* bufferList,
+                                Size_t startOffset,
+                                Size_t* pOutLength);
+    RChannelItem* ParsePartialItem(ChannelDataBufferList* bufferList,
+                                   Size_t startOffset,
+                                   RChannelBufferMarker*
+                                   markerBuffer);
+
+private:
+    DObjFactoryBase*    m_factory;
+    RChannelItemRef     m_pendingErrorItem;
+};
+
+class RChannelStdItemParser : public RChannelStdItemParserNoRefImpl
+{
+public:
+    RChannelStdItemParser(DObjFactoryBase* factory);
+    virtual ~RChannelStdItemParser();
+    DRREFCOUNTIMPL
+};
+
+
+/* many items are stored with an explicit length available near the
+   start of the item. The RChannelParserExplicitLength convenience
+   class is the simplest way to write a parser to deal with such
+   items.
+*/
+class RChannelLengthDelimitedItemParserNoRefImpl :
+    public RChannelItemParserNoRefImpl
+{
+public:
+    enum LengthStatus {
+        LS_ParseError,
+        LS_NeedsData,
+        LS_Ok
+    };
+
+    RChannelLengthDelimitedItemParserNoRefImpl();
+    virtual ~RChannelLengthDelimitedItemParserNoRefImpl();
+
+    /* GetNextItemLength should attempt to read the length of the next
+       item from availableData, assuming the next item starts at the
+       beginning of availableData. availableData is not growable.
+
+       If buffer does not contain enough data to read out the next
+       item length, GetNextItemLength should return LS_NeedsData and
+       *pErrorItem should be set to NULL.
+
+       If the prefix of buffer is malformed (does not contain a valid
+       prefix of an item), GetNextItemLength should return
+       LS_ParseError and *pErrorItem should contain an item of type
+       RChannelItem_ParseError describing the error.
+
+       Otherwise, GetNextItemLength should read the item length from
+       buffer, store it in *pOutLength, return LS_Ok and set
+       *pErrorItem to NULL.
+    */
+    virtual LengthStatus GetNextItemLength(DrMemoryBuffer* availableData,
+                                           Size_t* pOutLength,
+                                           RChannelItem** pErrorItem) = 0;
+
+    /* ParseItemWithLength is called after a successful call to
+       GetNextItemLength, and after cached data of the required length
+       has been accumulated from the channel. itemData is not growable
+       and has its AllocatedSize and AvailableSize set to itemLength,
+       which is the length returned by the previous call to
+       GetNextItemLength.
+
+       ParseItemWithLength should parse itemData into an RChannelItem
+       object of type RChannelItem_Data and return it. The parser is
+       allowed to behave as if the application now "owns" itemData,
+       e.g. it can increase itemData's refcount and store it inside
+       the returned item, or modify the data within itemData.
+
+       If a parse error occurs, ParseItemWithLength should return an
+       item of type RChannelItem_ItemHole describing the error.
+
+       The RChannelLengthDelimitedItemParser class will automatically
+       add entries to the returned item's metadata indicating what
+       range of the underlying channel the item was generated from.
+     */
+    virtual RChannelItem* ParseItemWithLength(DrMemoryBuffer* itemData,
+                                              Size_t itemLength) = 0;
+
+    /* these implement the RChannelItemParser interface */
+    void ResetParser();
+    RChannelItem* ParseNextItem(ChannelDataBufferList* bufferList,
+                                Size_t startOffset,
+                                Size_t* pOutLength);
+    RChannelItem* ParsePartialItem(ChannelDataBufferList* bufferList,
+                                   Size_t startOffset,
+                                   RChannelBufferMarker*
+                                   markerBuffer);
+
+private:
+    void AddMetaData(RChannelItem* item,
+                     ChannelDataBufferList* bufferList,
+                     Size_t startOffset,
+                     Size_t endOffset);
+    RChannelItem* FetchItem(ChannelDataBufferList* bufferList,
+                            Size_t startOffset,
+                            Size_t tailBufferSize);
+    RChannelItem* MaybeFetchItem(ChannelDataBufferList* bufferList,
+                                 Size_t startOffset,
+                                 Size_t* pOutLength);
+
+    Size_t  m_itemLength;
+    Size_t  m_accumulatedLength;
+};
+
+class RChannelLengthDelimitedItemParser :
+    public RChannelLengthDelimitedItemParserNoRefImpl
+{
+public:
+    virtual ~RChannelLengthDelimitedItemParser();
+    DRREFCOUNTIMPL
+};
+
+class DryadParserFactoryBase : public IDrRefCounter
+{
+public:
+    virtual ~DryadParserFactoryBase();
+    virtual void MakeParser(RChannelItemParserRef* pParser,
+                            DVErrorReporter* errorReporter) = 0;
+};
+
+
+typedef DrRef<DryadParserFactoryBase> DryadParserFactoryRef;
+
+class DryadParserFactory : public DryadParserFactoryBase
+{
+public:
+    virtual ~DryadParserFactory();
+    DRREFCOUNTIMPL
+};
+
+template<class _T> class StdParserFactory : public DryadParserFactory
+{
+public:
+    typedef _T Parser;
+    void MakeParser(RChannelItemParserRef* pParser,
+                    DVErrorReporter* errorReporter)
+    {
+        pParser->Attach(new Parser());
+    }
+};
+

DryadVertex/VertexHost/system/channel/include/concreterchannel.h

@@ -0,0 +1,114 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include <errorreporter.h>
+#include <channelinterface.h>
+#include <channelparser.h>
+#include <channelmarshaler.h>
+
+class WorkQueue;
+
+class RChannelOpenThrottler;
+
+class ConcreteRChannel
+{
+public:
+    static bool IsNTFSFile(const char* uri);
+//JC    static bool IsXComputeFile(const char* uri);
+//JC    static bool IsDryadStream(const char* uri);
+//JC    static bool IsDryadPipe(const char* uri);
+    static bool IsDscStream(const char* uri);
+    static bool IsDscPartition(const char* uri);
+    static bool IsHdfsFile(const char* uri);
+    static bool IsHdfsPartition(const char* uri);
+    static bool IsAzureBlob(const char* uri);
+    static bool IsUncPath(const char* uri);
+    static bool IsFifo(const char* uri);
+    static bool IsNull(const char* uri);
+    static bool IsNamedPipe(const char* uri);
+    static bool IsTidyFSStream(const char* uri);
+    static const UInt32 s_infiniteFifoBuffer = (UInt32) -1;
+};
+
+class RChannelReaderHolder : public DrRefCounter
+{
+public:
+    virtual ~RChannelReaderHolder();
+
+    virtual RChannelReader* GetReader() = 0;
+    virtual void FillInStatus(DryadInputChannelDescription* status) = 0;
+    virtual void Close() = 0;
+};
+
+typedef DrRef<RChannelReaderHolder> RChannelReaderHolderRef;
+
+class RChannelWriterHolder : public DrRefCounter
+{
+public:
+    virtual ~RChannelWriterHolder();
+
+    virtual RChannelWriter* GetWriter() = 0;
+    virtual void FillInStatus(DryadOutputChannelDescription* status) = 0;
+    virtual void Close() = 0;
+};
+
+typedef DrRef<RChannelWriterHolder> RChannelWriterHolderRef;
+
+class RChannelFactory
+{
+public:
+    static DrError OpenReader(const char* channelURI,
+                              DryadMetaData* metaData,
+                              RChannelItemParserBase* parser,
+                              UInt32 numberOfReaders,
+                              RChannelOpenThrottler* openThrottler,
+                              UInt32 maxParseBatchSize,
+                              UInt32 maxParseUnitsInFlight,
+                              WorkQueue* workQueue,
+                              DVErrorReporter* errorReporter,
+                              RChannelReaderHolderRef* pHolder /* out */,
+                              LPDWORD localInputChannels);
+
+    static DrError OpenWriter(const char* channelURI,
+                              DryadMetaData* metaData,
+                              RChannelItemMarshalerBase* marshaler,
+                              UInt32 numberOfWriters,
+                              RChannelOpenThrottler* openThrottler,
+                              UInt32 maxMarshalBatchSize,
+                              WorkQueue* workQueue,
+                              DVErrorReporter* errorReporter,
+                              RChannelWriterHolderRef* pHolder /* out */);
+
+    /* this returns an ID that is guaranteed different to any other
+       return value from this call in this process, and can be used
+       when minting internal fifo names to ensure that two vertices
+       don't accidentally use the same name */
+    static UInt32 GetUniqueFifoId();
+
+    /* this returns a throttler object that can be used to set the
+       maximum number of concurrently open readers and writers from a
+       set. When the throttler is no longer needed it should be passed
+       back to DiscardOpenThrottler. */
+    static RChannelOpenThrottler* MakeOpenThrottler(UInt32 maxOpens,
+                                                    WorkQueue* workQueue);
+    static void DiscardOpenThrottler(RChannelOpenThrottler* throttler);
+};

DryadVertex/VertexHost/system/channel/include/recordarray.h

@@ -0,0 +1,485 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "channelparser.h"
+#include "channelmarshaler.h"
+#include "channelinterface.h"
+#include <dryaderrordef.h>
+
+class RecordArrayBase : public RChannelDataItem
+{
+public:
+    RecordArrayBase();
+    virtual ~RecordArrayBase();
+
+    /* from the RChannelItem interface */
+    UInt64 GetNumberOfSubItems() const;
+    void TruncateSubItems(UInt64 numberOfSubItems);
+    virtual UInt64 GetItemSize() const;
+
+    virtual size_t GetRecordSize() const = 0;
+
+    UInt32 GetNumberOfRecords() const;
+    virtual void SetNumberOfRecords(UInt32 numberOfRecords);
+
+    void* GetRecordArrayUntyped();
+    void* GetRecordUntyped(UInt32 index);
+    void* NextRecordUntyped();
+
+    UInt32 GetRecordIndex() const;
+    void SetRecordIndex(UInt32 index);
+
+    bool AtEnd();
+    void Truncate();
+    void ResetRecordPointer();
+    void PopRecord();
+
+    virtual DrError DeSerialize(DrResettableMemoryReader* reader,
+                                Size_t availableSize);
+    virtual DrError Serialize(ChannelMemoryBufferWriter* writer);
+
+    void TransferRecord(void* dstRecord, void* srcRecord);
+    virtual void TransferRecord(RecordArrayBase* dstArray, void* dstRecord,
+                                RecordArrayBase* srcArray, void* srcRecord) = 0;
+
+    void StartSerializing();
+
+protected:
+    void FreeStorage();
+    void* TransferTruncatedArray(void* srcArray, UInt32 numberOfRecords);
+    virtual UInt32 ReadArray(DrMemoryBuffer* buffer,
+                             Size_t startOffset, UInt32 nRecords);
+    virtual UInt32 AttachArray(DryadLockedMemoryBuffer* buffer,
+                               Size_t startOffset);
+    virtual DrError ReadFinalArray(DrMemoryBuffer* buffer);
+
+    virtual void* MakeTypedArray(UInt32 numberOfRecords) = 0;
+    virtual void FreeTypedArray(void* untypedArray) = 0;
+
+    UInt32                       m_numberOfRecords;
+    UInt32                       m_nextRecord;
+    UInt32                       m_recordArraySize;
+    void*                        m_recordArray;
+    DrRef<DrMemoryBuffer>  m_buffer;
+    bool                         m_serializedAny;
+
+    friend class PackedRecordArrayParserBase;
+    friend class AlternativeRecordMarshalerBase;
+};
+
+template< class _R > class PackedRecordArray : public RecordArrayBase
+{
+public:
+    typedef _R RecordType;
+
+    virtual ~PackedRecordArray()
+    {
+        /* call this from here rather than the base class destructor
+           since it needs to call FreeTypedArray() which is
+           inaccessible from the base destructor */
+        FreeStorage();
+    }
+
+    size_t GetRecordSize() const
+    {
+        return sizeof(RecordType);
+    }
+
+    RecordType* GetRecordArray() const
+    {
+        return (RecordType *) m_recordArray;
+    }
+
+    RecordType* NextRecord()
+    {
+        return (RecordType *) NextRecordUntyped();
+    }
+
+    virtual void TransferRecord(RecordArrayBase* dstArray, void* dst,
+                                RecordArrayBase* srcArray, void* src)
+    {
+        RecordType* dstRecord = (RecordType *) dst;
+        RecordType* srcRecord = (RecordType *) src;
+        *dstRecord = *srcRecord;
+    }
+
+private:
+    void* MakeTypedArray(UInt32 numberOfRecords)
+    {
+        return new RecordType[numberOfRecords];
+    }
+
+    void FreeTypedArray(void* untypedArray)
+    {
+        RecordType* array = (RecordType *) untypedArray;
+        delete [] array;
+    }
+};
+
+template< class _R > class RecordArray : public PackedRecordArray<_R>
+{
+public:
+    virtual DrError DeSerialize(DrResettableMemoryReader* reader,
+                                Size_t availableSize)
+    {
+        if (GetNumberOfRecords() == 0)
+        {
+            SetNumberOfRecords(RChannelItem::s_defaultRecordBatchSize);
+        }
+
+        Size_t sizeUsed = 0;
+        Size_t remainingSize = availableSize;
+        DrError err = DrError_OK;
+
+        ResetRecordPointer();
+        RecordType* nextRecord;
+        while (remainingSize > 0 && (nextRecord = NextRecord()) != NULL)
+        {
+            err = nextRecord->DeSerialize(reader, remainingSize, false);
+            if (err != DrError_OK)
+            {
+                PopRecord();
+                reader->ResetToBufferOffset(sizeUsed);
+                break;
+            }
+
+            sizeUsed = reader->GetBufferOffset();
+            LogAssert(sizeUsed <= availableSize);
+            remainingSize = availableSize - sizeUsed;
+        }
+
+        Truncate();
+        ResetRecordPointer();
+
+        if (err == DrError_EndOfStream && AtEnd() == false)
+        {
+            /* AtEnd() == false after Truncate+Reset means there's at
+               least one item */
+            err = DrError_OK;
+        }
+
+        return err;
+    }
+
+    virtual DrError DeSerializePartial(DrResettableMemoryReader* reader,
+                                       Size_t availableSize)
+    {
+        SetNumberOfRecords(1);
+
+        ResetRecordPointer();
+        RecordType* nextRecord = NextRecord();
+        LogAssert(nextRecord != NULL);
+
+        DrError err = nextRecord->DeSerialize(reader, availableSize, true);
+
+        ResetRecordPointer();
+
+        return err;
+    }
+
+    virtual DrError Serialize(ChannelMemoryBufferWriter* writer)
+    {
+        StartSerializing();
+
+        DrError err = DrError_OK;
+        Size_t currentPosition = 0;
+
+        RecordType* nextRecord;
+        bool filledBuffer = writer->MarkRecordBoundary();
+        LogAssert(filledBuffer == false);
+        while (err == DrError_OK &&
+               filledBuffer == false &&
+               (nextRecord = NextRecord()) != NULL)
+        {
+            currentPosition = writer->GetBufferOffset();
+            err = nextRecord->Serialize(writer);
+            if (err == DrError_OK)
+            {
+                filledBuffer = writer->MarkRecordBoundary();
+            }
+        }
+
+        if (err != DrError_OK)
+        {
+            LogAssert(writer->GetStatus() == DrError_OK);
+            writer->SetBufferOffset(currentPosition);
+            return DryadError_ItemMarshalError;
+        }
+
+        if (filledBuffer)
+        {
+            return DrError_IncompleteOperation;
+        }
+
+        return DrError_OK;
+    }
+
+    virtual void TransferRecord(RecordArrayBase* dstArray, void* dst,
+                                RecordArrayBase* srcArray, void* src)
+    {
+        RecordType* dstRecord = (RecordType *) dst;
+        RecordType* srcRecord = (RecordType *) src;
+        dstRecord->TransferFrom(*srcRecord);
+    }
+
+private:
+    virtual UInt32 ReadArray(DrMemoryBuffer* buffer,
+                             Size_t startOffset, UInt32 nRecords)
+    {
+        LogAssert(false);
+        return 0;
+    }
+
+    virtual UInt32 AttachArray(DryadLockedMemoryBuffer* buffer,
+                               Size_t startOffset)
+    {
+        LogAssert(false);
+        return 0;
+    }
+};
+
+template< class _R > class RecordArrayEx : public RecordArray<_R>
+{
+public:
+    virtual DrError DeSerialize(DrResettableMemoryReader* reader,
+                                Size_t availableSize)
+    {
+        
+        if (GetNumberOfRecords() == 0)
+        {
+            SetNumberOfRecords(RChannelItem::s_defaultRecordBatchSize);
+        }
+
+        Size_t sizeUsed = 0;
+        Size_t remainingSize = availableSize;
+        DrError err = DrError_OK;
+
+        ResetRecordPointer();
+        RecordType* nextRecord;
+        while (remainingSize > 0 && (nextRecord = NextRecord()) != NULL)
+        {
+            err = nextRecord->DeSerialize(reader, remainingSize, false);
+            if (err != DrError_OK)
+            {
+                PopRecord();
+                reader->ResetToBufferOffset(sizeUsed);
+                break;
+            }
+
+            sizeUsed = reader->GetBufferOffset();
+            LogAssert(sizeUsed <= availableSize);
+            remainingSize = availableSize - sizeUsed;
+        }
+
+        Truncate();
+        ResetRecordPointer();
+
+        if (err == DrError_EndOfStream && AtEnd() == false)
+        {
+            /* AtEnd() == false after Truncate+Reset means there's at
+               least one item */
+            err = DrError_OK;
+        }
+
+        return err;
+    }
+
+    virtual DrError DeSerializePartial(DrResettableMemoryReader* reader,
+                                       Size_t availableSize)
+    {
+        SetNumberOfRecords(1);
+
+        ResetRecordPointer();
+        RecordType* nextRecord = NextRecord();
+        LogAssert(nextRecord != NULL);
+
+        DrError err = nextRecord->DeSerialize(reader, availableSize, true);
+
+        ResetRecordPointer();
+
+        return err;
+    }
+};
+
+template< class _A > class RecordArrayFactory : public DObjFactoryBase
+{
+public:
+    typedef _A ArrayType;
+
+    RecordArrayFactory(UInt32 arraySize)
+    {
+        m_arraySize = arraySize;
+    }
+
+private:
+    void* AllocateObjectUntyped()
+    {
+        ArrayType* newArray = new ArrayType();
+        newArray->SetNumberOfRecords(m_arraySize);
+        return newArray;
+    }
+
+    void FreeObjectUntyped(void* object)
+    {
+        DrRef<ArrayType> typedObject;
+        typedObject.Attach((ArrayType *) object);
+        /* let typedObject go out of scope, freeing the reference */
+    }
+
+    UInt32  m_arraySize;
+
+    DRREFCOUNTIMPL
+};
+
+class RecordArrayReaderBase
+{
+public:
+    RecordArrayReaderBase();
+    RecordArrayReaderBase(SyncItemReaderBase* reader);
+    virtual ~RecordArrayReaderBase();
+
+    void Initialize(SyncItemReaderBase* reader);
+
+    bool Advance();
+    UInt32 AdvanceBlock(UInt32 validEntriesRequested);
+    UInt32 GetValidCount() const;
+    void PushBack();
+
+    DrError GetStatus();
+    RChannelItem* GetTerminationItem();
+
+protected:
+    void PushBack(bool pushValid);
+    void DiscardCachedItems();
+    virtual bool AdvanceInternal(UInt32 slotNumber);
+
+    void**                 m_currentRecord;
+    SyncItemReaderBase*    m_reader;
+    RChannelItemRef        m_item;
+    RecordArrayBase*       m_arrayItem;
+    UInt32                 m_cacheSize;
+    RChannelItemRef*       m_itemCache;
+    UInt32                 m_valid;
+    UInt32                 m_cachedItemCount;
+};
+
+template< class _R > class RecordArrayReader :
+    public RecordArrayReaderBase
+{
+public:
+    typedef _R RecordType;
+
+    RecordArrayReader() {}
+    RecordArrayReader(SyncItemReaderBase* reader) :
+        RecordArrayReaderBase(reader)
+    {
+    }
+
+    RecordType* operator->() const
+    {
+        LogAssert(GetValidCount() > 0);
+        return (RecordType *) m_currentRecord[0];
+    }
+
+    RecordType& operator*() const
+    {
+        LogAssert(GetValidCount() > 0);
+        return *((RecordType *) m_currentRecord[0]);
+    }
+
+    RecordType& operator[](UInt32 index) const
+    {
+        LogAssert(index < GetValidCount());
+        return *((RecordType *) m_currentRecord[index]);
+    }
+};
+
+class RecordArrayWriterBase
+{
+public:
+    RecordArrayWriterBase();
+    RecordArrayWriterBase(SyncItemWriterBase* writer,
+                          DObjFactoryBase* factory);
+    virtual ~RecordArrayWriterBase();
+
+    void Initialize(SyncItemWriterBase* writer, DObjFactoryBase* factory);
+    void SetWriter(SyncItemWriterBase* writer);
+
+    void MakeValid();
+    void MakeValidBlock(UInt32 validEntriesRequested);
+    UInt32 GetValidCount() const;
+    void PushBack();
+
+    void* ReadValidUntyped(UInt32 index);
+
+    void Terminate();
+    void Flush();
+
+    DrError GetWriterStatus();
+
+protected:
+    void**                   m_currentRecord;
+
+private:
+    void Destroy();
+    void SendCachedItems();
+    void AdvanceInternal(UInt32 slotNumber);
+
+    DrRef<DObjFactoryBase>   m_factory;
+    SyncItemWriterBase*      m_writer;
+    DrRef<RecordArrayBase>   m_item;
+    UInt32                   m_cacheSize;
+    DrRef<RecordArrayBase>*  m_itemCache;
+    UInt32                   m_valid;
+    UInt32                   m_pushBackIndex;
+    UInt32                   m_cachedItemCount;
+};
+
+template< class _R > class RecordArrayWriter : public RecordArrayWriterBase
+{
+public:
+    typedef _R RecordType;
+
+    RecordArrayWriter() {}
+    RecordArrayWriter(SyncItemWriterBase* writer, DObjFactoryBase* factory) :
+        RecordArrayWriterBase(writer, factory)
+    {
+    }
+
+    RecordType* operator->() const
+    {
+        LogAssert(GetValidCount() > 0);
+        return (RecordType *) m_currentRecord[0];
+    }
+
+    RecordType& operator*() const
+    {
+        LogAssert(GetValidCount() > 0);
+        return *((RecordType *) m_currentRecord[0]);
+    }
+
+    RecordType& operator[](UInt32 index) const
+    {
+        LogAssert(index < GetValidCount());
+        return *((RecordType *) m_currentRecord[index]);
+    }
+};

DryadVertex/VertexHost/system/channel/include/recorditem.h

@@ -0,0 +1,239 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "recordparser.h"
+
+class RecordBundleInterfaceBase
+{
+public:
+    RecordBundleInterfaceBase()
+    {
+        m_marshalerFactory.Attach(new MarshalerFactory());
+    }
+
+    virtual ~RecordBundleInterfaceBase() {}
+
+    DryadParserFactoryBase* GetParserFactory()
+    {
+        return m_parserFactory;
+    }
+
+    DryadMarshalerFactoryBase* GetMarshalerFactory()
+    {
+        return m_marshalerFactory;
+    }
+
+    DObjFactoryBase* GetRecordFactory()
+    {
+        return m_factory;
+    }
+
+protected:
+    class MarshalerFactory : public DryadMarshalerFactory
+    {
+    public:
+        void MakeMarshaler(RChannelItemMarshalerRef* pMarshaler,
+                           DVErrorReporter* errorReporter)
+        {
+            pMarshaler->Attach(new RChannelStdItemMarshaler());
+        }
+    };
+
+    DrRef<DObjFactoryBase>   m_factory;
+    DryadParserFactoryRef    m_parserFactory;
+    DryadMarshalerFactoryRef m_marshalerFactory;
+};
+
+template< class _R > class PackedRecordBundleBase :
+    public RecordBundleInterfaceBase
+{
+public:
+    typedef _R RecordType;
+
+    typedef PackedRecordArray<RecordType> Array;
+    typedef RecordArrayReader<RecordType> Reader;
+    typedef RecordArrayWriter<RecordType> WriterBase;
+
+    class Writer : public WriterBase
+    {
+    public:
+        Writer() {}
+        Writer(PackedRecordBundleBase<_R>* bundle, SyncItemWriterBase* writer)
+        {
+            Initialize(bundle, writer);
+        }
+
+        void Initialize(PackedRecordBundleBase<_R>* bundle,
+                        SyncItemWriterBase* writer)
+        {
+            bundle->InitializeWriter(this, writer);
+        }
+    };
+
+    PackedRecordBundleBase() {}
+    PackedRecordBundleBase(DObjFactoryBase* factory)
+    {
+        InitializeBase(factory);
+    }
+    virtual ~PackedRecordBundleBase() {}
+
+    void InitializeBase(DObjFactoryBase* factory)
+    {
+        m_factory = factory;
+        m_parserFactory.Attach(new ParserFactory(m_factory));
+    }
+
+    void InitializeWriter(WriterBase* writer,
+                          SyncItemWriterBase* channelWriter)
+    {
+        writer->Initialize(channelWriter, m_factory);
+    }
+
+private:
+    class ParserFactory : public DryadParserFactory
+    {
+    public:
+        ParserFactory(DObjFactoryBase* factory)
+        {
+            m_factory = factory;
+        }
+
+        void MakeParser(RChannelItemParserRef* pParser,
+                        DVErrorReporter* errorReporter)
+        {
+            pParser->Attach(new PackedRecordArrayParser(m_factory));
+        }
+
+        DObjFactoryRef m_factory;
+    };
+};
+
+template< class _R > class PackedRecordBundle :
+    public PackedRecordBundleBase<_R>
+{
+public:
+    typedef RecordArrayFactory<Array> Factory;
+
+    PackedRecordBundle()
+    {
+        Initialize(RChannelItem::s_defaultRecordBatchSize);
+    }
+
+    PackedRecordBundle(UInt32 maxArraySize)
+    {
+        Initialize(maxArraySize);
+    }
+
+    void Initialize(UInt32 maxArraySize)
+    {
+        DrRef<Factory> factory;
+        factory.Attach(new Factory(maxArraySize));
+        InitializeBase(factory);
+    }
+};
+
+template< class _R > class RecordBundleBase :
+    public RecordBundleInterfaceBase
+{
+public:
+    typedef _R RecordType;
+
+    typedef RecordArray<RecordType> Array;
+    typedef RecordArrayReader<RecordType> Reader;
+    typedef RecordArrayWriter<RecordType> WriterBase;
+
+    class Writer : public WriterBase
+    {
+    public:
+        Writer() {}
+        Writer(RecordBundleBase<_R>* bundle, SyncItemWriterBase* writer)
+        {
+            Initialize(bundle, writer);
+        }
+
+        void Initialize(RecordBundleBase<_R>* bundle,
+                        SyncItemWriterBase* writer)
+        {
+            bundle->InitializeWriter(this, writer);
+        }
+    };
+
+    RecordBundleBase() {}
+    RecordBundleBase(DObjFactoryBase* factory)
+    {
+        InitializeBase(factory);
+    }
+    virtual ~RecordBundleBase() {}
+
+    void InitializeBase(DObjFactoryBase* factory)
+    {
+        m_factory = factory;
+        m_parserFactory.Attach(new ParserFactory(m_factory));
+    }
+
+    void InitializeWriter(WriterBase* writer,
+                          SyncItemWriterBase* channelWriter)
+    {
+        writer->Initialize(channelWriter, m_factory);
+    }
+
+private:
+    class ParserFactory : public DryadParserFactory
+    {
+    public:
+        ParserFactory(DObjFactoryBase* factory)
+        {
+            m_factory = factory;
+        }
+
+        void MakeParser(RChannelItemParserRef* pParser,
+                        DVErrorReporter* errorReporter)
+        {
+            pParser->Attach(new RChannelStdItemParser(m_factory));
+        }
+
+        DObjFactoryRef m_factory;
+    };
+};
+
+template< class _R > class RecordBundle : public RecordBundleBase<_R>
+{
+public:
+    typedef RecordArrayFactory<Array> Factory;
+
+    RecordBundle()
+    {
+        Initialize(RChannelItem::s_defaultRecordBatchSize);
+    }
+
+    RecordBundle(UInt32 maxArraySize)
+    {
+        Initialize(maxArraySize);
+    }
+
+    void Initialize(UInt32 maxArraySize)
+    {
+        DrRef<Factory> factory;
+        factory.Attach(new Factory(maxArraySize));
+        InitializeBase(factory);
+    }
+};

DryadVertex/VertexHost/system/channel/include/recordparser.h

@@ -0,0 +1,201 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "channelparser.h"
+
+class PackedRecordArrayParserBase : public RChannelItemParserNoRefImpl
+{
+public:
+    PackedRecordArrayParserBase(DObjFactoryBase* factory);
+    virtual ~PackedRecordArrayParserBase();
+
+    RChannelItem* ParseNextItem(ChannelDataBufferList* bufferList,
+                                Size_t startOffset,
+                                Size_t* pOutLength);
+    RChannelItem* ParsePartialItem(ChannelDataBufferList* bufferList,
+                                   Size_t startOffset,
+                                   RChannelBufferMarker* markerBuffer);
+
+private:
+    DrRef<DObjFactoryBase>    m_factory;
+};
+
+class PackedRecordArrayParser : public PackedRecordArrayParserBase
+{
+public:
+    PackedRecordArrayParser(DObjFactoryBase* factory);
+    virtual ~PackedRecordArrayParser();
+    DRREFCOUNTIMPL
+};
+
+typedef DrError RecordDeSerializerFunction(void* record,
+                                           DrMemoryBufferReader* reader,
+                                           Size_t availableSize,
+                                           bool lastRecordInStream);
+
+class AlternativeRecordParserBase : public RChannelItemParserNoRefImpl
+{
+public:
+    AlternativeRecordParserBase(DObjFactoryBase* factory);
+    AlternativeRecordParserBase(DObjFactoryBase* factory,
+                                RecordDeSerializerFunction* function);
+    virtual ~AlternativeRecordParserBase();
+
+    void ResetParser();
+
+    RChannelItem* ParseNextItem(ChannelDataBufferList* bufferList,
+                                Size_t startOffset,
+                                Size_t* pOutLength);
+    RChannelItem* ParsePartialItem(ChannelDataBufferList* bufferList,
+                                   Size_t startOffset,
+                                   RChannelBufferMarker*
+                                   markerBuffer);
+
+private:
+    DrError DeSerializeArray(RecordArrayBase* array,
+                             DrResettableMemoryReader* reader,
+                             Size_t availableSize);
+    virtual DrError DeSerializeUntyped(void* record,
+                                       DrMemoryBufferReader* reader,
+                                       Size_t availableSize,
+                                       bool lastRecordInStream);
+
+    DObjFactoryBase*              m_factory;
+    RChannelItemRef               m_pendingErrorItem;
+    RecordDeSerializerFunction*   m_function;
+};
+
+typedef DrRef<AlternativeRecordParserBase> AlternativeRecordParserRef;
+
+class UntypedAlternativeRecordParser : public AlternativeRecordParserBase
+{
+public:
+    UntypedAlternativeRecordParser(DObjFactoryBase* factory,
+                                   RecordDeSerializerFunction* function);
+
+    DRREFCOUNTIMPL
+};
+
+class StdAlternativeRecordParserFactory : public DryadParserFactoryBase
+{
+public:
+    StdAlternativeRecordParserFactory(DObjFactoryBase* factory,
+                                      RecordDeSerializerFunction* function);
+
+    void MakeParser(RChannelItemParserRef* pParser,
+                    DVErrorReporter* errorReporter);
+
+private:
+    DObjFactoryBase*              m_factory;
+    RecordDeSerializerFunction*   m_function;
+
+    DRREFCOUNTIMPL
+};
+
+template< class _R > class AlternativeRecordParser :
+    public AlternativeRecordParserBase
+{
+public:
+    typedef _R RecordType;
+
+    AlternativeRecordParser(DObjFactoryBase* factory);
+
+    virtual DrError DeSerialize(RecordType* record,
+                                DrMemoryBufferReader* reader,
+                                Size_t availableSize,
+                                bool lastRecordInStream) = 0;
+
+private:
+    DrError DeSerializeUntyped(void* record,
+                               DrMemoryBufferReader* reader,
+                               Size_t availableSize,
+                               bool lastRecordInStream)
+    {
+        return DeSerialize((RecordType *) record, reader,
+                           availableSize, lastRecordInStream);
+    }
+};
+
+
+typedef DrError RecordSerializerFunction(void* record,
+                                         DrMemoryBufferWriter* writer);
+
+class AlternativeRecordMarshalerBase : public RChannelItemMarshalerBase
+{
+public:
+    AlternativeRecordMarshalerBase();
+    AlternativeRecordMarshalerBase(RecordSerializerFunction* function);
+    virtual ~AlternativeRecordMarshalerBase();
+
+    void SetFunction(RecordSerializerFunction* function);
+
+    DrError MarshalItem(ChannelMemoryBufferWriter* writer,
+                        RChannelItem* item,
+                        bool flush,
+                        RChannelItemRef* pFailureItem);
+
+private:
+    virtual DrError SerializeUntyped(void* record,
+                                     DrMemoryBufferWriter* writer);
+
+    RecordSerializerFunction*   m_function;
+};
+
+typedef DrRef<AlternativeRecordMarshalerBase> AlternativeRecordMarshalerRef;
+
+class UntypedAlternativeRecordMarshaler : public AlternativeRecordMarshalerBase
+{
+public:
+    UntypedAlternativeRecordMarshaler(RecordSerializerFunction* function);
+
+    DRREFCOUNTIMPL
+};
+
+class StdAlternativeRecordMarshalerFactory : public DryadMarshalerFactoryBase
+{
+public:
+    StdAlternativeRecordMarshalerFactory(RecordSerializerFunction* function);
+
+    void MakeMarshaler(RChannelItemMarshalerRef* pMarshaler,
+                       DVErrorReporter* errorReporter);
+
+private:
+    RecordSerializerFunction*   m_function;
+
+    DRREFCOUNTIMPL
+};
+
+template< class _R > class AlternativeRecordMarshaler :
+    public AlternativeRecordMarshalerBase
+{
+public:
+    typedef _R RecordType;
+
+    virtual DrError Serialize(RecordType* record,
+                              DrMemoryBufferWriter* writer) = 0;
+
+private:
+    DrError SerializeUntyped(void* record, DrMemoryBufferWriter* writer)
+    {
+        return Serialize((RecordType *) record, writer);
+    }
+};

DryadVertex/VertexHost/system/channel/src/channelbuffer.cpp

@@ -0,0 +1,196 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#include <DrCommon.h>
+#include <channelbuffer.h>
+
+#pragma unmanaged
+
+
+RChannelBuffer::RChannelBuffer(RChannelBufferType type)
+{
+    m_type = type;
+    DryadMetaData::Create(&m_metaData);
+}
+
+RChannelBuffer::~RChannelBuffer()
+{
+}
+
+DryadMetaData* RChannelBuffer::GetMetaData()
+{
+    return m_metaData.Ptr();
+}
+
+void RChannelBuffer::ReplaceMetaData(DryadMetaData* metaData)
+{
+    m_metaData.Set(metaData);
+}
+
+bool RChannelBuffer::IsTerminationBuffer(RChannelBufferType type)
+{
+    return (type == RChannelBuffer_Restart ||
+            type == RChannelBuffer_Abort ||
+            type == RChannelBuffer_EndOfStream);
+}
+
+RChannelBufferType RChannelBuffer::GetType()
+{
+    return m_type;
+}
+
+RChannelBufferData::RChannelBufferData() : RChannelBuffer(RChannelBuffer_Data)
+{
+}
+
+RChannelBufferData::~RChannelBufferData()
+{
+}
+
+//
+// Return structure that contains item
+//
+RChannelBufferData* ChannelDataBufferList::CastOut(DrBListEntry* item)
+{
+    return DR_GET_CONTAINER(RChannelBufferData, item, m_dataListPtr);
+}
+
+//
+// Return pointer to data
+//
+DrBListEntry* ChannelDataBufferList::CastIn(RChannelBufferData* item)
+{
+    return &(item->m_dataListPtr);
+}
+
+RChannelBufferDataDefault::
+    RChannelBufferDataDefault(DryadLockedMemoryBuffer* dataBuffer,
+                              UInt64 startOffset,
+                              RChannelBufferDefaultHandler* parent)
+{
+    m_dataBuffer = dataBuffer;
+    m_startOffset = startOffset;
+    m_parent = parent;
+}
+
+RChannelBufferDataDefault::~RChannelBufferDataDefault()
+{
+    m_dataBuffer->DecRef();
+}
+
+RChannelBufferDataDefault*
+    RChannelBufferDataDefault::Create(DryadLockedMemoryBuffer* dataBuffer,
+                                      UInt64 startOffset,
+                                      RChannelBufferDefaultHandler* parent)
+{
+    return new RChannelBufferDataDefault(dataBuffer, startOffset,
+                                         parent);
+}
+
+DryadLockedMemoryBuffer* RChannelBufferDataDefault::GetData()
+{
+    return m_dataBuffer;
+}
+
+void RChannelBufferDataDefault::
+    GetOffsetMetaData(bool isStart,
+                      UInt64 offset,
+                      DryadMetaDataRef* dstMetaData)
+{
+    DryadMetaDataRef m;
+    GetMetaData()->Clone(&m);
+
+    DryadMTagRef tag;
+    tag.Attach(DryadMTagUInt64::Create((isStart) ?
+                                       Prop_Dryad_ItemBufferStartOffset :
+                                       Prop_Dryad_ItemBufferEndOffset,
+                                       offset));
+    bool brc = m->Append(tag, false);
+    LogAssert(brc == true);
+
+    UInt64 streamOffset =
+        (m_startOffset == RCHANNEL_BUFFER_OFFSET_UNDEFINED) ?
+        RCHANNEL_BUFFER_OFFSET_UNDEFINED : offset + m_startOffset;
+
+    tag.Attach(DryadMTagUInt64::Create((isStart) ?
+                                       Prop_Dryad_ItemStreamStartOffset :
+                                       Prop_Dryad_ItemStreamEndOffset,
+                                       streamOffset));
+    brc = m->Append(tag, false);
+    LogAssert(brc == true);
+
+    *dstMetaData = m;
+}
+
+//
+// Mark a buffer complete
+//
+void RChannelBufferDataDefault::ProcessingComplete(RChannelBufferPrefetchInfo*
+                                                   /* unused prefetchCookie*/)
+{
+    m_parent->ReturnBuffer(this);
+}
+
+RChannelBufferMarker::RChannelBufferMarker(RChannelBufferType type,
+                                           RChannelItem* item) :
+    RChannelBuffer(type)
+{
+    LogAssert(item != NULL);
+    m_item = item;
+}
+
+RChannelBufferMarker::~RChannelBufferMarker()
+{
+    m_item->DecRef();
+}
+
+RChannelItem* RChannelBufferMarker::GetItem()
+{
+    LogAssert(m_item != NULL);
+    return m_item;
+}
+
+RChannelBufferMarkerDefault::
+    RChannelBufferMarkerDefault(RChannelBufferType type,
+                                RChannelItem* item,
+                                RChannelBufferDefaultHandler* parent) :
+        RChannelBufferMarker(type, item)
+{
+    m_parent = parent;
+}
+
+RChannelBufferMarkerDefault::~RChannelBufferMarkerDefault()
+{
+}
+
+RChannelBufferMarkerDefault*
+    RChannelBufferMarkerDefault::Create(RChannelBufferType type,
+                                        RChannelItem* item,
+                                        RChannelBufferDefaultHandler* parent)
+{
+    return new RChannelBufferMarkerDefault(type, item, parent);
+}
+
+void RChannelBufferMarkerDefault::
+    ProcessingComplete(RChannelBufferPrefetchInfo*
+                       /* unused prefetchCookie*/)
+{
+    m_parent->ReturnBuffer(this);
+}

DryadVertex/VertexHost/system/channel/src/channelbufferhdfs.h

@@ -0,0 +1,157 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "channelreader.h"
+#include "channelwriter.h"
+
+#include <HdfsBridgeNative.h>
+
+class RChannelBufferHdfsReader
+    : public RChannelBufferReader, public RChannelBufferDefaultHandler
+{
+public:
+    static const char* s_hdfsPartitionPrefix;
+
+    RChannelBufferHdfsReader(const char* uri);
+    virtual ~RChannelBufferHdfsReader();
+
+    void Start(RChannelBufferPrefetchInfo* prefetchCookie,
+               RChannelBufferReaderHandler* handler);
+
+    void Interrupt();
+
+    void FillInStatus(DryadChannelDescription* status);
+
+    void Drain(RChannelItem* drainItem);
+
+    void Close();
+
+    bool GetTotalLength(UInt64* pLen);
+
+    /* the RChannelBufferDefaultHandler interface */
+    void ReturnBuffer(RChannelBuffer* buffer);
+
+private:
+    virtual Int64 ScanForSync(HdfsBridgeNative::Reader* reader,
+                              const char* fileName,
+                              Int64 startOffset, Int64 endOffset,
+                              RChannelBuffer** pErrorBuffer) = 0;
+
+    static unsigned __stdcall ThreadFunc(void* a);
+    void SendBuffer(RChannelBuffer* buffer, bool getSemaphore);
+    Int64 AdjustStartOffset(HdfsBridgeNative::Reader* reader,
+                            const char* fileName,
+                            Int64 startOffset,
+                            Int64 endOffset);
+    Int64 AdjustEndOffset(HdfsBridgeNative::Reader* reader,
+                          const char* fileName,
+                          Int64 endOffset);
+    Int64 ReadDataBuffer(HdfsBridgeNative::ReaderAccessor& ra,
+                         const char* fileName,
+                         Int64 offset,
+                         Int64 endOffset);
+    void ReadThread();
+
+    DrStr64                        m_uri;
+    RChannelBufferReaderHandler*   m_handler;
+    HANDLE                         m_readThread;
+    HANDLE                         m_blockSemaphore;
+    HANDLE                         m_abortHandle;
+
+    UInt64                         m_totalLength;
+    UInt64                         m_processedLength;
+    UInt32                         m_buffersOut;
+    CRITSEC                        m_cs;
+};
+
+class RChannelBufferHdfsReaderLineRecord : public RChannelBufferHdfsReader
+{
+public:
+    RChannelBufferHdfsReaderLineRecord(const char* uri);
+
+private:
+    Int64 ScanForSync(HdfsBridgeNative::Reader* reader,
+                      const char* fileName,
+                      Int64 startOffset, Int64 endOffset,
+                      RChannelBuffer** pErrorBuffer);
+};
+
+
+class RChannelBufferHdfsWriter : public RChannelBufferWriter
+{
+public:
+    static const char* s_hdfsFilePrefix;
+
+    RChannelBufferHdfsWriter(const char* uri);
+
+    DryadFixedMemoryBuffer* GetNextWriteBuffer();
+    DryadFixedMemoryBuffer* GetCustomWriteBuffer(Size_t bufferSize);
+
+    void Start();
+
+    bool WriteBuffer(DryadFixedMemoryBuffer* buffer,
+                     bool flushAfter,
+                     RChannelBufferWriterHandler* handler);
+
+    void ReturnUnusedBuffer(DryadFixedMemoryBuffer* buffer);
+
+    void WriteTermination(RChannelItemType reasonCode,
+                          RChannelBufferWriterHandler* handler);
+
+    void FillInStatus(DryadChannelDescription* status);
+
+    void Drain(RChannelItemRef* pReturnItem);
+
+    void Close();
+
+    /* Get/set a hint about the total length the channel is expected
+       to be. Some channel implementations can use this to improve
+       write performance and decrease disk fragmentation. A value of 0
+       (the default) means that the size is unknown. */
+    UInt64 GetInitialSizeHint();
+    void SetInitialSizeHint(UInt64 hint);
+
+private:
+    struct WriteEntry
+    {
+        DrRef<DryadFixedMemoryBuffer>  m_buffer;
+        bool                           m_flush;
+        RChannelItemType               m_type;
+        RChannelBufferWriterHandler*   m_handler;
+        DrBListEntry                   m_listPtr;
+    };
+
+    static unsigned __stdcall ThreadFunc(void* arg);
+    void WriteThread();
+    bool Open(HdfsBridgeNative::Instance** pInstance,
+              HdfsBridgeNative::Writer** pWriter);
+    bool AddToQueue(WriteEntry* entry);
+
+    DrStr64                        m_uri;
+    DryadBList<WriteEntry>         m_queue;
+    UInt32                         m_queueLength;
+    HANDLE                         m_queueHandle;
+    HANDLE                         m_writeThread;
+    RChannelItemRef                m_completionItem;
+    UInt64                         m_processedLength;
+    CRITSEC                        m_cs;
+};

DryadVertex/VertexHost/system/channel/src/channelbuffernativereader.h

@@ -0,0 +1,241 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "dryadnativeport.h"
+#include "channelreader.h"
+#include "concreterchannelhelpers.h"
+#include <dvertexcommand.h>
+#include <XCompute.h>
+
+#pragma warning(disable:4995)
+#include <map>
+
+class RChannelBufferReaderNative :
+    public RChannelBufferReader, public RChannelBufferDefaultHandler,
+    public RChannelThrottledStream
+{
+public:
+    class ReadHandler : public DryadNativePort::Handler
+    {
+    public:
+        ReadHandler(UInt64 requestOffset, UInt64 streamOffset,
+                    UInt32 bufferSize,
+                    size_t bufferAlignment);
+        virtual ~ReadHandler();
+
+        UInt64 GetStreamOffset();
+        void* GetData();
+        DryadAlignedReadBlock* GetBlock();
+        RChannelBuffer* TransferChannelBuffer();
+        bool IsLastDataBuffer();
+
+        virtual void QueueRead(DryadNativePort* port) = 0;
+
+    protected:
+        void SetChannelBuffer(RChannelBuffer* buffer);
+        void SignalLastDataBuffer();
+
+    private:
+        UInt64                       m_streamOffset;
+        DryadAlignedReadBlock*       m_block;
+        RChannelBuffer*              m_buffer;
+        bool                         m_isLastDataBuffer;
+        DrBListEntry                 m_listPtr;
+        friend class DryadBList<ReadHandler>;
+    };
+
+    RChannelBufferReaderNative(UInt32 prefetchBuffers,
+                               DryadNativePort* port,
+                               WorkQueue* workQueue,
+                               RChannelOpenThrottler* openThrottler,
+                               bool supportsLazyOpen);
+    virtual ~RChannelBufferReaderNative();
+
+    void Interrupt();
+
+    void ReturnBuffer(RChannelBuffer* buffer);
+
+    virtual void FillInStatus(DryadChannelDescription* status);
+
+    bool EnsureOpenForRead(ReadHandler* handler);
+
+    void OpenAfterThrottle();
+
+protected:
+    void StartNativeReader(RChannelBufferReaderHandler* handler);
+    void DrainNativeReader();
+
+    void SetPrefetchBufferCount(UInt32 numberOfBuffers);
+    UInt32 GetPrefetchBufferCount();
+
+    void AssociateHandleWithPort(HANDLE h);
+    void SetTotalLength(UInt64 totalLength);
+    bool GetTotalLength(UInt64* pLen);
+
+    /* called with baseDR held */
+    void OpenNativeReader();
+    /* called with baseDR held */
+    RChannelBuffer* CloseNativeReader();
+
+    void SetErrorBuffer(RChannelBuffer* buffer);
+
+    void DispatchBuffer(ReadHandler* handler, bool makeNewHandler,
+                        ReadHandler* fillInReadHandler);
+
+    RChannelBuffer* MakeEndOfStreamBuffer(UInt64 streamOffset);
+    RChannelBuffer* MakeErrorBuffer(UInt64 streamOffset,
+                                    DrError errorCode);
+    RChannelBuffer* MakeOpenErrorBuffer(DrError errorCode,
+                                        const char* description);
+    RChannelBuffer* MakeDataBuffer(UInt64 streamOffset,
+                                   DryadLockedMemoryBuffer* block);
+    virtual void MakeFileMetaData(DryadMetaData* metaData,
+                                  UInt64 streamOffset) = 0;
+    virtual bool LazyOpenFile();
+    virtual void FillInOpenedDetails(ReadHandler* handler);
+    virtual void EagerCloseFile();
+
+    CRITSEC* GetBaseDR();
+
+private:
+    enum State {
+        S_Closed,
+        S_Stopped,
+        S_Running,
+        S_Stopping
+    };
+
+    enum OpenState {
+        OS_Closed,
+        OS_NotOpened,
+        OS_Waiting,
+        OS_Opened,
+        OS_OpenError,
+        OS_Stopped
+    };
+
+    typedef DryadBList<ReadHandler> HandlerList;
+    typedef std::map<UInt64, ReadHandler *> OffsetHandlerMap;
+
+    bool FinishUsingFile();
+    virtual ReadHandler* GetNextReadHandler(bool lazyOpenDone) = 0;
+
+
+    UInt32                                   m_prefetchBuffers;
+
+    RChannelBufferReaderHandler*             m_handler;
+    DryadNativePort*                         m_port;
+    WorkQueue*                               m_workQueue;
+    RChannelOpenThrottler*                   m_openThrottler;
+    bool                                     m_supportsLazyOpen;
+
+    State                                    m_state;
+    OpenState                                m_openState;
+    bool                                     m_fetching;
+    bool                                     m_sentLastBuffer;
+    UInt32                                   m_outstandingHandlers;
+    UInt32                                   m_outstandingBuffers;
+    UInt64                                   m_nextStreamOffsetToProcess;
+    UInt64                                   m_totalLength;
+    OffsetHandlerMap                         m_reorderMap;
+    DrRef<RChannelBuffer>                    m_errorBuffer;
+    HANDLE                                   m_handlerReturnEvent;
+    HANDLE                                   m_bufferReturnEvent;
+    HandlerList                              m_openWaitingList;
+    bool                                     m_drainingOpenQueue;
+
+    DryadOrderedSendLatch<ChannelBufferList>  m_latch;
+
+protected:
+    CRITSEC                                  m_baseDR;
+
+    friend class ReadHandler;
+};
+
+class RChannelBufferReaderNativeFile : public RChannelBufferReaderNative
+{
+public:
+    class FileReadHandler : public RChannelBufferReaderNative::ReadHandler
+    {
+    public:
+        FileReadHandler(HANDLE fileHandle,
+                        bool detailsPresent,
+                        UInt64 streamOffset,
+                        UInt32 dataSize, size_t dataAlignment,
+                        RChannelBufferReaderNativeFile* parent);
+
+        void ProcessIO(DrError cse, UInt32 numBytes);
+        void SetFileHandle(HANDLE h);
+        HANDLE GetFileHandle();
+        void QueueRead(DryadNativePort* port);
+
+    private:
+        HANDLE                             m_fileHandle;
+        bool                               m_detailsPresent;
+        RChannelBufferReaderNativeFile*    m_parent;
+    };
+
+    RChannelBufferReaderNativeFile(UInt32 bufferSize,
+                                   size_t bufferAlignment,
+                                   UInt32 prefetchBuffers,
+                                   DryadNativePort* port,
+                                   WorkQueue* workQueue,
+                                   RChannelOpenThrottler* openThrottler);
+    ~RChannelBufferReaderNativeFile();
+
+    virtual void FillInStatus(DryadChannelDescription* status);
+
+    bool OpenA(const char* pathName);
+//JC    bool OpenW(const wchar_t* pathName);
+
+    void Start(RChannelBufferPrefetchInfo* prefetchCookie,
+               RChannelBufferReaderHandler* handler);
+
+    void Drain(RChannelItem* drainItem);
+
+    void Close();
+
+protected:
+    void EagerCloseFile();
+    bool LazyOpenFile();
+    void ResetFileNameAndErrorStateA(const char *fileName);
+
+private:
+    void MakeFileMetaData(DryadMetaData* metaData, UInt64 streamOffset);
+    RChannelBuffer* MakeFinalBuffer(UInt64 streamOffset);
+    void FillInOpenedDetails(ReadHandler* handler);
+
+    ReadHandler* GetNextReadHandler(bool lazyOpenDone);
+
+    UInt32                       m_bufferSize;
+    size_t                       m_bufferAlignment;
+    UInt64                       m_nextOffsetToRequest;
+    HANDLE                       m_fileHandle;
+    char*                        m_fileNameA;
+    size_t                       m_fileNameLength;
+    bool                         m_fileIsPipe;
+    wchar_t*                     m_fileNameW;
+    bool                         m_wideFileName;
+    DrRef<RChannelBuffer>        m_openErrorBuffer;
+
+    friend class FileReadHandler;
+};

DryadVertex/VertexHost/system/channel/src/channelbuffernativewriter.h

@@ -0,0 +1,336 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "dryadnativeport.h"
+#include "channelwriter.h"
+#include "concreterchannelhelpers.h"
+#ifdef TIDYFS
+#include <mdclient.h>
+#endif
+#include "DrFPrint.h"
+
+#pragma warning(disable:4995)
+#include <set>
+
+class RChannelBufferWriterNative :
+    public RChannelBufferWriter, public RChannelThrottledStream
+{
+public:
+    class WriteHandler : public DryadNativePort::Handler, public DrRefCounter
+    {
+    public:
+        WriteHandler(DryadFixedMemoryBuffer* block,
+                     UInt64 streamOffset,
+                     bool flushAfter,
+                     RChannelBufferWriterHandler* handler,
+                     RChannelBufferWriterNative* parent);
+        ~WriteHandler();
+
+        void ProcessingComplete(RChannelItemType statusCode);
+
+        UInt32 GetWriteLength();
+        UInt64 GetStreamOffset();
+        void* GetData();
+        DryadFixedMemoryBuffer* GetBlock();
+        bool IsFlush();
+
+        virtual void QueueWrite(DryadNativePort* port) = 0;
+
+    protected:
+        RChannelBufferWriterNative* GetParent();
+
+    private:
+        DryadFixedMemoryBuffer*            m_block;
+        UInt32                             m_writeLength;
+        bool                               m_flush;
+        RChannelBufferWriterHandler*       m_handler;
+        RChannelBufferWriterNative*        m_parent;
+        UInt64                             m_streamOffset;
+        void*                              m_data;
+        DrBListEntry                       m_listPtr;
+        friend class DryadBList<WriteHandler>;
+    };
+
+    typedef DryadBList<WriteHandler> WriteHandlerList;
+
+    RChannelBufferWriterNative(UInt32 outstandingWritesLowWatermark,
+                               UInt32 outstandingWritesHighWatermark,
+                               DryadNativePort* port,
+                               RChannelOpenThrottler* openThrottler,
+                               bool supportsLazyOpen);
+    ~RChannelBufferWriterNative();
+
+    void Start();
+
+    DryadFixedMemoryBuffer* GetNextWriteBuffer();
+    DryadFixedMemoryBuffer* GetCustomWriteBuffer(Size_t bufferSize);
+
+    bool WriteBuffer(DryadFixedMemoryBuffer* buffer,
+                     bool flushAfter,
+                     RChannelBufferWriterHandler* handler);
+
+    void ReturnUnusedBuffer(DryadFixedMemoryBuffer* buffer);
+
+    void WriteTermination(RChannelItemType reasonCode,
+                          RChannelBufferWriterHandler* handler);
+
+    void FillInStatus(DryadChannelDescription* status);
+
+    bool EnsureOpenForWrite(WriteHandler* handler);
+
+    void Drain(RChannelItemRef* returnItem);
+
+    void Close();
+
+    /* Get/set a hint about the total length the channel is expected
+       to be. Some channel implementations can use this to improve
+       write performance and decrease disk fragmentation. A value of 0
+       (the default) means that the size is unknown. */
+    UInt64 GetInitialSizeHint();
+    void SetInitialSizeHint(UInt64 hint);
+
+    void OpenAfterThrottle();
+
+protected:
+    CRITSEC* GetBaseDR();
+    DryadNativePort* GetPort();
+    void OpenInternal();
+    void ReceiveBufferInternal(WriteHandler* writeHandler, DrError errorCode);
+    void DecrementOutstandingHandlers();
+    bool FinishUsingFile();
+    /* called with baseDR held */
+    void SetProcessedLength(UInt64 processedLength);
+    void SetLowAndHighWaterMark(UInt32 outstandingWritesLowWatermark,
+                                UInt32 outstandingWritesHighWatermark);
+    void SetOpenErrorItem(RChannelItem* errorItem);
+    bool OpenError();
+
+private:
+    enum State {
+        S_Closed,
+        S_Running,
+        S_Stopping,
+        S_Stopped
+    };
+
+    enum OpenState {
+        OS_Closed,
+        OS_NotOpened,
+        OS_Waiting,
+        OS_Opened,
+        OS_OpenError,
+        OS_Stopped
+    };
+
+    typedef std::set<WriteHandler *> WriteHandlerSet;
+
+    virtual bool LazyOpenFile();
+    virtual void FillInOpenedDetails(WriteHandler* handler);
+    virtual void EagerCloseFile();
+    virtual void FinalCloseFile();
+    virtual DryadFixedMemoryBuffer* GetNextWriteBufferInternal() = 0;
+    virtual DryadFixedMemoryBuffer*
+        GetCustomWriteBufferInternal(Size_t bufferSize) = 0;
+    virtual void ReturnUnusedBufferInternal(DryadFixedMemoryBuffer*
+                                            buffer) = 0;
+    virtual WriteHandler* MakeWriteHandler(DryadFixedMemoryBuffer* block,
+                                           bool flushAfter,
+                                           RChannelBufferWriterHandler*
+                                           handler,
+                                           bool detailsPresent,
+                                           bool* extendFile) = 0;
+    virtual void ExtendFileValidLength() = 0;
+    virtual void StartConcreteWriter(RChannelItemRef* pCompletionItem) = 0;
+    virtual void DrainConcreteWriter() = 0;
+
+    bool AddToWriteQueue(WriteHandler* writeRequest,
+                         WriteHandlerList* writeQueue,
+                         bool extendFile);
+    void Unblock(WriteHandlerList* returnList,
+                 WriteHandlerList* processList);
+    void ConsumeErrorBuffer(DrError errorCode,
+                            WriteHandlerList* pReturnList,
+                            WriteHandlerList* pProcessList);
+    RChannelItemType ConsumeBufferCompletion(WriteHandler* writeHandler,
+                                             DrError errorCode,
+                                             WriteHandlerList* pReturnList,
+                                             WriteHandlerList* pProcessList);
+    RChannelItemType DealWithReturnHandlers(WriteHandler* writeHandler,
+                                            DrError errorCode);
+
+    UInt32                       m_lowWatermark;
+    UInt32                       m_highWatermark;
+
+    UInt32                       m_outstandingBuffers;
+    UInt32                       m_outstandingIOs;
+    UInt32                       m_outstandingHandlerSends;
+    bool                         m_outstandingTermination;
+    bool                         m_blocking;
+    bool                         m_flushing;
+    bool                         m_blockingForFileExtension;
+    WriteHandler*                m_terminationHandler;
+    WriteHandlerList             m_blockedList;
+    WriteHandlerSet              m_pendingWriteSet;
+    WriteHandlerSet              m_pendingUnblockSet;
+    RChannelItemRef              m_completionItem;
+    RChannelItemRef              m_openErrorItem;
+    WriteHandlerList             m_openWaitingList;
+    RChannelOpenThrottler*       m_openThrottler;
+
+    UInt64                       m_processedLength;
+    UInt64                       m_initialSizeHint;
+
+    HANDLE                       m_terminationEvent;
+
+    DryadNativePort*             m_port;
+
+    OpenState                    m_openState;
+    State                        m_state;
+    bool                         m_supportsLazyOpen;
+    bool                         m_drainingOpenQueue;
+
+    CRITSEC                      m_baseDR;
+};
+
+class RChannelBufferWriterNativeFile : public RChannelBufferWriterNative
+{
+public:
+    class FileWriteHandler :
+        public RChannelBufferWriterNative::WriteHandler
+    {
+    public:
+        FileWriteHandler(HANDLE fileHandle,
+                         bool detailsPresent,
+                         bool isBuffered,
+                         DryadFixedMemoryBuffer* block,
+                         UInt64 streamOffset,
+                         bool flushAfter,
+                         RChannelBufferWriterHandler* handler,
+                         RChannelBufferWriterNativeFile* parent);
+
+        void SetFileHandle(HANDLE h);
+        HANDLE GetFileHandle();
+        bool IsBuffered();
+
+        void ProcessIO(DrError errorCode, UInt32 numBytes);
+
+        void QueueWrite(DryadNativePort* port);
+
+    private:
+        HANDLE                             m_fileHandle;
+        bool                               m_detailsPresent;
+        bool                               m_isBuffered;
+    };
+
+    RChannelBufferWriterNativeFile(UInt32 bufferSize,
+                                   size_t bufferAlignment,
+                                   UInt32 outstandingWritesLowWatermark,
+                                   UInt32 outstandingWritesHighWatermark,
+                                   DryadNativePort* port,
+                                   RChannelOpenThrottler* openThrottler);
+    ~RChannelBufferWriterNativeFile();
+
+    DrError SetMetaData(DryadMetaData* metaData);
+
+    bool OpenA(const char* pathName);
+//JC    bool OpenW(const wchar_t* pathName);
+
+protected:
+    UInt64                       m_fpDataLength;
+    bool                         m_calcFP;
+    Dryad_dupelim_fprint_data_t    m_fpo;
+    Dryad_dupelim_fprint_uint64_t  m_fp;
+
+    
+private:
+    DrError TryToCreatePathA();
+//JC    DrError TryToCreatePathW();
+    HANDLE CreateFileAndPath(DrError* pErr, SECURITY_ATTRIBUTES *sa);
+    bool LazyOpenFile();
+    void FillInOpenedDetails(WriteHandler* handler);
+    void EagerCloseFile();
+    DryadFixedMemoryBuffer* GetNextWriteBufferInternal();
+    DryadFixedMemoryBuffer* GetCustomWriteBufferInternal(Size_t bufferSize);
+    void ReturnUnusedBufferInternal(DryadFixedMemoryBuffer* buffer);
+    WriteHandler* MakeWriteHandler(DryadFixedMemoryBuffer* block,
+                                   bool flushAfter,
+                                   RChannelBufferWriterHandler*
+                                   handler,
+                                   bool detailsPresent,
+                                   bool* extendFile);
+    void StartConcreteWriter(RChannelItemRef* pCompletionItem);
+    void DrainConcreteWriter();
+    void ExtendFileValidLength();
+
+    bool IsAligned(UInt64 offset);
+    UInt32 AlignmentGap(UInt64 offset);
+    void ReceiveBuffer(WriteHandler* writeHandler, DrError errorCode);
+
+    static bool TryToSetPrivilege();
+
+    UInt32                       m_bufferSize;
+    size_t                       m_bufferAlignment;
+
+    HANDLE                       m_rawFileHandle;
+    HANDLE                       m_bufferedFileHandle;
+
+    bool                         m_fileIsPipe;
+
+    UInt64                       m_nextOffsetToWrite;
+    UInt32                       m_realignmentSize;
+
+    char*                        m_fileNameA;
+//JC    wchar_t*                     m_fileNameW;
+    size_t                       m_fileNameLength;
+//JC    bool                         m_wideFileName;
+
+    bool                         m_tryToCreatePath;
+    bool                         m_canExtendFileLength;
+    UInt64                       m_fileLengthSet;
+
+    static bool                  s_triedToSetPrivilege;
+    static bool                  s_setPrivilege;
+    static CRITSEC               s_privilegeDR;
+
+    friend class FileWriteHandler;
+};
+
+#ifdef TIDYFS
+class RChannelBufferWriterNativeTidyFSStream : public RChannelBufferWriterNativeFile
+{
+public:
+    RChannelBufferWriterNativeTidyFSStream(UInt32 bufferSize,
+                                           size_t bufferAlignment,
+                                           UInt32 outstandingWritesLowWatermark,
+                                           UInt32 outstandingWritesHighWatermark,
+                                           DryadNativePort* port,
+                                           RChannelOpenThrottler* openThrottler);
+    ~RChannelBufferWriterNativeTidyFSStream();
+    DrError OpenA(const char* streamName, DryadMetaData *metaData);
+private:
+    void Close();
+    MDClient *m_client;
+    UInt64 m_partId;
+    char *m_hostname;
+
+};
+#endif

DryadVertex/VertexHost/system/channel/src/channelbufferqueue.cpp

@@ -0,0 +1,777 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#include "channelbufferqueue.h"
+#include "channelhelpers.h"
+#include "channelreader.h"
+
+#pragma unmanaged
+
+
+RChannelBufferQueue::RChannelBufferQueue(RChannelReaderImpl* parent,
+                                         RChannelBufferReader* bufferReader,
+                                         RChannelItemParserBase* parser,
+                                         UInt32 maxParseBatchSize,
+                                         UInt32 maxOutstandingUnits,
+                                         WorkQueue* workQueue)
+{
+    m_parent = parent;
+    m_bufferReader = bufferReader;
+    m_parser = parser;
+    m_workQueue = workQueue;
+
+    m_maxParseBatchSize = m_parser->GetMaxParseBatchSize();
+    if (m_maxParseBatchSize == 0)
+    {
+        m_maxParseBatchSize = maxParseBatchSize;
+    }
+
+    m_maxOutstandingUnits = maxOutstandingUnits;
+    m_outstandingUnits = 0;
+
+    m_state = BQ_Stopped;
+    m_shutDownRequested = false;
+    m_shutDownEvent = ::CreateEvent(NULL, TRUE, FALSE, NULL);
+    LogAssert(m_shutDownEvent != NULL);
+    /* alertCompleteEvent starts out signaled, and only gets reset
+       transiently during the period that ProcessAfterParsing may be
+       calling AlertApplication */
+    m_alertCompleteEvent = ::CreateEvent(NULL, TRUE, TRUE, NULL);
+    LogAssert(m_alertCompleteEvent != NULL);
+
+    m_currentBuffer = NULL;
+    m_firstBuffer = true;
+    m_nextDataSequenceNumber = 0;
+    m_nextDeliverySequenceNumber = 0;
+}
+
+RChannelBufferQueue::~RChannelBufferQueue()
+{
+    LogAssert(m_state == BQ_Stopped);
+    LogAssert(m_currentBuffer == NULL);
+    LogAssert(m_pendingList.IsEmpty());
+    BOOL bRet = ::CloseHandle(m_shutDownEvent);
+    LogAssert(bRet != 0);
+    bRet = ::CloseHandle(m_alertCompleteEvent);
+    LogAssert(bRet != 0);
+}
+
+void RChannelBufferQueue::
+    StartSupplier(RChannelBufferPrefetchInfo* prefetchCookie)
+{
+    {
+        AutoCriticalSection acs(&m_baseDR);
+
+        LogAssert(m_bufferReader != NULL);
+        LogAssert(m_parser != NULL);
+        LogAssert(m_state == BQ_Stopped);
+        LogAssert(m_currentBuffer == NULL);
+        LogAssert(m_pendingList.IsEmpty());
+        m_sendLatch.Start();
+
+        m_shutDownRequested = false;
+        m_firstBuffer = true;
+        m_state = BQ_Empty;
+
+        m_nextDataSequenceNumber = 0;
+        m_nextDeliverySequenceNumber = 0;
+    }
+
+    m_bufferReader->Start(prefetchCookie, this);
+}
+
+//
+// Return whether queue is shutting down or has been requested
+// to shut down
+//
+bool RChannelBufferQueue::ShutDownRequested()
+{
+    bool retval;
+
+    {
+        AutoCriticalSection acs(&m_baseDR);
+
+        LogAssert(m_state != BQ_Stopped);
+        retval = (m_shutDownRequested || m_state == BQ_Stopping);
+    }
+
+    return retval;
+}
+
+//
+// Return all buffers in list and queue a work request if supplied
+//
+void RChannelBufferQueue::DispatchEvents(ChannelBufferList* bufferList,
+                                         WorkRequest* workRequest)
+{
+    DrBListEntry* listEntry = bufferList->GetHead();
+    while (listEntry != NULL)
+    {
+        //
+        // Return all buffers
+        //
+        RChannelBuffer* buffer = bufferList->CastOut(listEntry);
+        listEntry = bufferList->GetNext(listEntry);
+        bufferList->Remove(bufferList->CastIn(buffer));
+//         DrLogD(
+//             "RChannelBufferQueue::RequestNextParse returning buffer");
+        buffer->ProcessingComplete(NULL);
+    }
+
+    if (workRequest != NULL)
+    {
+        //
+        // Queue up work request if specified
+        //
+        bool bRet = m_workQueue->EnQueue(workRequest);
+        LogAssert(bRet == true);
+    }
+}
+
+//
+// Add current and pending buffers to buffer list and forget about them
+// called with RChannelBufferQueue::m_baseDR locked 
+//
+void RChannelBufferQueue::ShutDownBufferQueue(ChannelBufferList* bufferList)
+{
+    LogAssert(m_currentBuffer != NULL);
+
+    bufferList->InsertAsTail(bufferList->CastIn(m_currentBuffer));
+    m_currentBuffer = NULL;
+    bufferList->TransitionToTail(&m_pendingList);
+}
+
+//
+// Queue for processing or clean up if buffer queue is shutting down
+//
+void RChannelBufferQueue::ProcessBuffer(RChannelBuffer* buffer)
+{
+    WorkRequest* workRequest = NULL;
+    bool returnBuffer = false;
+
+    {
+        AutoCriticalSection acs(&m_baseDR);
+
+        LogAssert(m_shutDownRequested == false);
+
+        if (m_state == BQ_Empty)
+        {
+            //
+            // If queue is currently empty, create a new work request for this buffer
+            //
+            m_state = BQ_InWorkQueue;
+            LogAssert(m_currentBuffer == NULL);
+            m_currentBuffer = buffer;
+            // todo: remove comment if not logging
+//             DrLogD( "scheduling parse");
+            workRequest = new RChannelParseRequest(this, true);
+        }
+        else if (m_state == BQ_Stopping)
+        {
+            //
+            // If queue is currently stopping, log and return
+            //
+            LogAssert(m_currentBuffer == NULL);
+            LogAssert(m_pendingList.IsEmpty());
+            returnBuffer = true;
+            DrLogD( "stopping: returning buffer");
+        }
+        else
+        {
+            //
+            // If queue is neither empty nor stopping, add buffer to tail of queue 
+            // to be processed later
+            //
+            LogAssert(m_state == BQ_BlockingItem ||
+                      m_state == BQ_InWorkQueue ||
+                      m_state == BQ_Locked);
+            LogAssert(m_currentBuffer != NULL);
+            m_pendingList.InsertAsTail(m_pendingList.CastIn(buffer));
+            // todo: remove comment if not logging
+//             DrLogD( "queueing buffer");
+        }
+    }
+
+    //
+    // make all calls into other components with no locks held 
+    //
+
+    //
+    // If stopping and buffer not being processed, return the buffer to the pool
+    //
+    if (returnBuffer)
+    {
+        buffer->ProcessingComplete(NULL);
+    }
+
+    //
+    // If new work request was created, enqueue it in work queue
+    //
+    if (workRequest != NULL)
+    {
+        bool bRet = m_workQueue->EnQueue(workRequest);
+        LogAssert(bRet == true);
+    }
+}
+
+//
+// Shut down queue if already requested, otherwise, mark it as locked
+//
+RChannelBuffer* RChannelBufferQueue::GetAndLockCurrentBuffer(bool* outReset)
+{
+    RChannelBuffer* buffer;
+    ChannelBufferList bufferList;
+
+    {
+        AutoCriticalSection acs(&m_baseDR);
+
+        LogAssert(m_currentBuffer != NULL);
+        LogAssert(m_state == BQ_InWorkQueue);
+
+        if (m_shutDownRequested)
+        {
+            //
+            // If shut down is requested, do it now
+            //
+            ShutDownBufferQueue(&bufferList);
+            m_state = BQ_Stopping;
+            BOOL bRet = ::SetEvent(m_shutDownEvent);
+            LogAssert(bRet != 0);
+            *outReset = false;
+            LogAssert(m_currentBuffer == NULL);
+        }
+        else
+        {
+            //
+            // If not shutting down, mark queue as locked 
+            //
+            m_state = BQ_Locked;
+            *outReset = m_firstBuffer;
+            m_firstBuffer = false;
+            LogAssert(m_currentBuffer != NULL);
+        }
+
+        //
+        // m_currentBuffer is NULL if and only if m_shutDownRequested
+        // was true 
+        //
+        buffer = m_currentBuffer;
+    }
+
+    //
+    // Return all buffers in list if queue is shutting down
+    //
+    DispatchEvents(&bufferList, NULL);
+
+    return buffer;
+}
+
+void RChannelBufferQueue::NotifyUnitConsumption()
+{
+    WorkRequest* workRequest = NULL;
+
+    {
+        AutoCriticalSection acs(&m_baseDR);
+
+        LogAssert(m_outstandingUnits > 0);
+        --m_outstandingUnits;
+
+        if (m_state == BQ_BlockingItem &&
+            m_outstandingUnits < m_maxOutstandingUnits)
+        {
+            workRequest = new RChannelParseRequest(this, false);
+            m_state = BQ_InWorkQueue;
+        }
+    }
+
+    if (workRequest != NULL)
+    {
+        bool bRet = m_workQueue->EnQueue(workRequest);
+        LogAssert(bRet == true);
+    }
+}
+
+void RChannelBufferQueue::
+    ProcessAfterParsing(NextParseAction nextParseAction,
+                        RChannelItemArray* itemArray,
+                        UInt64 numberOfSubItemsRead,
+                        UInt64 dataSizeRead,
+                        RChannelBufferPrefetchInfo* prefetchCookie)
+{
+    ChannelUnitList unitList;
+    ChannelBufferList bufferList;
+    WorkRequest* workRequest = NULL;
+    RChannelItemType lastItemType = RChannelItem_ItemHole;
+    bool mustWakeUp = false;
+    RChannelItem* lastItem = NULL;
+    RChannelItemRef alertItem = NULL;
+
+    if (itemArray->GetNumberOfItems() > 0)
+    {
+        lastItem =
+            itemArray->GetItemArray()[itemArray->GetNumberOfItems()-1];
+        lastItemType = lastItem->GetType();
+    }
+
+    if (lastItemType == RChannelItem_ParseError)
+    {
+        /* call into other components with no locks held.
+
+          Make sure no more buffers will be delivered since we are
+          going to stop parsing. When this returns we are sure no more
+          calls to ProcessBuffer will come in so it's safe to call
+          ShutDownBufferQueue below */
+        m_bufferReader->Interrupt();
+    }
+
+    {
+        AutoCriticalSection acs(&m_baseDR);
+
+        LogAssert(m_state == BQ_Locked);
+        LogAssert(m_currentBuffer != NULL);
+
+        if (m_shutDownRequested)
+        {
+            /* a shutdown request came in from the application while
+               we were in the parser, so we don't want to do anything
+               except throw out all pending buffers and return. */
+            ShutDownBufferQueue(&bufferList);
+            m_state = BQ_Stopping;
+            mustWakeUp = true;
+        }
+        else
+        {
+            if (lastItemType == RChannelItem_ParseError)
+            {
+                /* ditch any buffers we haven't parsed yet along with
+                   the current buffer */
+                ShutDownBufferQueue(&bufferList);
+                m_state = BQ_Stopping;
+                mustWakeUp = true;
+            }
+            else if (nextParseAction != NPA_RequestItem)
+            {
+                /* we're done with the current buffer; put it in the
+                   unit queue for completion when the items in it have
+                   been processed */
+                RChannelUnit* unit =
+                    new RChannelBufferBoundaryUnit(m_currentBuffer,
+                                                   prefetchCookie);
+                unitList.InsertAsTail(unitList.CastIn(unit));
+                m_currentBuffer = NULL;
+            }
+
+            if (itemArray->GetNumberOfItems() > 0)
+            {
+                RChannelItemRef terminationItem;
+                if (RChannelItem::IsTerminationItem(lastItemType))
+                {
+                    terminationItem = lastItem;
+                }
+
+                RChannelItemUnit* unit =
+                    new RChannelSerializedUnit(this,
+                                               itemArray, terminationItem);
+                unit->SetSizes(numberOfSubItemsRead, dataSizeRead);
+                ++m_outstandingUnits;
+
+                unitList.InsertAsTail(unitList.CastIn(unit));
+                if (nextParseAction == NPA_StopParsing)
+                {
+                    alertItem = lastItem;
+                    if (alertItem != NULL)
+                    {
+                        /* reset the event so InterruptSupplier won't
+                           complete until the application has been
+                           alerted */
+                        BOOL bRet = ::ResetEvent(m_alertCompleteEvent);
+                        LogAssert(bRet != 0);
+                    }
+                }
+            }
+
+            switch (nextParseAction)
+            {
+            case NPA_RequestBuffer:
+                LogAssert(m_currentBuffer == NULL);
+                /* we would like to start processing the next buffer,
+                   so put it in the queue if there is one */
+                if (m_pendingList.IsEmpty())
+                {
+                    m_state = BQ_Empty;
+                }
+                else
+                {
+                    m_currentBuffer =
+                        m_pendingList.CastOut(m_pendingList.RemoveHead());
+                    workRequest = new RChannelParseRequest(this, true);
+                    m_state = BQ_InWorkQueue;
+                }
+                break;
+
+            case NPA_RequestItem:
+                LogAssert(m_currentBuffer != NULL);
+                if (m_outstandingUnits >= m_maxOutstandingUnits)
+                {
+                    m_state = BQ_BlockingItem;
+                }
+                else
+                {
+                    workRequest = new RChannelParseRequest(this, false);
+                    m_state = BQ_InWorkQueue;
+                }
+                break;
+
+            case NPA_EndOfStream:
+                LogAssert(m_pendingList.IsEmpty());
+                LogAssert(m_currentBuffer == NULL);
+                LogAssert(lastItemType == RChannelItem_EndOfStream);
+                m_state = BQ_Stopping;
+                break;
+
+            case NPA_StopParsing:
+                LogAssert(m_pendingList.IsEmpty());
+                LogAssert(m_currentBuffer == NULL);
+                LogAssert(lastItemType == RChannelItem_ParseError ||
+                          lastItemType == RChannelItem_Abort ||
+                          lastItemType == RChannelItem_Restart);
+                m_state = BQ_Stopping;
+                break;
+
+            default:
+                LogAssert(false);
+            }
+        }
+
+        m_sendLatch.AcceptList(&unitList);
+    }
+
+    /* make all calls into other components with no locks held */
+
+    DispatchEvents(&bufferList, workRequest);
+
+    while (unitList.IsEmpty() == false)
+    {
+        m_parent->AddUnitList(&unitList);
+
+        {
+            AutoCriticalSection acs(&m_baseDR);
+
+            m_sendLatch.TransferList(&unitList);
+        }
+    }
+
+    if (nextParseAction == NPA_StopParsing)
+    {
+        if (alertItem != NULL)
+        {
+            m_parent->AlertApplication(alertItem);
+            /* set the event so InterruptSupplier can complete now
+               that the application has been alerted */
+            BOOL bRet = ::SetEvent(m_alertCompleteEvent);
+            LogAssert(bRet != 0);
+        }
+    }
+    else
+    {
+        LogAssert(alertItem == NULL);
+    }
+
+    if (mustWakeUp)
+    {
+        BOOL bRet = ::SetEvent(m_shutDownEvent);
+        LogAssert(bRet != 0);
+    }
+}
+
+//
+// Called when RChannelParseRequest.Process is called in work queue
+//
+void RChannelBufferQueue::ParseRequest(bool useNewBuffer)
+{
+//     DrLogD(
+//         "RChannelBufferQueue::ParseRequest entered",
+//         "useNewBuffer: %s", (useNewBuffer) ? "true" : "false");
+
+    RChannelBuffer* buffer;
+    bool resetParser = false;
+    UInt64 numberOfSubItemsRead = 0;
+    UInt64 dataSizeRead = 0;
+
+    //
+    // Handle shutdown logic if necessary or lock the queue
+    //
+    buffer = GetAndLockCurrentBuffer(&resetParser);
+    if (buffer == NULL)
+    {
+        //
+        // there was a shutdown request while this work item was in
+        // the queue, which has now been dealt with, so just exit 
+        //
+        return;
+    }
+
+    RChannelBufferType bufferType = buffer->GetType();
+    RChannelItemArrayRef itemArray;
+    itemArray.Attach(new RChannelItemArray());
+    RChannelBufferPrefetchInfo* prefetchCookie = NULL;
+    NextParseAction nextParseAction;
+
+    if (bufferType == RChannelBuffer_Data ||
+        bufferType == RChannelBuffer_Hole ||
+        bufferType == RChannelBuffer_EndOfStream)
+    {
+        /* get the parser to give us the next item, if any */
+        RChannelBuffer* parseBuffer = (useNewBuffer) ? buffer : NULL;
+
+        UInt32 parsedItemCount = 0;
+        itemArray->SetNumberOfItems(m_maxParseBatchSize);
+        RChannelItemRef* items = itemArray->GetItemArray();
+
+        do
+        {
+            RChannelItem* item =
+                m_parser->RawParseItem(resetParser, parseBuffer,
+                                       &prefetchCookie);
+            resetParser = false;
+            parseBuffer = NULL;
+
+            if (item == NULL)
+            {
+                /* the parser should only ask for another buffer if
+                   there's more data and it didn't give back an item */
+                LogAssert(bufferType != RChannelBuffer_EndOfStream);
+                nextParseAction = NPA_RequestBuffer;
+            }
+            else
+            {
+                LogAssert(prefetchCookie == NULL);
+
+                RChannelItemType itemType = item->GetType();
+                if (itemType == RChannelItem_EndOfStream)
+                {
+                    LogAssert(bufferType == RChannelBuffer_EndOfStream);
+                    nextParseAction = NPA_EndOfStream;
+                }
+                else if (itemType == RChannelItem_ParseError)
+                {
+                    nextParseAction = NPA_StopParsing;
+                }
+                else
+                {
+                    LogAssert(itemType == RChannelItem_Data ||
+                              itemType == RChannelItem_ItemHole ||
+                              itemType == RChannelItem_BufferHole);
+                    nextParseAction = NPA_RequestItem;
+                }
+
+                item->SetDeliverySequenceNumber(m_nextDeliverySequenceNumber);
+                ++m_nextDeliverySequenceNumber;
+                /* Only data items merit a new sequence number; everything
+                   else gets the sequence number of the next data item. */
+                item->SetDataSequenceNumber(m_nextDataSequenceNumber);
+                if (itemType == RChannelItem_Data)
+                {
+                    ++m_nextDataSequenceNumber;
+                }
+
+                numberOfSubItemsRead += item->GetNumberOfSubItems();
+                dataSizeRead += item->GetItemSize();
+
+                items[parsedItemCount].Attach(item);
+                ++parsedItemCount;
+            }
+        } while (nextParseAction == NPA_RequestItem &&
+                 parsedItemCount < m_maxParseBatchSize);
+
+        itemArray->TruncateToSize(parsedItemCount);
+    }
+    else
+    {
+        LogAssert(useNewBuffer == true);
+        RChannelBufferMarker* markerBuffer = (RChannelBufferMarker *) buffer;
+        RChannelItem* item = markerBuffer->GetItem();
+
+        item->SetDeliverySequenceNumber(m_nextDeliverySequenceNumber);
+        ++m_nextDeliverySequenceNumber;
+        /* Only data items merit a new sequence number; everything
+           else gets the sequence number of the next data item. */
+        item->SetDataSequenceNumber(m_nextDataSequenceNumber);
+
+        itemArray->SetNumberOfItems(1);
+        itemArray->GetItemArray()[0] = item;
+        LogAssert(bufferType == RChannelBuffer_Restart ||
+                  bufferType == RChannelBuffer_Abort);
+        nextParseAction = NPA_StopParsing;
+    }
+
+    ProcessAfterParsing(nextParseAction,
+                        itemArray, numberOfSubItemsRead, dataSizeRead,
+                        prefetchCookie);
+}
+
+//
+// Stop creation of new buffers
+//
+void RChannelBufferQueue::InterruptSupplier()
+{
+    bool waitForSend = false;
+    bool waitForLock = false;
+
+    //
+    // Interrupt the reader associated with this queue
+    // call into other components with no locks held. When this
+    // returns we are sure no more calls to ProcessBuffer will come in
+    //
+    m_bufferReader->Interrupt();
+
+    WorkRequest* workRequest = NULL;
+
+    {
+        AutoCriticalSection acs(&m_baseDR);
+
+        LogAssert(m_shutDownRequested == false);
+
+        if (m_state == BQ_BlockingItem)
+        {
+            //
+            // the easiest thing to do is to transition into the
+            // InWorkQueue state and continue interrupting using the
+            // normal path 
+            //
+            workRequest = new RChannelParseRequest(this, false);
+            m_state = BQ_InWorkQueue;
+        }
+
+        if (m_state == BQ_Locked ||
+            m_state == BQ_InWorkQueue)
+        {
+            //
+            // If queue is locked or working, we're going to
+            // request a shutdown but wait for the lock
+            //
+            m_shutDownRequested = true;
+            waitForLock = true;
+            BOOL bRet = ::ResetEvent(m_shutDownEvent);
+            LogAssert(bRet != 0);
+        }
+        else
+        {
+            if (m_state == BQ_Empty)
+            {
+                m_state = BQ_Stopping;
+            }
+
+            LogAssert(m_state == BQ_Stopping);
+            LogAssert(m_currentBuffer == NULL);
+            LogAssert(m_pendingList.IsEmpty());
+        }
+
+        //
+        // Interrupt the send latch. waitForSend = true if currently sending
+        //
+        waitForSend = m_sendLatch.Interrupt();
+    }
+
+    if (workRequest != NULL)
+    {
+        //
+        // If blocking, add request to queue to call this.ParseRequest, which will shut down the queue
+        //
+        bool bRet = m_workQueue->EnQueue(workRequest);
+        LogAssert(bRet == true);
+    }
+
+    if (waitForSend)
+    {
+        //
+        // Wait on send latch to stop sending if necessary
+        //
+        m_sendLatch.Wait();
+    }
+
+    //
+    // Wait until any alerts have been delivered to the application 
+    //
+    BOOL bRet = ::WaitForSingleObject(m_alertCompleteEvent, INFINITE);
+    LogAssert(bRet == WAIT_OBJECT_0);
+
+    if (waitForLock)
+    {
+        //
+        // Wait for shut down if work queue is still processing
+        //
+        bRet = ::WaitForSingleObject(m_shutDownEvent, INFINITE);
+        LogAssert(bRet == WAIT_OBJECT_0);
+    }
+
+    {
+        //
+        // Ensure that everything is shut down correctly
+        //
+        AutoCriticalSection acs(&m_baseDR);
+
+        LogAssert(m_state == BQ_Stopping);
+        LogAssert(m_currentBuffer == NULL);
+        LogAssert(m_pendingList.IsEmpty());
+        m_shutDownRequested = false;
+    }
+}
+
+void RChannelBufferQueue::DrainSupplier(RChannelItem* drainItem)
+{
+    LogAssert(RChannelItem::IsTerminationItem(drainItem->GetType()));
+
+    InterruptSupplier();
+
+    /* this won't return until all outstanding buffers have had their
+       completion handlers called. By the time this happens everything
+       must have drained out of the work queue and the parser */
+    m_bufferReader->Drain(drainItem);
+
+    {
+        AutoCriticalSection acs(&m_baseDR);
+
+        LogAssert(m_state == BQ_Stopping);
+        LogAssert(m_outstandingUnits == 0);
+        LogAssert(m_currentBuffer == NULL);
+        LogAssert(m_pendingList.IsEmpty());
+        m_state = BQ_Stopped;
+        m_sendLatch.Stop();
+    }
+}
+
+void RChannelBufferQueue::CloseSupplier()
+{
+    m_bufferReader->Close();
+
+    {
+        AutoCriticalSection acs(&m_baseDR);
+
+        LogAssert(m_state == BQ_Stopped);
+        m_bufferReader = NULL;
+        m_parser = NULL;
+    }
+}
+
+bool RChannelBufferQueue::GetTotalLength(UInt64* pLen)
+{
+    return m_bufferReader->GetTotalLength(pLen);
+}

DryadVertex/VertexHost/system/channel/src/channelbufferqueue.h

@@ -0,0 +1,116 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+class RChannelReaderImpl;
+class RChannelBuffer;
+class RChannelBufferReader;
+class RChannelBufferPrefetchInfo;
+class RChannelUnit;
+class RChannelParseRequest;
+
+#include "channelbuffer.h"
+#include <channelreader.h>
+#include <channelparser.h>
+#include <workqueue.h>
+#include <orderedsendlatch.h>
+
+typedef DryadBList<RChannelUnit> ChannelUnitList;
+
+class RChannelBufferQueue :
+    public RChannelReaderSupplier,
+    public RChannelBufferReaderHandler
+{
+public:
+    RChannelBufferQueue(RChannelReaderImpl* parent,
+                        RChannelBufferReader* bufferReader,
+                        RChannelItemParserBase* parser,
+                        UInt32 maxParseBatchSize,
+                        UInt32 maxOutstandingUnits,
+                        WorkQueue* workQueue);
+    ~RChannelBufferQueue();
+
+    void StartSupplier(RChannelBufferPrefetchInfo* prefetchCookie);
+    void InterruptSupplier();
+    void DrainSupplier(RChannelItem* drainItem);
+
+    void ProcessBuffer(RChannelBuffer* buffer);
+    void NotifyUnitConsumption();
+
+    void CloseSupplier();
+
+    bool GetTotalLength(UInt64* pLen);
+
+private:
+    enum QueueState {
+        BQ_Stopped,
+        BQ_Empty,
+        BQ_BlockingItem,
+        BQ_InWorkQueue,
+        BQ_Locked,
+        BQ_Stopping
+    };
+
+    enum NextParseAction {
+        NPA_RequestBuffer,
+        NPA_RequestItem,
+        NPA_StopParsing,
+        NPA_EndOfStream
+    };
+
+    void DispatchEvents(ChannelBufferList* bufferList,
+                        WorkRequest* workRequest);
+    void ShutDownBufferQueue(ChannelBufferList* bufferList);
+    RChannelBuffer* GetAndLockCurrentBuffer(bool* outReset);
+    void ProcessAfterParsing(NextParseAction nextParseAction,
+                             RChannelItemArray* itemArray,
+                             UInt64 numberOfSubItemsRead,
+                             UInt64 dataSizeRead,
+                             RChannelBufferPrefetchInfo* prefetchCookie);
+
+    void ParseRequest(bool useNewBuffer);
+    bool ShutDownRequested();
+
+    RChannelReaderImpl*                  m_parent;
+    RChannelBufferReader*                m_bufferReader;
+    RChannelItemParserRef                m_parser;
+    UInt32                               m_maxParseBatchSize;
+    UInt32                               m_maxOutstandingUnits;
+    WorkQueue*                           m_workQueue;
+
+    QueueState                           m_state;
+    DryadOrderedSendLatch<ChannelUnitList> m_sendLatch;
+    bool                                 m_shutDownRequested;
+    HANDLE                               m_shutDownEvent;
+    HANDLE                               m_alertCompleteEvent;
+
+    RChannelBuffer*                      m_currentBuffer;
+    bool                                 m_firstBuffer;
+    ChannelBufferList                    m_pendingList;
+    UInt32                               m_outstandingUnits;
+
+    UInt64                               m_nextDataSequenceNumber;
+    UInt64                               m_nextDeliverySequenceNumber;
+
+    CRITSEC                              m_baseDR;
+
+    friend class RChannelParseRequest;
+};

DryadVertex/VertexHost/system/channel/src/channelfifo.h

@@ -0,0 +1,241 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include <channelreader.h>
+#include <channelwriter.h>
+#include <dryadeventcache.h>
+
+class RChannelFifo;
+class RChannelFifoUnit;
+typedef DryadBListDerived<RChannelFifoUnit,RChannelUnit> ChannelFifoUnitList;
+
+class RChannelFifoWriterBase :
+    public RChannelWriter,
+    public RChannelReaderSupplier
+{
+public:
+    RChannelFifo* GetParent();
+
+    /* partial RChannelWriter interface */
+    void WriteItemArray(RChannelItemArrayRef& itemArray,
+                        bool flushAfter,
+                        RChannelItemArrayWriterHandler* handler);
+    RChannelItemType WriteItemArraySync(RChannelItemArrayRef& itemArray,
+                                        bool flush,
+                                        RChannelItemArrayRef* failureArray);
+    void Close();
+
+    /* partial RChannelReaderSupplier interface */
+    void CloseSupplier();
+
+    UInt64 GetDataSizeWritten();
+
+    /* Get/Set the URI of the channel. */
+    const char* GetURI();
+    void SetURI(const char* uri);
+
+    /* these are just dummies for a fifo: the hint is always returned
+       as zero */
+    UInt64 GetInitialSizeHint();
+    void SetInitialSizeHint(UInt64 hint);
+
+protected:
+    enum WriterState {
+        WS_Closed,
+        WS_Running,
+        WS_Draining,
+        WS_Stopped
+    };
+
+    enum SupplierState {
+        SS_Closed,
+        SS_Running,
+        SS_Interrupted,
+        SS_Draining,
+        SS_Stopped
+    };
+
+    class SyncHandler : public RChannelItemArrayWriterHandler
+    {
+    public:
+        SyncHandler();
+        ~SyncHandler();
+
+        void ProcessWriteArrayCompleted(RChannelItemType status,
+                                        RChannelItemArray* failureArray);
+
+        void UseEvent(DryadHandleListEntry* event);
+        bool UsingEvent();
+        RChannelItemType GetStatusCode();
+        void Wait();
+        DryadHandleListEntry* GetEvent();
+
+    private:
+        DryadHandleListEntry*       m_event;
+        LONG volatile               m_usingEvent;
+        RChannelItemType            m_statusCode;
+    };
+
+    RChannelFifoWriterBase(RChannelFifo* parent);
+    virtual ~RChannelFifoWriterBase();
+
+    virtual void AcceptReturningUnit(RChannelFifoUnit* unit) = 0;
+    virtual void EnqueueItemArray(RChannelItemArrayRef& itemArray,
+                                  RChannelItemArrayWriterHandler* handler,
+                                  SyncHandler* SyncHandler) = 0;
+
+    void SetReader(RChannelReaderImpl* reader);
+
+    void ReturnHandlers(ChannelFifoUnitList* returnList);
+    void SendUnits(ChannelUnitList* unitList);
+    RChannelFifoUnit* DuplicateUnit(RChannelFifoUnit* unit,
+                                    RChannelItemArray* itemArray);
+
+    RChannelFifo*                             m_parent;
+    WriterState                               m_writerState;
+    SupplierState                             m_supplierState;
+    ChannelFifoUnitList                       m_blockedList;
+    DryadOrderedSendLatch<ChannelUnitList>    m_sendLatch;
+    DryadOrderedSendLatch<ChannelFifoUnitList>    m_returnLatch;
+    RChannelReaderImpl*                       m_reader;
+    UInt64                                    m_nextDataSequenceNumber;
+    UInt64                                    m_nextDeliverySequenceNumber;
+    UInt64                                    m_numberOfSubItemsWritten;
+    UInt64                                    m_dataSizeWritten;
+    UInt32                                    m_outstandingUnits;
+    RChannelItemRef                           m_writerTerminationItem;
+    RChannelItemRef                           m_readerTerminationItem;
+    UInt32                                    m_writerEpoch;
+    UInt32                                    m_supplierEpoch;
+    HANDLE                                    m_supplierDrainEvent;
+    DryadEventCache                           m_eventCache;
+
+    DrStr128                                   m_uri;
+
+    CRITSEC                                   m_baseDR;
+
+    friend class RChannelFifoUnit;
+    friend class RChannelFifo;
+};
+
+class RChannelFifoWriter :
+    public RChannelFifoWriterBase
+{
+public:
+    /* RChannelWriter interface */
+    void Start();
+    void Drain(DrTimeInterval timeOut, RChannelItemRef* pReturnItem);
+    void GetTerminationItems(RChannelItemRef* pWriterDrainItem,
+                             RChannelItemRef* pReaderDrainItem);
+
+    /* RChannelReaderSupplier interface */
+    void StartSupplier(RChannelBufferPrefetchInfo* prefetchCookie);
+    void InterruptSupplier();
+    void DrainSupplier(RChannelItem* drainItem);
+
+private:
+    RChannelFifoWriter(RChannelFifo* parent, UInt32 fifoLength);
+    ~RChannelFifoWriter();
+
+    bool ReWriteBlockedListForEarlyReturn(RChannelItemType drainType);
+    bool CheckForTerminationItem(RChannelFifoUnit* unit);
+    void StartBlocking(RChannelFifoUnit* unit,
+                       SyncHandler* SyncHandler);
+    void EnqueueItemArray(RChannelItemArrayRef& itemArray,
+                          RChannelItemArrayWriterHandler* handler,
+                          SyncHandler* SyncHandler);
+    void AcceptReturningUnit(RChannelFifoUnit* unit);
+
+    UInt32                                    m_availableUnits;
+    UInt32                                    m_fifoLength;
+    RChannelFifoUnit*                         m_terminationUnit;
+    HANDLE                                    m_writerDrainEvent;
+
+    friend class RChannelFifoUnit;
+    friend class RChannelFifo;
+};
+
+class RChannelFifoNBWriter :
+    public RChannelFifoWriterBase
+{
+public:
+    /* RChannelWriter interface */
+    void Start();
+    void Drain(DrTimeInterval timeOut, RChannelItemRef* pReturnItem);
+    void GetTerminationItems(RChannelItemRef* pWriterDrainItem,
+                             RChannelItemRef* pReaderDrainItem);
+
+    /* RChannelReaderSupplier interface */
+    void StartSupplier(RChannelBufferPrefetchInfo* prefetchCookie);
+    void InterruptSupplier();
+    void DrainSupplier(RChannelItem* drainItem);
+
+private:
+    RChannelFifoNBWriter(RChannelFifo* parent);
+
+    bool CheckForTerminationItem(RChannelFifoUnit* unit);
+    void EnqueueItemArray(RChannelItemArrayRef& itemArray,
+                          RChannelItemArrayWriterHandler* handler,
+                          SyncHandler* SyncHandler);
+    void AcceptReturningUnit(RChannelFifoUnit* unit);
+
+    friend class RChannelFifoUnit;
+    friend class RChannelFifo;
+};
+
+class RChannelFifoReader : public RChannelReaderImpl
+{
+public:
+    RChannelFifo* GetParent();
+
+    bool GetTotalLength(UInt64* pLen);
+    bool GetExpectedLength(UInt64* pLen);
+    void SetExpectedLength(UInt64 expectedLength);
+
+private:
+    RChannelFifoReader(RChannelFifo* parent,
+                       WorkQueue* workQueue);
+    ~RChannelFifoReader();
+
+    RChannelFifo*   m_parent;
+
+    friend class RChannelFifo;
+};
+
+class RChannelFifo
+{
+public:
+    RChannelFifo(const char* name,
+                 UInt32 fifoLength, WorkQueue* workQueue);
+    ~RChannelFifo();
+
+    const char* GetName();
+    RChannelFifoReader* GetReader();
+    RChannelFifoWriterBase* GetWriter();
+
+    static const UInt32 s_infiniteBuffer = (UInt32) -1;
+
+private:
+    DrStr64                   m_name;
+    RChannelFifoReader*       m_reader;
+    RChannelFifoWriterBase*   m_writer;
+};

DryadVertex/VertexHost/system/channel/src/channelhelpers.cpp

@@ -0,0 +1,316 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#include "channelparser.h"
+#include "channelhelpers.h"
+#include "channelreader.h"
+#include "channelbuffer.h"
+#include "channelbufferqueue.h"
+#include "channelfifo.h"
+
+#pragma unmanaged
+
+
+RChannelUnit::RChannelUnit(RChannelUnitType type)
+{
+    m_type = type;
+}
+
+RChannelUnit::~RChannelUnit()
+{
+}
+
+RChannelUnitType RChannelUnit::GetType()
+{
+    return m_type;
+}
+
+RChannelItemUnit::RChannelItemUnit(RChannelItemArray* payload,
+                                   RChannelItem* terminationItem) :
+    RChannelUnit(RChannelUnit_Item)
+{
+    m_payload = payload;
+    m_terminationItem = terminationItem;
+    m_numberOfSubItems = 0;
+    m_dataSize = 0;
+}
+
+RChannelItemUnit::~RChannelItemUnit()
+{
+}
+
+RChannelItemArray* RChannelItemUnit::GetItemArray()
+{
+    return m_payload;
+}
+
+RChannelItem* RChannelItemUnit::GetTerminationItem()
+{
+    return m_terminationItem;
+}
+
+void RChannelItemUnit::DiscardItems()
+{
+    m_payload = NULL;
+    m_terminationItem = NULL;
+}
+
+void RChannelItemUnit::SetSizes(UInt64 numberOfSubItems,
+                                UInt64 dataSize)
+{
+    m_numberOfSubItems = numberOfSubItems;
+    m_dataSize = dataSize;
+}
+
+UInt64 RChannelItemUnit::GetNumberOfSubItems()
+{
+    return m_numberOfSubItems;
+}
+
+UInt64 RChannelItemUnit::GetDataSize()
+{
+    return m_dataSize;
+}
+
+
+RChannelSerializedUnit::RChannelSerializedUnit(RChannelBufferQueue* parent,
+                                               RChannelItemArray* payload,
+                                               RChannelItem* terminationItem) :
+    RChannelItemUnit(payload, terminationItem)
+{
+    LogAssert(m_payload != NULL);
+    m_parent = parent;
+}
+
+void RChannelSerializedUnit::ReturnToSupplier()
+{
+    /* we should have already transferred away the array before getting
+       here, and don't need to do anything more */
+    LogAssert(m_payload == NULL);
+    m_parent->NotifyUnitConsumption();
+    delete this;
+}
+
+
+RChannelFifoUnit::RChannelFifoUnit(RChannelFifoWriterBase* parent) :
+    RChannelItemUnit(NULL, NULL)
+{
+    m_parent = parent;
+    m_handler = NULL;
+    m_statusCode = RChannelItem_Data;
+}
+
+RChannelFifoUnit::~RChannelFifoUnit()
+{
+    LogAssert(m_handler == NULL);
+}
+
+void RChannelFifoUnit::SetPayload(RChannelItemArray* itemArray,
+                                  RChannelItemArrayWriterHandler* handler,
+                                  RChannelItemType statusCode)
+{
+    LogAssert(itemArray != NULL);
+    LogAssert(m_payload == NULL);
+    LogAssert(m_terminationItem == NULL);
+    LogAssert(m_handler == NULL);
+
+    m_payload = itemArray;
+    m_handler = handler;
+    m_statusCode = statusCode;
+}
+
+void RChannelFifoUnit::SetTerminationItem(RChannelItem* terminationItem)
+{
+    LogAssert(RChannelItem::IsTerminationItem(terminationItem->GetType()));
+    m_terminationItem = terminationItem;
+}
+
+void RChannelFifoUnit::Disgorge(RChannelItemArrayWriterHandler** pHandler,
+                                RChannelItemType* pStatusCode)
+{
+    LogAssert(m_payload == NULL);
+
+    *pHandler = m_handler;
+    m_handler = NULL;
+
+    *pStatusCode = m_statusCode;
+    m_statusCode = RChannelItem_Data;
+}
+
+void RChannelFifoUnit::SetStatusCode(RChannelItemType statusCode)
+{
+    LogAssert(m_handler != NULL);
+    m_statusCode = statusCode;
+}
+
+RChannelItemType RChannelFifoUnit::GetStatusCode()
+{
+    return m_statusCode;
+}
+
+void RChannelFifoUnit::ReturnToSupplier()
+{
+    LogAssert(m_parent != NULL);
+    LogAssert(m_payload == NULL);
+    m_parent->AcceptReturningUnit(this);
+}
+
+RChannelBufferBoundaryUnit::
+    RChannelBufferBoundaryUnit(RChannelBuffer* buffer,
+                               RChannelBufferPrefetchInfo* prefetchCookie) :
+        RChannelUnit(RChannelUnit_BufferBoundary)
+{
+    LogAssert(buffer != NULL);
+    m_buffer = buffer;
+    m_prefetchCookie = prefetchCookie;
+}
+
+RChannelBufferBoundaryUnit::~RChannelBufferBoundaryUnit()
+{
+    LogAssert(m_buffer == NULL);
+}
+
+void RChannelBufferBoundaryUnit::ReturnToSupplier()
+{
+    LogAssert(m_buffer != NULL);
+    m_buffer->ProcessingComplete(m_prefetchCookie);
+    m_buffer = NULL;
+    m_prefetchCookie = NULL;
+    delete this;
+}
+
+
+RChannelProcessRequest::
+    RChannelProcessRequest(RChannelReaderImpl* parent,
+                           RChannelItemArrayReaderHandler* handler,
+                           void* cancelCookie)
+{
+    m_aborted = 0;
+    m_parent = parent;
+    m_handler = handler;
+    m_cookie = cancelCookie;
+}
+
+RChannelProcessRequest::~RChannelProcessRequest()
+{
+}
+
+//
+// Have request parent process the item
+//
+void RChannelProcessRequest::Process()
+{
+    m_parent->ProcessItemArrayRequest(this);
+    m_itemArray = NULL;
+}
+
+bool RChannelProcessRequest::ShouldAbort()
+{
+    return (::InterlockedExchangeAdd(&m_aborted, 0) != 0);
+}
+
+void RChannelProcessRequest::SetItemArray(RChannelItemArray* itemArray)
+{
+    m_itemArray = itemArray;
+}
+
+RChannelItemArray* RChannelProcessRequest::GetItemArray()
+{
+    return m_itemArray;
+}
+
+RChannelItemArrayReaderHandler* RChannelProcessRequest::GetHandler()
+{
+    LogAssert(m_handler != NULL);
+    return m_handler;
+}
+
+void* RChannelProcessRequest::GetCookie()
+{
+    return m_cookie;
+}
+
+
+void RChannelProcessRequest::Cancel()
+{
+    ::InterlockedIncrement(&m_aborted);
+}
+
+
+RChannelParseRequest::
+    RChannelParseRequest(RChannelBufferQueue* parent,
+                         bool useNewBuffer)
+{
+    m_parent = parent;
+    m_useNewBuffer = useNewBuffer;
+}
+
+void RChannelParseRequest::Process()
+{
+    m_parent->ParseRequest(m_useNewBuffer);
+}
+
+bool RChannelParseRequest::ShouldAbort()
+{
+    return m_parent->ShutDownRequested();
+}
+
+
+RChannelMarshalRequest::
+    RChannelMarshalRequest(RChannelSerializedWriter* parent)
+{
+    m_parent = parent;
+}
+
+void RChannelMarshalRequest::Process()
+{
+    m_parent->MarshalItems();
+}
+
+bool RChannelMarshalRequest::ShouldAbort()
+{
+    return false;
+}
+
+
+RChannelReaderSyncWaiter::
+    RChannelReaderSyncWaiter(RChannelReaderImpl* parent,
+                             HANDLE event,
+                             RChannelItemArrayRef* itemDstArray)
+{
+    m_parent = parent;
+    m_event = event;
+    m_itemDstArray = itemDstArray;
+}
+
+RChannelReaderSyncWaiter::~RChannelReaderSyncWaiter()
+{
+    LogAssert(m_event == INVALID_HANDLE_VALUE);
+}
+
+void RChannelReaderSyncWaiter::ProcessItemArray(RChannelItemArray* itemArray)
+{
+    LogAssert(m_event != INVALID_HANDLE_VALUE);
+    m_parent->ThreadSafeSetItemArray(m_itemDstArray, itemArray);
+    HANDLE event = m_event;
+    m_event = INVALID_HANDLE_VALUE;
+    BOOL bRet = ::SetEvent(event);
+    LogAssert(bRet != 0);
+}

DryadVertex/VertexHost/system/channel/src/channelhelpers.h

@@ -0,0 +1,201 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+class RChannelBuffer;
+class RChannelBufferQueue;
+class RChannelFifoWriterBase;
+
+#include "channelparser.h"
+#include "channelreader.h"
+#include "channelwriter.h"
+#include <workqueue.h>
+
+enum RChannelUnitType {
+    RChannelUnit_Item,
+    RChannelUnit_BufferBoundary
+};
+
+class RChannelUnit
+{
+public:
+    virtual ~RChannelUnit();
+
+    RChannelUnitType GetType();
+
+    virtual void ReturnToSupplier() = 0;
+
+protected:
+    RChannelUnit(RChannelUnitType type);
+
+private:
+    RChannelUnitType  m_type;
+    DrBListEntry      m_listPtr;
+    friend class DryadBList<RChannelUnit>;
+};
+
+typedef class DryadBList<RChannelUnit> ChannelUnitList;
+
+class RChannelItemUnit : public RChannelUnit
+{
+public:
+    virtual ~RChannelItemUnit();
+
+    RChannelItemArray* GetItemArray();
+    RChannelItem* GetTerminationItem();
+
+    void DiscardItems();
+
+    void SetSizes(UInt64 numberOfSubItems, UInt64 dataSize);
+    UInt64 GetNumberOfSubItems();
+    UInt64 GetDataSize();
+
+protected:
+    RChannelItemUnit(RChannelItemArray* payload,
+                     RChannelItem* terminationItem);
+
+    RChannelItemArrayRef  m_payload;
+    RChannelItemRef       m_terminationItem;
+    UInt64                m_numberOfSubItems;
+    UInt64                m_dataSize;
+};
+
+class RChannelSerializedUnit : public RChannelItemUnit
+{
+public:
+    RChannelSerializedUnit(RChannelBufferQueue* parent,
+                           RChannelItemArray* payload,
+                           RChannelItem* terminationItem);
+    void ReturnToSupplier();
+
+private:
+    RChannelBufferQueue*  m_parent;
+};
+
+class RChannelFifoUnit : public RChannelItemUnit
+{
+public:
+    RChannelFifoUnit(RChannelFifoWriterBase* parent);
+    ~RChannelFifoUnit();
+
+    void SetPayload(RChannelItemArray* itemArray,
+                    RChannelItemArrayWriterHandler* handler,
+                    RChannelItemType statusCode);
+
+    void SetTerminationItem(RChannelItem* terminationItem);
+
+
+    void ReturnToSupplier();
+
+    void Disgorge(RChannelItemArrayWriterHandler** pHandler,
+                  RChannelItemType* pStatusCode);
+
+    void SetStatusCode(RChannelItemType statusCode);
+    RChannelItemType GetStatusCode();
+
+private:
+    RChannelFifoWriterBase*            m_parent;
+    RChannelItemArrayWriterHandler*    m_handler;
+    RChannelItemType                   m_statusCode;
+};
+
+class RChannelBufferBoundaryUnit : public RChannelUnit
+{
+public:
+    RChannelBufferBoundaryUnit(RChannelBuffer* buffer,
+                               RChannelBufferPrefetchInfo* prefetchCookie);
+    ~RChannelBufferBoundaryUnit();
+
+    void ReturnToSupplier();
+
+private:
+    RChannelBuffer*                    m_buffer;
+    RChannelBufferPrefetchInfo*        m_prefetchCookie;
+};
+
+class RChannelProcessRequest : public WorkRequest
+{
+public:
+    RChannelProcessRequest(RChannelReaderImpl* parent,
+                           RChannelItemArrayReaderHandler* handler,
+                           void* cancelCookie);
+    ~RChannelProcessRequest();
+
+    void Process();
+    bool ShouldAbort();
+
+    void SetItemArray(RChannelItemArray* itemArray);
+    RChannelItemArray* GetItemArray();
+    RChannelItemArrayReaderHandler* GetHandler();
+    void* GetCookie();
+
+    void Cancel();
+
+private:
+    LONG                             m_aborted;
+    RChannelReaderImpl*              m_parent;
+
+    RChannelItemArrayRef             m_itemArray;
+    RChannelItemArrayReaderHandler*  m_handler;
+    void*                            m_cookie;
+};
+
+class RChannelMarshalRequest : public WorkRequest
+{
+public:
+    RChannelMarshalRequest(RChannelSerializedWriter* parent);
+
+    void Process();
+    bool ShouldAbort();
+
+private:
+    RChannelSerializedWriter*                m_parent;
+};
+
+class RChannelParseRequest : public WorkRequest
+{
+public:
+    RChannelParseRequest(RChannelBufferQueue* parent,
+                         bool useNewBuffer);
+
+    void Process();
+    bool ShouldAbort();
+
+private:
+    RChannelBufferQueue*                m_parent;
+    bool                                m_useNewBuffer;
+};
+
+class RChannelReaderSyncWaiter : public RChannelItemArrayReaderHandlerImmediate
+{
+public:
+    RChannelReaderSyncWaiter(RChannelReaderImpl* parent,
+                             HANDLE event,
+                             RChannelItemArrayRef* itemDstArray);
+    ~RChannelReaderSyncWaiter();
+
+    void ProcessItemArray(RChannelItemArray* itemArray);
+
+private:
+    RChannelReaderImpl*       m_parent;
+    HANDLE                    m_event;
+    RChannelItemArrayRef*     m_itemDstArray;
+};

DryadVertex/VertexHost/system/channel/src/channelitem.cpp

@@ -0,0 +1,343 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#include <DrExecution.h>
+#include <channelitem.h>
+#include <dryadmetadata.h>
+#include <dryaderrordef.h>
+
+#pragma unmanaged
+
+
+DrResettableMemoryReader::
+    DrResettableMemoryReader(DrMemoryBuffer* pMemoryBuffer) :
+        DrMemoryBufferReader(pMemoryBuffer)
+{
+}
+
+void DrResettableMemoryReader::ResetToBufferOffset(Size_t offset)
+{
+    ResetMemoryReader();
+    SetBufferOffset(offset);
+}
+
+bool RChannelItem::IsTerminationItem(RChannelItemType type)
+{
+    return (type == RChannelItem_Restart ||
+            type == RChannelItem_Abort ||
+            type == RChannelItem_ParseError ||
+            type == RChannelItem_MarshalError ||
+            type == RChannelItem_EndOfStream);
+}
+
+RChannelItem::RChannelItem(RChannelItemType type)
+{
+    m_type = type;
+    m_dataSequenceNumber = s_invalidSequenceNumber;
+    m_deliverySequenceNumber = s_invalidSequenceNumber;
+}
+
+RChannelItem::~RChannelItem()
+{
+}
+
+void RChannelItem::Clone(RChannelItemRef* pClonedItem)
+{
+    DrLogA( "Clone method not implemented");
+    *pClonedItem = NULL;
+}
+
+UInt64 RChannelItem::GetNumberOfSubItems() const
+{
+    /* the base class includes marker items and by default has no
+       subitems */
+    return 0;
+}
+
+void RChannelItem::TruncateSubItems(UInt64 numberOfSubItems)
+{
+    /* the base class includes marker items and by default has no
+       subitems */
+    LogAssert(false);
+}
+
+UInt64 RChannelItem::GetItemSize() const
+{
+    /* the base class includes marker items and by default marshals to
+       zero size */
+    return 0;
+}
+
+RChannelItemType RChannelItem::GetType()
+{
+    return m_type;
+}
+
+UInt64 RChannelItem::GetDataSequenceNumber()
+{
+    return m_dataSequenceNumber;
+}
+
+void RChannelItem::SetDataSequenceNumber(UInt64 dataSequenceNumber)
+{
+    m_dataSequenceNumber = dataSequenceNumber;
+}
+
+UInt64 RChannelItem::GetDeliverySequenceNumber()
+{
+    return m_deliverySequenceNumber;
+}
+
+void RChannelItem::SetDeliverySequenceNumber(UInt64 deliverySequenceNumber)
+{
+    m_deliverySequenceNumber = deliverySequenceNumber;
+}
+
+DrError RChannelItem::DeSerialize(DrResettableMemoryReader* reader,
+                                  Size_t availableSize)
+{
+    DrLogA("Default DeSerialize method cannot be called on RChannelItem");
+    return DrError_Fail;
+}
+
+DrError RChannelItem::
+    DeSerializePartial(DrResettableMemoryReader* reader,
+                       Size_t availableSize)
+{
+    /* by default, any partial buffer signifies an error */
+    return DryadError_ItemParseError;
+}
+
+DrError RChannelItem::Serialize(ChannelMemoryBufferWriter* writer)
+{
+    DrLogA("Default Serialize method cannot be called on RChannelItem");
+    return DrError_Fail;
+}
+
+DryadMetaData* RChannelItem::GetMetaData()
+{
+    return m_metaData.Ptr();
+}
+
+void RChannelItem::ReplaceMetaData(DryadMetaData* metaData)
+{
+    m_metaData.Set(metaData);
+}
+
+DrError RChannelItem::GetErrorFromItem()
+{
+    if (m_metaData.Ptr() != NULL)
+    {
+        DryadMTagDrError* tag =
+            m_metaData.Ptr()->LookUpDrErrorTag(Prop_Dryad_ErrorCode);
+        if (tag != NULL)
+        {
+            return tag->GetDrError();
+        }
+    }
+
+    DrError err;
+    switch (m_type)
+    {
+    case RChannelItem_Data:
+        err = DrError_OK;
+        break;
+
+    case RChannelItem_BufferHole:
+        err = DryadError_BufferHole;
+        break;
+
+    case RChannelItem_ItemHole:
+        err = DryadError_ItemHole;
+        break;
+
+    case RChannelItem_EndOfStream:
+        err = DrError_EndOfStream;
+        break;
+
+    case RChannelItem_Restart:
+        err = DryadError_ChannelRestart;
+        break;
+
+    case RChannelItem_Abort:
+        err = DryadError_ChannelAbort;
+        break;
+
+    case RChannelItem_ParseError:
+        err = DryadError_ItemParseError;
+        break;
+
+    case RChannelItem_MarshalError:
+        err = DryadError_ItemMarshalError;
+        break;
+
+    default:
+        LogAssert(false);
+        err = DrError_InvalidParameter;
+    }
+
+    return err;
+}
+ 
+RChannelMarkerItem::RChannelMarkerItem(RChannelItemType type) :
+    RChannelItem(type)
+{
+}
+
+RChannelMarkerItem::~RChannelMarkerItem()
+{
+}
+
+RChannelMarkerItem* RChannelMarkerItem::Create(RChannelItemType type,
+                                               bool withMetaData)
+{
+    RChannelMarkerItem* item = new RChannelMarkerItem(type);
+    if (withMetaData)
+    {
+        DryadMetaDataRef emptyMetaData;
+        DryadMetaData::Create(&emptyMetaData);
+        item->ReplaceMetaData(emptyMetaData);
+    }
+    return item;
+}
+
+void RChannelMarkerItem::Clone(RChannelItemRef* pClonedItem)
+{
+    RChannelMarkerItem* clone = new RChannelMarkerItem(GetType());
+    clone->ReplaceMetaData(GetMetaData());
+    pClonedItem->Attach(clone);
+}
+
+//
+// Create a custom error item with specified type and error code
+//
+RChannelItem* RChannelMarkerItem::CreateErrorItem(RChannelItemType itemType,
+                                                  DrError errorCode)
+{
+    RChannelItem* item = Create(itemType, true);
+    DryadMetaData* m = item->GetMetaData();
+    DryadMTagRef tag;
+    tag.Attach(DryadMTagDrError::Create(Prop_Dryad_ErrorCode, errorCode));
+    m->Append(tag, false);
+    return item;
+}
+
+RChannelItem* RChannelMarkerItem::
+    CreateErrorItemWithDescription(RChannelItemType itemType,
+                                   DrError errorCode,
+                                   const char* errorDescription)
+{
+    RChannelItem* item = Create(itemType, true);
+    DryadMetaData* m = item->GetMetaData();
+    DryadMTagRef tag;
+
+    tag.Attach(DryadMTagDrError::Create(Prop_Dryad_ErrorCode, errorCode));
+    m->Append(tag, false);
+
+    tag.Attach(DryadMTagString::Create(Prop_Dryad_ErrorString,
+                                       errorDescription));
+    m->Append(tag, false);
+
+    return item;
+}
+
+
+RChannelDataItem::RChannelDataItem() : RChannelItem(RChannelItem_Data)
+{
+}
+
+RChannelDataItem::~RChannelDataItem()
+{
+}
+
+UInt64 RChannelDataItem::GetNumberOfSubItems() const
+{
+    /* by default an item is indivisible, i.e. has one subitem */
+    return 1;
+}
+
+UInt64 RChannelDataItem::GetItemSize() const
+{
+    /* the base class doesn't know the size, so just return 1 */
+    return 1;
+}
+
+RChannelItemArray::RChannelItemArray()
+{
+    m_numberOfItems = 0;
+    m_baseItemArray = NULL;
+    m_itemArray = NULL;
+}
+
+RChannelItemArray::~RChannelItemArray()
+{
+    delete [] m_baseItemArray;
+}
+
+void RChannelItemArray::SetNumberOfItems(UInt32 numberOfItems)
+{
+    m_numberOfItems = numberOfItems;
+    delete [] m_baseItemArray;
+    m_baseItemArray = new RChannelItemRef [m_numberOfItems];
+    m_itemArray = m_baseItemArray;
+}
+
+void RChannelItemArray::ExtendNumberOfItems(UInt32 numberOfItems)
+{
+    if (m_numberOfItems < numberOfItems)
+    {
+        RChannelItemRef* newArray = new RChannelItemRef[numberOfItems];
+        LogAssert(newArray != NULL);
+
+        UInt32 i;
+        for (i=0; i<m_numberOfItems; ++i)
+        {
+            newArray[i].TransferFrom(m_itemArray[i]);
+        }
+
+        delete [] m_baseItemArray;
+        m_baseItemArray = newArray;
+        m_itemArray = m_baseItemArray;
+        m_numberOfItems = numberOfItems;
+    }
+}
+
+UInt32 RChannelItemArray::GetNumberOfItems()
+{
+    return m_numberOfItems;
+}
+
+RChannelItemRef* RChannelItemArray::GetItemArray()
+{
+    return m_itemArray;
+}
+
+void RChannelItemArray::TruncateToSize(UInt32 size)
+{
+    LogAssert(size <= m_numberOfItems);
+    m_numberOfItems = size;
+}
+
+void RChannelItemArray::DiscardPrefix(UInt32 prefix)
+{
+    LogAssert(prefix <= m_numberOfItems);
+    m_numberOfItems -= prefix;
+    m_itemArray += prefix;
+}

DryadVertex/VertexHost/system/channel/src/channelmarshaler.cpp

@@ -0,0 +1,115 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#include <channelmarshaler.h>
+#include <dryaderrordef.h>
+
+#pragma unmanaged
+
+
+RChannelItemMarshalerBase::RChannelItemMarshalerBase()
+{
+    m_maxMarshalBatchSize = 0;
+}
+
+RChannelItemMarshalerBase::~RChannelItemMarshalerBase()
+{
+}
+
+void RChannelItemMarshalerBase::Reset()
+{
+}
+
+void RChannelItemMarshalerBase::
+    SetMaxMarshalBatchSize(UInt32 maxMarshalBatchSize)
+{
+    m_maxMarshalBatchSize = maxMarshalBatchSize;
+}
+
+UInt32 RChannelItemMarshalerBase::GetMaxMarshalBatchSize()
+{
+    return m_maxMarshalBatchSize;
+}
+
+void RChannelItemMarshalerBase::SetMarshalerIndex(UInt32 index)
+{
+    m_index = index;
+}
+
+UInt32 RChannelItemMarshalerBase::GetMarshalerIndex()
+{
+    return m_index;
+}
+
+void RChannelItemMarshalerBase::SetMarshalerContext(RChannelContext* context)
+{
+    m_context = context;
+}
+
+RChannelContext* RChannelItemMarshalerBase::GetMarshalerContext()
+{
+    return m_context;
+}
+
+
+RChannelItemMarshaler::~RChannelItemMarshaler()
+{
+}
+
+RChannelStdItemMarshalerBase::~RChannelStdItemMarshalerBase()
+{
+}
+
+DrError RChannelStdItemMarshalerBase::
+    MarshalItem(ChannelMemoryBufferWriter* writer,
+                RChannelItem* item,
+                bool flush,
+                RChannelItemRef* pFailureItem)
+{
+    if (item->GetType() != RChannelItem_Data)
+    {
+        return MarshalMarker(writer, item, flush, pFailureItem);
+    }
+
+    DrError err = item->Serialize(writer);
+
+    if (err != DrError_OK && err != DrError_IncompleteOperation)
+    {
+        pFailureItem->Attach(RChannelMarkerItem::
+                             CreateErrorItem(RChannelItem_MarshalError,
+                                             err));
+        return DryadError_ChannelAbort;
+    }
+
+    return err;
+}
+
+DrError RChannelStdItemMarshalerBase::
+    MarshalMarker(ChannelMemoryBufferWriter* writer,
+                  RChannelItem* item,
+                  bool flush,
+                  RChannelItemRef* pFailureItem)
+{
+    return DrError_OK;
+}
+
+RChannelStdItemMarshaler::~RChannelStdItemMarshaler()
+{
+}

DryadVertex/VertexHost/system/channel/src/channelparser.cpp

@@ -0,0 +1,879 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#include <channelparser.h>
+#include <channelbuffer.h>
+#include <channelmemorybuffers.h>
+#include <dryadmetadata.h>
+#include <dryadtagsdef.h>
+
+#pragma unmanaged
+
+
+void RChannelBufferRecord::SetData(DrMemoryBuffer* data)
+{
+    m_buffer = data;
+}
+
+DrMemoryBuffer* RChannelBufferRecord::GetData() const
+{
+    return m_buffer;
+}
+
+
+RChannelBufferItem::RChannelBufferItem(DrMemoryBuffer* buffer)
+{
+    SetNumberOfRecords(1);
+    GetRecordArray()[0].SetData(buffer);
+}
+
+RChannelBufferItem* RChannelBufferItem::Create(DrMemoryBuffer* buffer)
+{
+    return new RChannelBufferItem(buffer);
+}
+
+DrMemoryBuffer* RChannelBufferItem::GetData() const
+{
+    return GetRecordArray()[0].GetData();
+}
+
+UInt64 RChannelBufferItem::GetItemSize() const
+{
+    return GetData()->GetAvailableSize();
+}
+
+
+RChannelItemParserBase::RChannelItemParserBase()
+{
+    m_maxParseBatchSize = 0;
+    m_index = 0;
+}
+
+RChannelItemParserBase::~RChannelItemParserBase()
+{
+}
+
+void RChannelItemParserBase::SetMaxParseBatchSize(UInt32 maxParseBatchSize)
+{
+    m_maxParseBatchSize = maxParseBatchSize;
+}
+
+UInt32 RChannelItemParserBase::GetMaxParseBatchSize()
+{
+    return m_maxParseBatchSize;
+}
+
+void RChannelItemParserBase::SetParserIndex(UInt32 index)
+{
+    m_index = index;
+}
+
+UInt32 RChannelItemParserBase::GetParserIndex()
+{
+    return m_index;
+}
+
+void RChannelItemParserBase::SetParserContext(RChannelContext* context)
+{
+    m_context = context;
+}
+
+RChannelContext* RChannelItemParserBase::GetParserContext()
+{
+    return m_context;
+}
+
+
+RChannelRawItemParser::~RChannelRawItemParser()
+{
+}
+
+
+RChannelItemTransformerBase::~RChannelItemTransformerBase()
+{
+}
+
+void RChannelItemTransformerBase::
+    InitializeTransformer(RChannelItemParserBase* parent,
+                          DVErrorReporter* errorReporter)
+{
+}
+
+void RChannelItemTransformerBase::
+    TransformItem(RChannelItemRef& inputItem,
+                  SyncItemWriterBase* writer,
+                  DVErrorReporter* errorReporter)
+{
+    writer->WriteItemSyncConsumingReference(inputItem);
+}
+
+void RChannelItemTransformerBase::
+    FlushTransformer(SyncItemWriterBase* writer,
+                     DVErrorReporter* errorReporter)
+{
+}
+
+void RChannelItemTransformerBase::
+    ReportTransformerErrorItem(RChannelItemRef& errorItem,
+                               DVErrorReporter* errorReporter)
+{
+}
+
+
+RChannelItemTransformer::~RChannelItemTransformer()
+{
+}
+
+
+RChannelTransformerParserBase::RChannelTransformerParserBase()
+{
+    m_transformedAny = false;
+}
+
+RChannelTransformerParserBase::~RChannelTransformerParserBase()
+{
+    LogAssert(m_itemList.IsEmpty());
+}
+
+void RChannelTransformerParserBase::
+    SetTransformer(RChannelItemTransformerBase* transformer)
+{
+    m_transformer = transformer;
+}
+
+RChannelItemTransformerBase* RChannelTransformerParserBase::GetTransformer()
+{
+    return m_transformer;
+}
+
+RChannelItem* RChannelTransformerParserBase::
+    RawParseItem(bool restartParser,
+                 RChannelBuffer* inData,
+                 RChannelBufferPrefetchInfo** outPrefetchCookie)
+{
+    *outPrefetchCookie = NULL;
+
+    if (restartParser)
+    {
+        if (m_transformedAny)
+        {
+            DrLogI("Flushing transformer for restart");
+
+            m_transformer->FlushTransformer(this, this);
+            /* clear any errors */
+            ReportError(DrError_OK, (DryadMetaData*) NULL);
+            
+            RChannelItemRef item;
+            while (m_itemList.IsEmpty() == false)
+            {
+                /* remove the saved item from the list; it will be
+                   garbage collected when the refcount goes out of
+                   scope */
+                item.Attach(m_itemList.CastOut(m_itemList.RemoveHead()));
+            }
+
+            m_transformedAny = false;
+        }
+        else
+        {
+            LogAssert(m_itemList.IsEmpty());
+            m_transformer->InitializeTransformer(this, this);
+        }
+    }
+
+    if (m_itemList.IsEmpty() == false)
+    {
+        LogAssert(m_transformedAny);
+        LogAssert(inData == NULL);
+        RChannelItem* item = m_itemList.CastOut(m_itemList.RemoveHead());
+        return item;
+    }
+
+    if (inData == NULL)
+    {
+        LogAssert(m_transformedAny);
+        /* we have sent back all the items that got written from the
+           last buffer, but the base class doesn't know this yet, so
+           inform it by returning NULL */
+        return NULL;
+    }
+
+    m_transformedAny = true;
+
+    RChannelBufferType bType = inData->GetType();
+    if (bType == RChannelBuffer_Hole ||
+        bType == RChannelBuffer_EndOfStream)
+    {
+        RChannelBufferMarker* mBuffer =
+            dynamic_cast<RChannelBufferMarker*>(inData);
+        LogAssert(mBuffer != NULL);
+        RChannelItemRef markerItem = mBuffer->GetItem();
+
+        if (bType == RChannelBuffer_Hole)
+        {
+            m_transformer->ReportTransformerErrorItem(markerItem, this);
+        }
+
+        m_transformer->FlushTransformer(this, this);
+
+        if (!NoError())
+        {
+            markerItem.
+                Attach(RChannelMarkerItem::Create(RChannelItem_ParseError,
+                                                  false));
+            markerItem->ReplaceMetaData(GetErrorMetaData());
+        }
+
+        m_itemList.InsertAsTail(m_itemList.CastIn(markerItem.Detach()));
+    }
+    else
+    {
+        LogAssert(bType == RChannelBuffer_Data);
+        RChannelBufferData* dBuffer =
+            dynamic_cast<RChannelBufferData*>(inData);
+        LogAssert(dBuffer != NULL);
+
+        DryadLockedMemoryBuffer* block = dBuffer->GetData();
+        LogAssert(block->GetAvailableSize() > 0);
+        LogAssert(block->IsGrowable() == false);
+
+        RChannelItemRef dataItem;
+        dataItem.Attach(RChannelBufferItem::Create(block));
+
+        m_transformer->TransformItem(dataItem, this, this);
+
+        if (!NoError())
+        {
+            RChannelItem* errorItem;
+
+            errorItem =
+                RChannelMarkerItem::Create(RChannelItem_ParseError, false);
+            errorItem->ReplaceMetaData(GetErrorMetaData());
+
+            m_itemList.InsertAsTail(m_itemList.CastIn(errorItem));
+        }
+    }
+
+    if (m_itemList.IsEmpty())
+    {
+        return NULL;
+    }
+    else
+    {
+        RChannelItem* item = m_itemList.CastOut(m_itemList.RemoveHead());
+        return item;
+    }
+}
+
+void RChannelTransformerParserBase::
+    WriteItemSyncConsumingReference(RChannelItemRef& item)
+{
+    m_itemList.InsertAsTail(m_itemList.CastIn(item.Detach()));
+}
+
+DrError RChannelTransformerParserBase::GetWriterStatus()
+{
+    return DrError_OK;
+}
+
+
+RChannelItemParserNoRefImpl::RChannelItemParserNoRefImpl()
+{
+    m_needsReset = true;
+}
+
+RChannelItemParserNoRefImpl::~RChannelItemParserNoRefImpl()
+{
+    ResetParserInternal();
+}
+
+void RChannelItemParserNoRefImpl::ResetParser()
+{
+}
+
+RChannelItem* RChannelItemParserNoRefImpl::
+    ParsePartialItem(ChannelDataBufferList* bufferList,
+                     Size_t startOffset,
+                     RChannelBufferMarker*
+                     markerBuffer)
+{
+    return NULL;
+}
+
+void RChannelItemParserNoRefImpl::ResetParserInternal()
+{
+    m_savedItem = NULL;
+
+    DrBListEntry* listEntry = m_bufferList.GetHead();
+    while (listEntry != NULL)
+    {
+        RChannelBufferData* buffer = m_bufferList.CastOut(listEntry);
+        listEntry = m_bufferList.GetNext(listEntry);
+        m_bufferList.Remove(m_bufferList.CastIn(buffer));
+        buffer->DecRef();
+    }
+
+    m_bufferListStartOffset = 0;
+
+    m_needsData = true;
+}
+
+void RChannelItemParserNoRefImpl::DiscardBufferPrefix(Size_t discardLength)
+{
+    while (discardLength > 0)
+    {
+        LogAssert(m_bufferList.IsEmpty() == false);
+        RChannelBufferData* buffer =
+            m_bufferList.CastOut(m_bufferList.GetHead());
+        DryadLockedMemoryBuffer* mBuffer = buffer->GetData();
+
+        LogAssert(mBuffer->GetAvailableSize() > m_bufferListStartOffset);
+        Size_t tailLength =
+            mBuffer->GetAvailableSize() - m_bufferListStartOffset;
+
+        if (discardLength >= tailLength)
+        {
+            discardLength -= tailLength;
+            m_bufferList.Remove(m_bufferList.CastIn(buffer));
+            buffer->DecRef();
+            m_bufferListStartOffset = 0;
+        }
+        else
+        {
+            m_bufferListStartOffset += discardLength;
+            discardLength = 0;
+        }
+    }
+}
+
+RChannelItem* RChannelItemParserNoRefImpl::
+    DealWithPartialBuffer(RChannelBufferMarker* mBuffer)
+{
+    RChannelItem* item;
+
+    item = ParsePartialItem(&m_bufferList, m_bufferListStartOffset,
+                            mBuffer);
+
+    if (m_bufferList.IsEmpty())
+    {
+        LogAssert(m_bufferListStartOffset == 0);
+    }
+    else
+    {
+        DrBListEntry* listEntry = m_bufferList.GetHead();
+        while (listEntry != NULL)
+        {
+            RChannelBufferData* buffer = m_bufferList.CastOut(listEntry);
+            listEntry = m_bufferList.GetNext(listEntry);
+            m_bufferList.Remove(m_bufferList.CastIn(buffer));
+            buffer->DecRef();
+        }
+
+        m_bufferListStartOffset = 0;
+    }
+
+    RChannelItemRef markerItem = mBuffer->GetItem();
+    LogAssert(markerItem != NULL);
+
+    if (mBuffer->GetType() == RChannelBuffer_Hole)
+    {
+        LogAssert(markerItem->GetType() == RChannelItem_BufferHole);
+    }
+    else
+    {
+        LogAssert(markerItem->GetType() == RChannelItem_EndOfStream);
+    }
+
+    if (item == NULL)
+    {
+        item = markerItem.Detach();
+    }
+    else
+    {
+        m_savedItem = markerItem;
+    }
+
+    return item;
+}
+
+RChannelItem* RChannelItemParserNoRefImpl::
+    RawParseItem(bool restartParser,
+                 RChannelBuffer* inData,
+                 RChannelBufferPrefetchInfo** outPrefetchCookie)
+{
+    *outPrefetchCookie = NULL;
+
+    if (m_needsReset)
+    {
+        LogAssert(restartParser == true);
+    }
+
+    if (restartParser)
+    {
+        // todo: remove comment if not logging
+//         DrLogD( "resetting parser");
+
+        ResetParser();
+
+        ResetParserInternal();
+    }
+
+    LogAssert(m_needsData == (inData != NULL));
+
+    RChannelItem* item = NULL;
+
+    if (inData != NULL)
+    {
+        LogAssert(m_savedItem == NULL);
+
+        RChannelBufferType bType = inData->GetType();
+        if (bType == RChannelBuffer_Hole ||
+            bType == RChannelBuffer_EndOfStream)
+        {
+            item = DealWithPartialBuffer((RChannelBufferMarker *) inData);
+        }
+        else
+        {
+            LogAssert(bType == RChannelBuffer_Data);
+            RChannelBufferData* dBuffer = (RChannelBufferData *) inData;
+            dBuffer->IncRef();
+            DryadLockedMemoryBuffer* block = dBuffer->GetData();
+            LogAssert(block->GetAvailableSize() > 0);
+            LogAssert(block->IsGrowable() == false);
+            m_bufferList.InsertAsTail(m_bufferList.CastIn(dBuffer));
+        }
+    }
+
+    if (item == NULL)
+    {
+        if (m_savedItem != NULL)
+        {
+            item = m_savedItem.Detach();
+        }
+        else
+        {
+            Size_t itemLength = 0;
+
+            if (m_bufferList.IsEmpty() == false)
+            {
+                item = ParseNextItem(&m_bufferList, m_bufferListStartOffset,
+                                     &itemLength);
+            }
+
+            if (item == NULL)
+            {
+                LogAssert(itemLength == 0);
+            }
+            else
+            {
+                DiscardBufferPrefix(itemLength);
+            }
+        }
+    }
+
+    if (item == NULL)
+    {
+        m_needsReset = false;
+        m_needsData = true;
+    }
+    else
+    {
+        m_needsReset = RChannelItem::IsTerminationItem(item->GetType());
+        m_needsData = false;
+    }
+
+    return item;
+}
+
+RChannelItemParser::~RChannelItemParser()
+{
+}
+
+
+RChannelStdItemParserNoRefImpl::
+    RChannelStdItemParserNoRefImpl(DObjFactoryBase* factory)
+{
+    m_factory = factory;
+}
+
+RChannelStdItemParserNoRefImpl::~RChannelStdItemParserNoRefImpl()
+{
+}
+
+void RChannelStdItemParserNoRefImpl::ResetParser()
+{
+    m_pendingErrorItem = NULL;
+}
+
+RChannelItem* RChannelStdItemParserNoRefImpl::
+    ParseNextItem(ChannelDataBufferList* bufferList,
+                  Size_t startOffset,
+                  Size_t* pOutLength)
+{
+    if (m_pendingErrorItem != NULL)
+    {
+        return m_pendingErrorItem.Detach();
+    }
+
+    LogAssert(bufferList->IsEmpty() == false);
+
+    RChannelBufferData* tailBuffer =
+        bufferList->CastOut(bufferList->GetTail());
+    Size_t tailBufferSize =
+        tailBuffer->GetData()->GetAvailableSize();
+
+    DrRef<DrMemoryBuffer> buffer;
+    buffer.Attach(new RChannelReaderBuffer(bufferList,
+                                           startOffset,
+                                           tailBufferSize));
+
+    DrResettableMemoryReader reader(buffer);
+
+    RChannelItem* item = (RChannelItem *) m_factory->AllocateObjectUntyped();
+    DrError err = item->DeSerialize(&reader, buffer->GetAvailableSize());
+
+    if (err == DrError_OK)
+    {
+        *pOutLength = reader.GetBufferOffset();
+        return item;
+    }
+
+    m_factory->FreeObjectUntyped(item);
+
+    if (err == DrError_EndOfStream)
+    {
+        return NULL;
+    }
+    else
+    {
+        return RChannelMarkerItem::CreateErrorItem(RChannelItem_ParseError,
+                                                   err);
+    }
+}
+
+RChannelItem* RChannelStdItemParserNoRefImpl::
+    ParsePartialItem(ChannelDataBufferList* bufferList,
+                     Size_t startOffset,
+                     RChannelBufferMarker*
+                     markerBuffer)
+{
+    if (m_pendingErrorItem != NULL)
+    {
+        return m_pendingErrorItem.Detach();
+    }
+
+    if (bufferList->IsEmpty())
+    {
+        return NULL;
+    }
+
+    RChannelBufferData* tailBuffer =
+        bufferList->CastOut(bufferList->GetTail());
+    Size_t tailBufferSize =
+        tailBuffer->GetData()->GetAvailableSize();
+
+    DrRef<DrMemoryBuffer> buffer;
+    buffer.Attach(new RChannelReaderBuffer(bufferList,
+                                           startOffset,
+                                           tailBufferSize));
+
+    DrResettableMemoryReader reader(buffer);
+
+    RChannelItem* item = (RChannelItem *) m_factory->AllocateObjectUntyped();
+    DrError err = item->DeSerializePartial(&reader,
+                                           buffer->GetAvailableSize());
+
+    if (err == DrError_OK)
+    {
+        return item;
+    }
+
+    m_factory->FreeObjectUntyped(item);
+
+    if (err == DrError_EndOfStream)
+    {
+        return NULL;
+    }
+    else
+    {
+        return RChannelMarkerItem::CreateErrorItem(RChannelItem_ParseError,
+                                                   err);
+    }
+}
+
+RChannelStdItemParser::RChannelStdItemParser(DObjFactoryBase* factory) :
+    RChannelStdItemParserNoRefImpl(factory)
+{
+}
+
+RChannelStdItemParser::~RChannelStdItemParser()
+{
+}
+
+
+RChannelLengthDelimitedItemParserNoRefImpl::
+    RChannelLengthDelimitedItemParserNoRefImpl()
+{
+    ResetParser();
+}
+
+RChannelLengthDelimitedItemParserNoRefImpl::
+    ~RChannelLengthDelimitedItemParserNoRefImpl()
+{
+}
+
+void RChannelLengthDelimitedItemParserNoRefImpl::ResetParser()
+{
+    m_itemLength = 0;
+    m_accumulatedLength = 0;
+}
+
+void RChannelLengthDelimitedItemParserNoRefImpl::
+    AddMetaData(RChannelItem* item,
+                ChannelDataBufferList*
+                bufferList,
+                Size_t startOffset,
+                Size_t endOffset)
+{
+    LogAssert(bufferList->IsEmpty() == false);
+
+    RChannelBufferData* buffer = bufferList->CastOut(bufferList->GetHead());
+    DryadMetaDataRef startMetaData;
+    buffer->GetOffsetMetaData(true, startOffset, &startMetaData);
+
+    buffer = bufferList->CastOut(bufferList->GetTail());
+    DryadMetaDataRef endMetaData;
+    buffer->GetOffsetMetaData(false, endOffset, &endMetaData);
+
+    DryadMetaData* m = item->GetMetaData();
+    /* this is only called for items which already have metadata for
+       performance reasons */
+    LogAssert(m != NULL);
+
+    DryadMTagMetaDataRef tag;
+    bool brc;
+
+    tag = m->LookUpMetaDataTag(DryadTag_ItemStart);
+    if (tag == NULL)
+    {
+        tag.Attach(DryadMTagMetaData::Create(DryadTag_ItemStart,
+                                             startMetaData, true));
+        brc = m->Append(tag, false);
+        LogAssert(brc == true);
+    }
+    else
+    {
+        tag->GetMetaData()->AppendMetaDataTags(startMetaData, false);
+    }
+
+    tag = m->LookUpMetaDataTag(DryadTag_ItemEnd);
+    if (tag == NULL)
+    {
+        tag.Attach(DryadMTagMetaData::Create(DryadTag_ItemEnd,
+                                             endMetaData, true));
+        brc = m->Append(tag, false);
+        LogAssert(brc == true);
+    }
+    else
+    {
+        tag->GetMetaData()->AppendMetaDataTags(endMetaData, false);
+    }
+}
+
+RChannelItem* RChannelLengthDelimitedItemParserNoRefImpl::
+    FetchItem(ChannelDataBufferList*
+              bufferList,
+              Size_t startOffset,
+              Size_t tailBufferSize)
+{
+    Size_t tailGapLength = m_accumulatedLength - m_itemLength;
+
+    LogAssert(tailBufferSize > tailGapLength);
+
+    Size_t endOffset = tailBufferSize - tailGapLength;
+
+    RChannelReaderBuffer* itemBuffer =
+        new RChannelReaderBuffer(bufferList, startOffset, endOffset);
+    LogAssert(itemBuffer->GetAvailableSize() == m_itemLength);
+
+    RChannelItem* item = ParseItemWithLength(itemBuffer, m_itemLength);
+    LogAssert(item != NULL);
+    LogAssert(item->GetType() == RChannelItem_Data ||
+              item->GetType() == RChannelItem_ItemHole ||
+              item->GetType() == RChannelItem_ParseError);
+
+    itemBuffer->DecRef();
+
+    if (item->GetMetaData() != NULL)
+    {
+        AddMetaData(item, bufferList, startOffset, endOffset);
+    }
+
+    ResetParser();
+
+    return item;
+}
+
+RChannelItem* RChannelLengthDelimitedItemParserNoRefImpl::
+   MaybeFetchItem(ChannelDataBufferList* bufferList,
+                  Size_t startOffset,
+                  Size_t* pOutLength)
+{
+    /* make sure there are at least two items */
+    LogAssert(bufferList->GetHead() != bufferList->GetTail());
+
+    RChannelBufferData* buffer = bufferList->CastOut(bufferList->GetTail());
+    Size_t tailBufferSize = buffer->GetData()->GetAvailableSize();
+
+    LogAssert(m_accumulatedLength < m_itemLength);
+    m_accumulatedLength += tailBufferSize;
+
+    if (m_accumulatedLength >= m_itemLength)
+    {
+        Size_t thisLength = m_itemLength;
+        RChannelItem* item = FetchItem(bufferList, startOffset,
+                                       tailBufferSize);
+        *pOutLength = thisLength;
+        return item;
+    }
+    else
+    {
+        *pOutLength = 0;
+        return NULL;
+    }
+}
+
+RChannelItem* RChannelLengthDelimitedItemParserNoRefImpl::
+    ParseNextItem(ChannelDataBufferList* bufferList,
+                  Size_t startOffset,
+                  Size_t* pOutLength)
+{
+    if (m_itemLength > 0)
+    {
+        return MaybeFetchItem(bufferList, startOffset, pOutLength);
+    }
+    else
+    {
+        LogAssert(bufferList->IsEmpty() == false);
+
+        RChannelBufferData* buffer =
+            bufferList->CastOut(bufferList->GetTail());
+        Size_t tailBufferSize =
+            buffer->GetData()->GetAvailableSize();
+        LogAssert(bufferList->GetHead() != bufferList->GetTail() ||
+                  startOffset < tailBufferSize);
+
+        RChannelReaderBuffer* lengthCheckBuffer =
+            new RChannelReaderBuffer(bufferList, startOffset, tailBufferSize);
+
+        m_accumulatedLength = lengthCheckBuffer->GetAvailableSize();
+
+        RChannelItem* errorItem = NULL;
+        LengthStatus ls = GetNextItemLength(lengthCheckBuffer,
+                                            &m_itemLength,
+                                            &errorItem);
+
+        lengthCheckBuffer->DecRef();
+
+        switch (ls)
+        {
+        default:
+            LogAssert(false);
+            return NULL;
+
+        case LS_ParseError:
+            LogAssert(errorItem != NULL);
+            LogAssert(errorItem->GetType() == RChannelItem_ParseError);
+            LogAssert(errorItem->GetMetaData() != NULL);
+            AddMetaData(errorItem, bufferList, startOffset, tailBufferSize);
+            ResetParser();
+            *pOutLength = 0;
+            return errorItem;
+
+        case LS_NeedsData:
+            LogAssert(errorItem == NULL);
+            m_itemLength = 0;
+            *pOutLength = 0;
+            return NULL;
+
+        case LS_Ok:
+            LogAssert(errorItem == NULL);
+            LogAssert(m_itemLength > 0);
+
+            if (m_accumulatedLength >= m_itemLength)
+            {
+                Size_t thisLength = m_itemLength;
+                RChannelItem* item = FetchItem(bufferList, startOffset,
+                                               tailBufferSize);
+                *pOutLength = thisLength;
+                return item;
+            }
+            else
+            {
+                *pOutLength = 0;
+                return NULL;
+            }
+        }
+    }
+}
+
+RChannelItem* RChannelLengthDelimitedItemParserNoRefImpl::
+    ParsePartialItem(ChannelDataBufferList* bufferList,
+                     Size_t startOffset,
+                     RChannelBufferMarker*
+                     markerBuffer)
+{
+    if (bufferList->IsEmpty())
+    {
+        return NULL;
+    }
+    else
+    {
+        RChannelBufferData* buffer =
+            bufferList->CastOut(bufferList->GetTail());
+        Size_t tailBufferSize =
+            buffer->GetData()->GetAvailableSize();
+        RChannelItem* item =
+            RChannelMarkerItem::Create(RChannelItem_ItemHole, true);
+        AddMetaData(item, bufferList, startOffset, tailBufferSize);
+        ResetParser();
+        return item;
+    }
+}
+
+RChannelLengthDelimitedItemParser::~RChannelLengthDelimitedItemParser()
+{
+}
+
+DryadParserFactoryBase::~DryadParserFactoryBase()
+{
+}
+
+DryadParserFactory::~DryadParserFactory()
+{
+}
+
+DryadMarshalerFactoryBase::~DryadMarshalerFactoryBase()
+{
+}
+
+DryadMarshalerFactory::~DryadMarshalerFactory()
+{
+}

DryadVertex/VertexHost/system/channel/src/channelreader.h

@@ -0,0 +1,385 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+#pragma warning(disable:4512)  // KLUDGE -- build for now, fix later.
+
+#include <channelitem.h>
+#include <orderedsendlatch.h>
+#include <dryadlisthelper.h>
+#include <dryadeventcache.h>
+#include <channelinterface.h>
+#include <channelparser.h>
+#pragma warning(disable:4995)
+#include <map>
+
+class WorkQueue;
+class WorkRequest;
+
+class RChannelUnit;
+
+class RChannelParseRequest;
+class RChannelProcessRequest;
+class RChannelBufferBoundaryUnit;
+class RChannelItemUnit;
+class RChannelBuffer;
+class RChannelBufferReader;
+class RChannelBufferQueue;
+class RChannelBufferPrefetchInfo;
+
+typedef DryadBList<RChannelUnit> ChannelUnitList;
+
+typedef DryadBListDerived<RChannelProcessRequest,WorkRequest>
+    ChannelProcessRequestList;
+
+class RChannelReaderSupplier
+{
+public:
+    virtual ~RChannelReaderSupplier();
+    virtual void StartSupplier(RChannelBufferPrefetchInfo* prefetchCookie) = 0;
+    virtual void InterruptSupplier() = 0;
+    virtual void DrainSupplier(RChannelItem* drainItem) = 0;
+    virtual void CloseSupplier() = 0;
+};
+
+class RChannelReaderImpl : public RChannelReader
+{
+public:
+    virtual ~RChannelReaderImpl();
+
+    /* a client must call Start to cause the channel to start
+       generating items for the first time, or after a Drain has
+       completed before the channel is restarted.
+
+       prefetchCookie is passed to the byte-oriented buffer layer and
+       should be NULL for now.
+    */
+    void Start(RChannelBufferPrefetchInfo* prefetchCookie);
+
+    /* SupplyHandler passes a handler which will be "returned" via a
+       matching call to handler->ProcessItem. Multiple handlers may be
+       outstanding at a given time. The channel blocks when there are
+       no handlers available, so this is the primary flow control
+       mechanism to allow the reader to exert back-pressure on a
+       channel.
+
+       If the channel has not been started or has delivered a
+       termination item to the application since the last call to
+       Start, handler->ProcessItem will be called with value NULL on
+       the calling thread before SupplyHandler returns.
+
+       If there is an item waiting to be delivered and handler is an
+       RChannelItemReaderHandlerImmediate then handler will be called
+       back with the item on the calling thread before SupplyHandler
+       returns.
+
+       If handler is an RChannelItemReaderHandlerQueued it will never
+       be called back on the calling thread. If there is an item
+       waiting to be delivered, a processing request will be queued
+       for the item with handler, otherwise handler will be queued
+       waiting for the next item to be ready.
+
+       If thread B submits an RChannelItemReaderHandlerImmediate with
+       a call to SupplyHandler that is overlapped with thread A's call
+       to SupplyHandler, B's immediate handler may be called on A's
+       calling thread before A's call to SupplyHandler returns.
+
+       After a handler has been submitted to SupplyHandler with a
+       given value of cancelCookie, that handler is guaranteed to be
+       returned before any matching call to Cancel with the same value
+       of cancelCookie returns. If there is no item available at the
+       time the handler is cancelled it will be called with
+       NULL. cancelCookie may take any value, including NULL, however
+       it is safest to use NULL or the address of an object owned by
+       the caller. The cancellation mechanism is used internally by
+       the synchronous FetchNextItem call which uses an allocated heap
+       address as its cancelCookie. Using the heap address of an
+       object which has not been freed before the call to Cancel will
+       avoid any danger of a cookie collision.
+
+       For any item A which has already been returned via an async
+       handler or call to FetchNextItem when SupplyHandler is called
+       it is guaranteed that A's sequence number is less than or equal
+       to the sequence number of the item eventually returned on
+       handler. Beyond this constraint, if multiple handlers are
+       outstanding at once, or while calls to FetchNextItem are in
+       progress, the order of delivered items is undefined.
+     */
+    void SupplyHandler(RChannelItemArrayReaderHandler* handler,
+                       void* cancelCookie);
+
+    /* any handler passed to SupplyHandler with value cancelCookie is
+       guaranteed to be returned before a call to Cancel completes. */
+    void Cancel(void* cancelCookie);
+
+    /* FetchNextItem blocks waiting until an item is available, a
+       termination item is delivered on another thread, or the timeOut
+       interval has elapsed (timeOut can be DrTimeInterval_Infinite in
+       which case FetchNextItem will block indefinitely).
+
+       If the timeout expires FetchNextItem returns false and *outItem
+       is NULL. Otherwise FetchNextItem returns true and the returned
+       item is stored in *outItem. *outItem may be NULL even if
+       FetchNextItem returns true: this will happen if Start has not
+       been called or a termination item has already been delivered on
+       the channel since the last call to Start.
+     */
+    bool FetchNextItemArray(UInt32 maxListSize,
+                            RChannelItemArrayRef* itemArray,
+                            DrTimeInterval timeOut);
+
+    /* Instruct the channel to return all outstanding handlers via
+       their handler->ProcessItem callbacks in preparation for either
+       closing or restarting the channel. Once the Drain method
+       returns all outstanding handler callbacks will have completed
+       and all waiting calls to FetchNextItem will have been unblocked
+       (though of course they may not have returned to the calling
+       thread). For obvious reasons, Drain may not be called from a
+       handler's ProcessItem callback.
+
+       The drainItem may be of type RChannelItem_Abort,
+       RChannelItem_Restart, RChannelItem_EndOfStream or
+       RChannelItem_ParseError. If the channel is a pipe which has not
+       broken, the drainItem will be delivered to the remote end as
+       part of the shutdown procedure.
+     */
+    void Interrupt(RChannelItem* interruptItem);
+    void Drain();
+
+    /* return the drain item passed to the most recent call to Drain,
+       or NULL if Drain has not been called since the last call to
+       Start. The return value is undefined if this call overlaps
+       with a call to Start or Drain, and it is illegal to call
+       GetTerminationItems after Close has been called. */
+    void GetTerminationItems(RChannelItemRef* pWriterDrainItem,
+                             RChannelItemRef* pReaderDrainItem);
+
+    UInt64 GetDataSizeRead();
+
+    /* Close may only be called if Start has never been called, or if
+       Drain has completed since the last call to Start. Close must be
+       called before the RChannelReader is destroyed. After Close has
+       been called no other methods may be called on RChannelReader.
+    */
+    void Close();
+
+    /* Get/Set the URI of the channel. */
+    const char* GetURI();
+    void SetURI(const char* uri);
+
+protected:
+    RChannelReaderImpl();
+    void Initialize(RChannelReaderSupplier* supplier,
+                    WorkQueue* workQueue,
+                    bool lazyStart);
+
+private:
+    typedef std::multimap<void *, RChannelProcessRequest *>
+        CookieHandlerMap;
+    typedef std::map<void *, HANDLE>
+        CookieEventMap;
+
+    enum ReaderState {
+        RS_Stopped,
+        RS_Running,
+        RS_InterruptingSupplier,
+        RS_Stopping,
+        RS_Closed
+    };
+
+    bool IsRunning();
+
+    void FillEmptyHandlers(ChannelProcessRequestList* handlerDispatch);
+    void TransferWaitingItems(const char* caller,
+                              ChannelProcessRequestList* requestList,
+                              ChannelUnitList* returnUnitList);
+    void ReturnUnits(ChannelUnitList* unitList);
+    void DispatchRequests(const char* caller,
+                          ChannelProcessRequestList* requestList,
+                          ChannelUnitList* returnUnitList);
+    void AddUnitToQueue(const char* caller,
+                        RChannelUnit* unit,
+                        ChannelProcessRequestList* requestList,
+                        ChannelUnitList* returnUnitList);
+    void AddUnitList(ChannelUnitList* unitList);
+    void AddHandlerToQueue(const char* caller,
+                           RChannelProcessRequest* request,
+                           ChannelProcessRequestList* requestList,
+                           ChannelUnitList* returnUnitList);
+    void RemoveFromCancelMap(RChannelProcessRequest* request,
+                             void* cancelCookie);
+    void MaybeTriggerCancelEvent(void* cancelCookie);
+    void ProcessItemArrayRequest(RChannelProcessRequest* handler);
+    void AlertApplication(RChannelItem* item);
+    void ThreadSafeSetItemArray(RChannelItemArrayRef* dstItemArray,
+                                RChannelItemArray* srcItemArray);
+
+    WorkQueue*                      m_workQueue;
+
+    CookieHandlerMap                m_cookieMap;
+    CookieEventMap                  m_eventMap;
+    ChannelUnitList                 m_unitList;
+    ChannelProcessRequestList       m_handlerList;
+    RChannelInterruptHandler*       m_interruptHandler;
+
+    ReaderState                     m_state;
+    DryadOrderedSendLatch<ChannelProcessRequestList> m_sendLatch;
+    DryadOrderedSendLatch<ChannelUnitList> m_unitLatch;
+    HANDLE                          m_drainEvent;
+    HANDLE                          m_interruptEvent;
+    HANDLE                          m_startedSupplierEvent;
+    bool                            m_lazyStart;
+    bool                            m_startedSupplier;
+    DryadEventCache                 m_eventCache;
+    RChannelItemRef                 m_readerTerminationItem;
+    RChannelItemRef                 m_writerTerminationItem;
+    UInt64                          m_numberOfSubItemsRead;
+    UInt64                          m_dataSizeRead;
+
+    RChannelBufferPrefetchInfo*     m_prefetchCookie;
+    RChannelReaderSupplier*         m_supplier;
+
+    DrStr128                        m_uri;
+
+    CRITSEC                         m_baseDR;
+
+    friend class RChannelParseRequest;
+    friend class RChannelProcessRequest;
+    friend class RChannelBufferBoundaryRequest;
+    friend class RChannelReaderSyncWaiter;
+    friend class RChannelItemUnit;
+    friend class RChannelBufferQueue;
+    friend class RChannelFifoWriterBase;
+};
+
+class RChannelSerializedReader : public RChannelReaderImpl
+{
+public:
+    RChannelSerializedReader(RChannelBufferReader* bufferReader,
+                             RChannelItemParserBase* parser,
+                             UInt32 maxParseBatchSize,
+                             UInt32 maxOutstandingUnits,
+                             bool lazyStart,
+                             WorkQueue* workQueue);
+    ~RChannelSerializedReader();
+
+    bool GetTotalLength(UInt64* pLen);
+    bool GetExpectedLength(UInt64* pLen);
+    void SetExpectedLength(UInt64 expectedLength);
+    
+private:
+    UInt64                 m_expectedLength;
+    RChannelBufferQueue*   m_bufferQueue;
+};
+
+/* interface used to signal that a buffer has arrived and is ready for
+   reading */
+class RChannelBufferReaderHandler
+{
+public:
+    virtual ~RChannelBufferReaderHandler();
+
+    /* When an i/o completes on a buffer reader, the buffer is
+       delivered via this callback to the consumer.
+       buffer->ProcessingComplete should be called when the consumer
+       has finished using the data in the buffer. Buffers are
+       delivered in order, and to enforce this there will be at most
+       one call to ProcessBuffer in progress at a time on any given
+       channel.
+
+       If buffer->GetType() is a termination type
+       (RChannelBuffer::IsTerminationBuffer returns true) then no more
+       calls to ProcessBuffer will be made before
+       RChannelBufferReader::Drain has completed.
+
+       The completion callback mechanism is used to implement flow
+       control, as the buffer-oriented i/o will only allow the
+       consumer to hold a bounded number of outstanding buffers before
+       blocking further reads.
+    */
+    virtual void ProcessBuffer(RChannelBuffer* buffer) = 0;
+};
+
+/* base class to wrap byte-oriented read implementations */
+class RChannelBufferReader
+{
+public:
+    virtual ~RChannelBufferReader();
+
+    /* Instruct the i/o reader to start fetching buffers from the
+       start of the Channel and delivering them via handler.
+
+       prefetchCookie is an optional hint which may be used to
+       influence initial read buffer sizes or prefetching
+       behaviour. It is dependent on the implementation of the
+       underlying buffer-oriented i/o class and should be NULL for
+       now.
+     */
+    virtual void Start(RChannelBufferPrefetchInfo* prefetchCookie,
+                       RChannelBufferReaderHandler* handler) = 0;
+
+    /* Instruct the i/o reader to prepare to Drain the Channel of
+       buffers. After a call to Interrupt returns, the BufferReader
+       will not make any more calls to handler unless the Channel is
+       Drained and restarted.
+
+       A client which wants to abort or restart the Channel should
+       first call Interrupt (which guarantees no new buffers will be
+       delivered), then ensure that any outstanding buffers have been
+       returned via their completion handlers, then call Drain.
+     */
+    virtual void Interrupt() = 0;
+
+    virtual void FillInStatus(DryadChannelDescription* status);
+
+    /* Complete the synchronisation in the case of restarting or
+       closing the stream. Drain will not return until all outstanding
+       buffer completion handlers have been called.
+
+       drainItem will have type RChannelItem_Abort,
+       RChannelItem_Restart, RChannelItem_EndOfStream or
+       RChannelItem_ParseError and if the channel is a pipe it should
+       be communicated to the process at the remote end.
+
+       After Drain returns, Start and Close are the only legal method
+       calls.
+     */
+    virtual void Drain(RChannelItem* drainItem) = 0;
+
+    /* shut down the channel. After Close returns no further calls can
+       be made to this interface.
+    */
+    virtual void Close() = 0;
+
+    virtual bool GetTotalLength(UInt64* pLen) = 0;
+};
+
+class RChannelNullBufferReader :
+    public RChannelBufferReader, public RChannelBufferDefaultHandler
+{
+public:
+    void Start(RChannelBufferPrefetchInfo* prefetchCookie,
+               RChannelBufferReaderHandler* handler);
+    void Interrupt();
+    void Drain(RChannelItem* drainItem);
+    void Close();
+    void ReturnBuffer(RChannelBuffer* buffer);
+    bool GetTotalLength(UInt64* pLen);
+};

DryadVertex/VertexHost/system/channel/src/channelwriter.h

@@ -0,0 +1,318 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#pragma once
+#pragma warning(disable:4512)
+
+#include <channelinterface.h>
+#include <channelbuffer.h>
+#include <channelmarshaler.h>
+#include <orderedsendlatch.h>
+#include <dryadeventcache.h>
+
+class RChannelMarshalRequest;
+class RChannelBufferWriter;
+class WorkQueue;
+
+class RChannelBufferWriterHandler
+{
+public:
+    virtual ~RChannelBufferWriterHandler();
+
+    virtual void ProcessWriteCompleted(RChannelItemType status) = 0;
+};
+
+/*
+  The RChannelWriter is the primary mechanism for writing
+  application-specific structured items to an underlying byte-oriented
+  channel.
+
+  The application first calls Start, then sends a series of items,
+  primarily of type RChannelItem_Data, though markers may be
+  interspersed, followed by a termination item of type
+  RChannelItem_Restart, RChannelItem_Abort or
+  RChannelItem_EndOfStream. In the case of a pipe which has not
+  broken, the termination item will be sent to the remote end so that
+  the consuming process learns the reason for the pipe closure.
+
+  After sending a termination item no further items will be sent on
+  the channel until a call to Drain has completed. After this point
+  the channel may call Start again in the case of a restart, and start
+  sending items again.
+
+  Each item is marshaled into the byte-stream using an
+  application-specific marshaler object which must co-operate with the
+  application so that bare RChannelItem objects can be cast by it into
+  objects containing meaningful data.
+ */
+class RChannelSerializedWriter :
+    public RChannelWriter,
+    public RChannelBufferWriterHandler
+{
+public:
+    RChannelSerializedWriter(RChannelBufferWriter* writer,
+                             RChannelItemMarshalerBase* marshaler,
+                             bool breakBufferOnRecordBoundaries,
+                             UInt32 maxMarshalBatchSize,
+                             WorkQueue* workQueue);
+    ~RChannelSerializedWriter();
+
+    /* a client must call Start before writing items to the channel
+       the first time, or after a Drain has completed before writing
+       to a restarted channel.
+    */
+    void Start();
+
+    void WriteItemArray(RChannelItemArrayRef& itemArray,
+                        bool flushAfter,
+                        RChannelItemArrayWriterHandler* handler);
+
+    RChannelItemType WriteItemArraySync(RChannelItemArrayRef& itemArray,
+                                        bool flush,
+                                        RChannelItemArrayRef* pFailureArray);
+
+    virtual void Drain(DrTimeInterval timeOut,
+                       RChannelItemRef* pRemoteStatus);
+
+    void GetTerminationItems(RChannelItemRef* pWriterDrainItem,
+                             RChannelItemRef* pReaderDrainItem);
+
+    /* Close may not be called unless Start has never been called or
+       Drain has completed since the last call to Start. After Close
+       is called the channel may not be restarted. */
+    void Close();
+
+    /* this is the implementation of the RChannelBufferWriterHandler
+       interface and must not be called by clients. */
+    void ProcessWriteCompleted(RChannelItemType status);
+
+    /* Get/Set the URI of the channel. */
+    const char* GetURI();
+    void SetURI(const char* uri);
+
+    /* Get/set a hint about the total length the channel is expected
+       to be. Some channel implementations can use this to improve
+       write performance and decrease disk fragmentation. A value of 0
+       (the default) means that the size is unknown. */
+    UInt64 GetInitialSizeHint();
+    void SetInitialSizeHint(UInt64 hint);
+
+private:
+    class DummyItemHandler : public RChannelItemArrayWriterHandler
+    {
+    public:
+        void ProcessWriteArrayCompleted(RChannelItemType returnCode,
+                                        RChannelItemArray* failureArray);
+    };
+
+    class WriteRequest
+    {
+    public:
+        WriteRequest(RChannelItemArray* itemArray,
+                     bool flushAfter,
+                     RChannelItemArrayWriterHandler* handler);
+        ~WriteRequest();
+
+        void SetHandler(RChannelItemArrayWriterHandler* handler);
+        bool ShouldFlush();
+
+        RChannelItem* GetNextItem();
+        void SetSuccessItem();
+        void SetFailureItem(RChannelItem* marshalFailureItem, bool abort);
+        bool LastItem();
+        bool Completed();
+
+        void ProcessMarshalCompleted(RChannelItemType returnType);
+
+    private:
+        RChannelItemArrayRef             m_itemArray;
+        bool                             m_flushAfter;
+        RChannelItemArrayWriterHandler*  m_handler;
+        UInt32                           m_currentItem;
+        RChannelItemArrayRef             m_failureArray;
+        bool                             m_aborted;
+        DrBListEntry                     m_listPtr;
+        friend class DryadBList<WriteRequest>;
+    };
+
+    typedef DryadBList<WriteRequest> WriteRequestList;
+
+    class SyncHandler : public RChannelItemArrayWriterHandler
+    {
+    public:
+        SyncHandler();
+        ~SyncHandler();
+
+        void ProcessWriteArrayCompleted(RChannelItemType returnType,
+                                        RChannelItemArray* failureArray);
+
+        RChannelItemType GetStatusCode();
+        void GetFailureItemArray(RChannelItemArrayRef* pFailureArray);
+
+        void UseEvent(DryadHandleListEntry* event);
+        DryadHandleListEntry* GetEvent();
+
+        bool UsingEvent();
+        void Wait();
+
+    private:
+        LONG                        m_usingEvent;
+        DryadHandleListEntry*       m_event;
+        RChannelItemType            m_statusCode;
+        RChannelItemArrayRef        m_failureArray;
+    };
+
+    enum CWState {
+        CW_Closed,
+        CW_Empty,
+        CW_InWorkQueue,
+        CW_Marshaling,
+        CW_Blocking,
+        CW_Stopping,
+        CW_Stopped
+    };
+
+    void MakeCachedWriter();
+    Size_t DisposeOfCachedWriter();
+    void ReturnHandlers(WriteRequestList* completedList,
+                        RChannelItemType returnCode);
+    bool CheckForTerminationItem(RChannelItemArray* itemArray);
+    void RestorePreMarshalBuffers(Size_t preMarshalAvailableSize);
+    void CollapseToSingleBuffer();
+    void ShuffleBuffersOnRecordBoundaries(Size_t preMarshalAvailableSize);
+    RChannelItemType PerformSingleMarshal(WriteRequest* writeRequest);
+    bool PerformMarshal(WriteRequestList* pendingRequestList,
+                        WriteRequestList* completedRequestList);
+    void PostMarshal(WriteRequestList* pendingRequestList,
+                     WriteRequestList* completedRequestList,
+                     bool shouldBlock,
+                     SyncHandler* syncHandler);
+    bool SendCompletedBuffers(bool shouldFlush,
+                              RChannelItemType terminationType);
+    void MarshalItems();
+    void AcceptReturningHandlers(UInt32 handlerCount);
+
+    bool                                       m_breakBufferOnRecordBoundaries;
+    RChannelBufferWriter*                      m_writer;
+    RChannelItemMarshalerRef                   m_marshaler;
+    WorkQueue*                                 m_workQueue;
+
+    UInt32                                     m_maxMarshalBatchSize;
+
+    CWState                                    m_state;
+    /* the pending list is the writes which have been submitted but
+       not yet queued for the marshaler. */
+    WriteRequestList                           m_pendingList;
+    /* the blocked handler list is the writes which have been
+       marshaled and sent to the buffer writer and are waiting for the
+       channel to unblock */
+    WriteRequestList                           m_blockedHandlerList;
+    /* outstandingBuffers is the count of buffers which have been sent
+       to the buffer writer and not yet completed */
+    UInt32                                     m_outstandingBuffers;
+    /* outstandingBuffers is the count of handlers which have been
+       received from the client and not yet returned. */
+    UInt32                                     m_outstandingHandlers;
+    HANDLE                                     m_handlerReturnEvent;
+    bool                                       m_marshaledTerminationItem;
+    HANDLE                                     m_marshaledLastItemEvent;
+
+    DryadFixedBufferList                       m_bufferList;
+    ChannelMemoryBufferWriter*                 m_cachedWriter;
+    DryadOrderedSendLatch<WriteRequestList>    m_returnLatch;
+
+    DryadEventCache                            m_eventCache;
+
+    RChannelItemType                           m_channelTermination;
+    RChannelItemRef                            m_writerTerminationItem;
+    RChannelItemRef                            m_readerTerminationItem;
+
+    DrStr128                                   m_uri;
+
+    CRITSEC                                    m_baseDR;
+
+    friend class RChannelMarshalRequest;
+};
+
+class RChannelBufferWriter
+{
+public:
+    virtual ~RChannelBufferWriter();
+
+    virtual void Start() = 0;
+
+    virtual DryadFixedMemoryBuffer* GetNextWriteBuffer() = 0;
+    virtual DryadFixedMemoryBuffer* GetCustomWriteBuffer(Size_t bufferSize) = 0;
+
+    virtual bool WriteBuffer(DryadFixedMemoryBuffer* buffer,
+                             bool flushAfter,
+                             RChannelBufferWriterHandler* handler) = 0;
+
+    virtual void ReturnUnusedBuffer(DryadFixedMemoryBuffer* buffer) = 0;
+
+    virtual void WriteTermination(RChannelItemType reasonCode,
+                                  RChannelBufferWriterHandler* handler) = 0;
+
+    virtual void FillInStatus(DryadChannelDescription* status);
+
+    virtual void Drain(RChannelItemRef* pReturnItem) = 0;
+
+    /* shut down the channel. After Close returns no further calls can
+       be made to this interface.
+    */
+    virtual void Close() = 0;
+
+    /* Get/set a hint about the total length the channel is expected
+       to be. Some channel implementations can use this to improve
+       write performance and decrease disk fragmentation. A value of 0
+       (the default) means that the size is unknown. */
+    virtual UInt64 GetInitialSizeHint() = 0;
+    virtual void SetInitialSizeHint(UInt64 hint) = 0;
+};
+
+class RChannelNullWriter : public RChannelWriter
+{
+public:
+    RChannelNullWriter(const char* uri);
+    void Start();
+    void WriteItemArray(RChannelItemArrayRef& itemArray,
+                        bool flushAfter,
+                        RChannelItemArrayWriterHandler* handler);
+    RChannelItemType WriteItemArraySync(RChannelItemArrayRef& itemArray,
+                                        bool flush,
+                                        RChannelItemArrayRef* pFailureArray);
+    void Drain(DrTimeInterval timeOut,
+               RChannelItemRef* pRemoteStatus);
+    void GetTerminationItems(RChannelItemRef* pWriterDrainItem,
+                             RChannelItemRef* pReaderDrainItem);
+    void Close();
+
+    UInt64 GetInitialSizeHint();
+    void SetInitialSizeHint(UInt64 hint);
+    const char* GetURI();
+
+private:
+    bool              m_started;
+    RChannelItemRef   m_writeTerminationItem;
+    DrStr128          m_uri;
+    CRITSEC           m_baseDR;
+};

DryadVertex/VertexHost/system/channel/src/concreterchannelhelpers.h

@@ -0,0 +1,194 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "concreterchannel.h"
+#include "channelfifo.h"
+#include <workqueue.h>
+
+class RChannelThrottledStream
+{
+public:
+    ~RChannelThrottledStream();
+    virtual void OpenAfterThrottle() = 0;
+};
+
+class RChannelOpenThrottler
+{
+public:
+    class Dispatch : public WorkRequest
+    {
+    public:
+        Dispatch(RChannelThrottledStream* stream);
+
+        void Process();
+        bool ShouldAbort();
+
+    private:
+        RChannelThrottledStream*  m_stream;
+    };
+
+    RChannelOpenThrottler(UInt32 maxOpenFiles, WorkQueue* workQueue);
+    ~RChannelOpenThrottler();
+
+    bool QueueOpen(RChannelThrottledStream* stream);
+    void NotifyFileCompleted();
+
+private:
+    WorkQueue*                           m_workQueue;
+    UInt32                               m_maxOpenFiles;
+    UInt32                               m_openFileCount;
+    std::list<RChannelThrottledStream*>  m_blockedFileList;
+    CRITSEC                              m_baseDR;
+};
+
+class RChannelFifoHolder
+{
+public:
+    RChannelFifoHolder(const char* channelURI, UInt32 fifoLength,
+                       bool isReader, WorkQueue* workQueue);
+    ~RChannelFifoHolder();
+
+    RChannelFifo* GetFifo();
+
+    bool MakeReader();
+    bool MakeWriter();
+
+    bool DiscardReader();
+    bool DiscardWriter();
+
+private:
+    RChannelFifo*   m_fifo;
+    WorkQueue*      m_workQueue;
+    bool            m_madeReader;
+    bool            m_discardedReader;
+    bool            m_madeWriter;
+    bool            m_discardedWriter;
+
+    CRITSEC         m_atomic;
+};
+
+class RChannelFifoReaderHolder : public RChannelReaderHolder
+{
+public:
+    RChannelFifoReaderHolder(const char* channelURI,
+                             WorkQueue* workQueue,
+                             DVErrorReporter* errorReporter);
+    ~RChannelFifoReaderHolder();
+
+    RChannelReader* GetReader();
+    void FillInStatus(DryadInputChannelDescription* status);
+    void Close();
+
+private:
+    RChannelFifoReader* m_reader;
+};
+
+class RChannelFifoWriterHolder : public RChannelWriterHolder
+{
+public:
+    RChannelFifoWriterHolder(const char* channelURI,
+                             DVErrorReporter* errorReporter);
+    ~RChannelFifoWriterHolder();
+
+    RChannelWriter* GetWriter();
+    void FillInStatus(DryadOutputChannelDescription* status);
+    void Close();
+
+private:
+    RChannelFifoWriterBase* m_writer;
+};
+
+class RChannelBufferedReaderHolder : public RChannelReaderHolder
+{
+public:
+    RChannelBufferedReaderHolder(const char* channelURI,
+                                 RChannelOpenThrottler* openThrottler,
+                                 DryadMetaData* metaData,
+                                 RChannelItemParserBase* parser, 
+                                 UInt32 numberOfReaders,
+                                 UInt32 maxParseBatchSize,
+                                 UInt32 maxParseUnitsInFlight,
+                                 WorkQueue* workQueue,
+                                 DVErrorReporter* errorReporter,
+                                 LPDWORD localInputChannels);
+    ~RChannelBufferedReaderHolder();
+
+    RChannelReader* GetReader();
+    void FillInStatus(DryadInputChannelDescription* status);
+    void Close();
+
+private:
+    bool CreateBufferReader(UInt32 numberOfReaders,
+                            RChannelOpenThrottler* openThrottler,
+                            WorkQueue* workQueue,
+                            const char* channelURI,
+                            DryadMetaData* metaData,
+                            DVErrorReporter* errorReporter,
+                            LPDWORD localInputChannels);
+
+    RChannelItemParserRef  m_parser;
+    RChannelBufferReader*  m_bufferReader;
+    RChannelReader*        m_reader;
+};
+
+class RChannelBufferedWriterHolder : public RChannelWriterHolder
+{
+public:
+    RChannelBufferedWriterHolder(const char* channelURI,
+                                 RChannelOpenThrottler* openThrottler,
+                                 DryadMetaData* metaData,
+                                 RChannelItemMarshalerBase* marshaler,
+                                 UInt32 numberOfWriters,
+                                 UInt32 maxMarshalBatchSize,
+                                 WorkQueue* workQueue,
+                                 DVErrorReporter* errorReporter);
+    ~RChannelBufferedWriterHolder();
+
+    RChannelWriter* GetWriter();
+    void FillInStatus(DryadOutputChannelDescription* status);
+    void Close();
+
+private:
+    void CreateBufferWriter(UInt32 numberOfWriters,
+                            RChannelOpenThrottler* openThrottler,
+                            const char* channelURI,
+                            DryadMetaData* metaData,
+                            bool* pBreakOnBufferBoundaries,
+                            DVErrorReporter* errorReporter);
+
+    RChannelItemMarshalerRef  m_marshaler;
+    RChannelBufferWriter*     m_bufferWriter;
+    RChannelWriter*           m_writer;
+};
+
+class RChannelNullWriterHolder : public RChannelWriterHolder
+{
+public:
+    RChannelNullWriterHolder(const char* uri);
+    ~RChannelNullWriterHolder();
+    RChannelWriter* GetWriter();
+    void FillInStatus(DryadOutputChannelDescription* s);
+    void Close();
+
+private:
+    RChannelWriter*   m_writer;
+};

DryadVertex/VertexHost/system/channel/src/memorybuffers.cpp

@@ -0,0 +1,415 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#include "channelmemorybuffers.h"
+#include "channelreader.h"
+#include "channelwriter.h"
+
+#pragma unmanaged
+
+
+RChannelReaderBuffer::Current::Current(Size_t initialStartOffset,
+                                       Size_t finalEndOffset,
+                                       DryadLockedMemoryBuffer**
+                                       bufferArray,
+                                       size_t nBuffers,
+                                       Size_t offset)
+{
+    m_currentBuffer = 0;
+    m_currentBufferBase = 0;
+    m_currentHeadCutLength = initialStartOffset;
+    m_currentTailOffset = 0;
+
+    SetTailBufferData(bufferArray, nBuffers, finalEndOffset);
+
+    while (offset >= m_currentTailOffset)
+    {
+        ++m_currentBuffer;
+        LogAssert(m_currentBuffer < nBuffers);
+
+        m_currentBufferBase = m_currentTailOffset;
+        m_currentHeadCutLength = 0;
+
+        SetTailBufferData(bufferArray, nBuffers, finalEndOffset);
+    }
+}
+
+void* RChannelReaderBuffer::Current::
+    GetDataAddress(DryadLockedMemoryBuffer** bufferArray,
+                   Size_t offset,
+                   Size_t *puSize,
+                   Size_t *puPriorSize)
+{
+    LogAssert(offset >= m_currentBufferBase &&
+              offset < m_currentTailOffset);
+
+    Size_t offsetInAvailable = offset - m_currentBufferBase;
+    Size_t offsetInBuffer = offsetInAvailable + m_currentHeadCutLength;
+    Size_t remaining = m_currentTailOffset - offset;
+
+    void* dataAddress =
+        bufferArray[m_currentBuffer]->GetDataAddress(offsetInBuffer,
+                                                     puSize,
+                                                     puPriorSize);
+    LogAssert(dataAddress != NULL);
+
+    if (*puSize > remaining)
+    {
+        *puSize = remaining;
+    }
+
+    if (puPriorSize != NULL && *puPriorSize > offsetInAvailable)
+    {
+        *puPriorSize = offsetInAvailable;
+    }
+
+    return dataAddress;
+}
+
+void RChannelReaderBuffer::Current::
+    SetTailBufferData(DryadLockedMemoryBuffer** bufferArray,
+                      size_t nBuffers,
+                      Size_t finalEndOffset)
+{
+    Size_t bSize = bufferArray[m_currentBuffer]->GetAvailableSize();
+
+    if (m_currentBuffer == nBuffers-1)
+    {
+        LogAssert(finalEndOffset >= m_currentHeadCutLength);
+        LogAssert(finalEndOffset <= bSize);
+
+        m_currentTailOffset += (finalEndOffset - m_currentHeadCutLength);
+    }
+    else
+    {
+        m_currentTailOffset += (bSize - m_currentHeadCutLength);
+    }
+}
+
+
+RChannelReaderBuffer::RChannelReaderBuffer(DryadLockedBufferList* bufferList,
+                                           Size_t startOffset,
+                                           Size_t endOffset)
+{
+    m_nBuffers = bufferList->CountLinks();
+    LogAssert(m_nBuffers > 0);
+    m_bufferArray = new DryadLockedMemoryBuffer *[m_nBuffers];
+
+    LogAssert(m_uAllocatedSize == 0);
+    size_t i;
+    DrBListEntry* listEntry = bufferList->GetHead();
+    for (i=0; i < m_nBuffers; ++i)
+    {
+        LogAssert(listEntry != NULL);
+        m_bufferArray[i] = bufferList->CastOut(listEntry);
+        m_bufferArray[i]->IncRef();
+        m_uAllocatedSize += m_bufferArray[i]->GetAvailableSize();
+        listEntry = bufferList->GetNext(listEntry);
+    }
+    LogAssert(listEntry == NULL);
+
+    Initialise(startOffset, endOffset);
+}
+
+RChannelReaderBuffer::RChannelReaderBuffer(ChannelDataBufferList* bufferList,
+                                           Size_t startOffset,
+                                           Size_t endOffset)
+{
+    m_nBuffers = bufferList->CountLinks();
+    LogAssert(m_nBuffers > 0);
+    m_bufferArray = new DryadLockedMemoryBuffer *[m_nBuffers];
+
+    LogAssert(m_uAllocatedSize == 0);
+    size_t i;
+    DrBListEntry* listEntry = bufferList->GetHead();
+    for (i=0; i < m_nBuffers; ++i)
+    {
+        LogAssert(listEntry != NULL);
+        m_bufferArray[i] = (bufferList->CastOut(listEntry))->GetData();
+        m_bufferArray[i]->IncRef();
+        m_uAllocatedSize += m_bufferArray[i]->GetAvailableSize();
+        listEntry = bufferList->GetNext(listEntry);
+    }
+    LogAssert(listEntry == NULL);
+
+    Initialise(startOffset, endOffset);
+}
+
+void RChannelReaderBuffer::Initialise(Size_t startOffset, Size_t endOffset)
+{
+    m_fIsGrowable = false;
+
+    LogAssert(startOffset <= m_bufferArray[0]->GetAvailableSize());
+    LogAssert(endOffset <= m_bufferArray[m_nBuffers-1]->GetAvailableSize());
+    Size_t tailCutLength =
+        m_bufferArray[m_nBuffers-1]->GetAvailableSize() - endOffset;
+    LogAssert(m_uAllocatedSize >= (startOffset + tailCutLength));
+    m_uAllocatedSize -= (startOffset + tailCutLength);
+
+    SetAvailableSize(m_uAllocatedSize);
+
+    m_initialStartOffset = startOffset;
+    m_finalEndOffset = endOffset;
+}
+
+RChannelReaderBuffer::~RChannelReaderBuffer()
+{
+    size_t i;
+
+    for (i=0; i<m_nBuffers; ++i)
+    {
+        m_bufferArray[i]->DecRef();
+    }
+    delete [] m_bufferArray;
+}
+
+void* RChannelReaderBuffer::GetDataAddress(Size_t uOffset,
+                                           Size_t *puSize,
+                                           Size_t *puPriorSize)
+{
+    if (uOffset >= m_uAllocatedSize)
+    {
+        return NULL;
+    }
+
+    Current current(m_initialStartOffset, m_finalEndOffset,
+                    m_bufferArray, m_nBuffers, uOffset);
+
+    void* dataAddress = current.GetDataAddress(m_bufferArray,
+                                               uOffset,
+                                               puSize,
+                                               puPriorSize);
+
+    return dataAddress;
+}
+
+void RChannelReaderBuffer::IncreaseAllocatedSize(Size_t uSize)
+{
+    LogAssert(uSize <= m_uAllocatedSize);
+}
+
+
+RChannelWriterBuffer::
+    RChannelWriterBuffer(RChannelBufferWriter* bufferProvider,
+                         DryadFixedBufferList* bufferList)
+{
+    m_bufferProvider = bufferProvider;
+    m_bufferList = bufferList;
+    if (m_bufferList->IsEmpty() == false)
+    {
+        LogAssert(m_bufferList->CountLinks() == 1);
+        DryadFixedMemoryBuffer* currentBuffer =
+            m_bufferList->CastOut(m_bufferList->GetHead());
+
+        LogAssert(currentBuffer->GetAllocatedSize() >
+                  currentBuffer->GetAvailableSize());
+
+        m_baseBufferOffset = currentBuffer->GetAvailableSize();
+        m_uAllocatedSize =
+            currentBuffer->GetAllocatedSize() - m_baseBufferOffset;
+    }
+    else
+    {
+        m_baseBufferOffset = 0;
+        m_uAllocatedSize = 0;
+    }
+
+    m_currentBufferOffset = 0;
+    m_currentBaseOffset = m_baseBufferOffset;
+
+    m_availableHighWaterMark = 0;
+    m_availableStartOffset = m_baseBufferOffset;
+    m_availableBufferOffset = 0;
+    m_lastAvailableBuffer = m_bufferList->GetHead();
+}
+
+RChannelWriterBuffer::~RChannelWriterBuffer()
+{
+}
+
+void* RChannelWriterBuffer::GetDataAddress(Size_t uOffset,
+                                           Size_t *puSize,
+                                           Size_t *puPriorSize)
+{
+    if (m_bufferList->IsEmpty())
+    {
+        return NULL;
+    }
+    else
+    {
+        LogAssert(uOffset >= m_currentBufferOffset);
+
+        Size_t offsetInAvailable = uOffset - m_currentBufferOffset;
+        Size_t offsetInBuffer = offsetInAvailable + m_currentBaseOffset;
+
+        DryadFixedMemoryBuffer* buffer =
+            m_bufferList->CastOut(m_bufferList->GetTail());
+        void* dataAddress =
+            buffer->GetDataAddress(offsetInBuffer, puSize, puPriorSize);
+
+        if (puPriorSize != NULL && *puPriorSize > offsetInAvailable)
+        {
+            *puPriorSize = offsetInAvailable;
+        }
+
+        return dataAddress;
+    }
+}
+
+void RChannelWriterBuffer::IncreaseAllocatedSize(Size_t uSize)
+{
+    while (m_uAllocatedSize < uSize)
+    {
+        DryadFixedMemoryBuffer* newBuffer =
+            m_bufferProvider->GetNextWriteBuffer();
+        m_bufferList->InsertAsTail(m_bufferList->CastIn(newBuffer));
+        m_currentBufferOffset = m_uAllocatedSize;
+        m_uAllocatedSize += newBuffer->GetAllocatedSize();
+    }
+
+    /* this is zero for all but (optionally) the first buffer in the
+       list */
+    m_currentBaseOffset = 0;
+}
+
+void RChannelWriterBuffer::InternalSetAvailableSize(Size_t uSize)
+{
+    DrMemoryBuffer::InternalSetAvailableSize(uSize);
+
+    LogAssert(uSize >= m_availableHighWaterMark);
+
+    if (m_lastAvailableBuffer == NULL)
+    {
+        LogAssert(m_availableBufferOffset == 0);
+        LogAssert(m_availableHighWaterMark == 0);
+        LogAssert(m_availableStartOffset == m_baseBufferOffset);
+
+        if (m_bufferList->IsEmpty())
+        {
+            LogAssert(uSize == 0);
+            return;
+        }
+
+        m_lastAvailableBuffer = m_bufferList->GetHead();
+    }
+
+    DrBListEntry* listEntry = m_lastAvailableBuffer;
+    if (listEntry == NULL)
+    {
+        listEntry = m_bufferList->GetHead();
+        LogAssert(listEntry != NULL);
+    }
+
+    do
+    {
+        DryadFixedMemoryBuffer* b = m_bufferList->CastOut(listEntry);
+        listEntry = m_bufferList->GetNext(listEntry);
+
+        if (listEntry == NULL)
+        {
+            LogAssert(uSize >= m_availableBufferOffset);
+            Size_t thisAvailable = uSize - m_availableBufferOffset;
+
+            LogAssert(m_availableStartOffset + thisAvailable <=
+                      b->GetAllocatedSize());
+            b->SetAvailableSize(m_availableStartOffset + thisAvailable);
+        }
+        else
+        {
+            b->SetAvailableSize(b->GetAllocatedSize());
+            m_availableBufferOffset +=
+                b->GetAllocatedSize() - m_availableStartOffset;
+            LogAssert(m_availableBufferOffset < uSize);
+            m_lastAvailableBuffer = listEntry;
+            m_availableStartOffset = 0;
+        }
+    } while (listEntry != NULL);
+
+    m_availableHighWaterMark = uSize;
+}
+
+ChannelMemoryBufferWriter::
+    ChannelMemoryBufferWriter(DrMemoryBuffer* writeBuffer,
+                              DryadFixedBufferList* bufferList) :
+        DrMemoryBufferWriter(writeBuffer)
+{
+    m_bufferList = bufferList;
+
+    if (m_bufferList->IsEmpty())
+    {
+        m_initialBoundary = 0;
+    }
+    else
+    {
+        LogAssert(m_bufferList->CountLinks() == 1);
+        DryadFixedMemoryBuffer* buffer =
+            m_bufferList->CastOut(m_bufferList->GetHead());
+        m_initialBoundary = buffer->GetAvailableSize();
+        LogAssert(m_initialBoundary < buffer->GetAllocatedSize());
+    }
+
+    m_lastRecordBoundary = 0;
+}
+
+bool ChannelMemoryBufferWriter::MarkRecordBoundary()
+{
+    if (m_bufferList->IsEmpty())
+    {
+        LogAssert(m_initialBoundary == 0);
+        LogAssert(m_lastRecordBoundary == 0);
+        return false;
+    }
+    else if (m_bufferList->CountLinks() == 1)
+    {
+        DryadFixedMemoryBuffer* buffer =
+            m_bufferList->CastOut(m_bufferList->GetHead());
+        Size_t offset = GetBufferOffset();
+        Size_t allocated = buffer->GetAllocatedSize();
+
+        if (m_initialBoundary + offset >= allocated)
+        {
+            DrError errTmp = FlushMemoryWriter();
+            // sammck: should this assert here or return an error
+            LogAssert(errTmp == DrError_OK);
+        }
+
+        Size_t boundary = buffer->GetAvailableSize();
+        if (boundary == allocated)
+        {
+            return true;
+        }
+        else
+        {
+            m_lastRecordBoundary = offset;
+            LogAssert(m_initialBoundary + m_lastRecordBoundary < allocated);
+            return false;
+        }
+    }
+    else
+    {
+        return true;
+    }
+}
+
+Size_t ChannelMemoryBufferWriter::GetLastRecordBoundary()
+{
+    return m_initialBoundary + m_lastRecordBoundary;
+}
+    

DryadVertex/VertexHost/system/classlib/classlib.vcxproj

@@ -0,0 +1,205 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{016E71D3-9A6F-425C-AB4F-8C5EDEFFE7FA}</ProjectGuid>
+    <RootNamespace>classlib</RootNamespace>
+    <Keyword>Win32Proj</Keyword>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+    <CLRSupport>false</CLRSupport>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>StaticLibrary</ConfigurationType>
+    <CharacterSet>Unicode</CharacterSet>
+    <CLRSupport>false</CLRSupport>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup>
+    <_ProjectFileVersion>10.0.40219.1</_ProjectFileVersion>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Debug\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Debug\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(Platform)\$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(Platform)\$(Configuration)\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Release\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Release\</IntDir>
+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(Platform)\$(Configuration)\</OutDir>
+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(Platform)\$(Configuration)\</IntDir>
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>true</MinimalRebuild>
+      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <Optimization>Disabled</Optimization>
+      <AdditionalIncludeDirectories>..\..\common\include;..\..\classlib\include;include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <MinimalRebuild>false</MinimalRebuild>
+      <BasicRuntimeChecks>Default</BasicRuntimeChecks>
+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <Midl>
+      <TargetEnvironment>X64</TargetEnvironment>
+    </Midl>
+    <ClCompile>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+      <AdditionalIncludeDirectories>..\..\common\include;..\..\classlib\include;include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+    </ClCompile>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClInclude Include="include\basic_types.h" />
+    <ClInclude Include="include\DrBList.h" />
+    <ClInclude Include="include\DrCommon.h" />
+    <ClInclude Include="include\DrCriticalSection.h" />
+    <ClInclude Include="include\DrError.h" />
+    <ClInclude Include="include\DrErrorDef.h" />
+    <ClInclude Include="include\DrExecution.h" />
+    <ClInclude Include="include\DrExitCodes.h" />
+    <ClInclude Include="include\DrExitCodesDef.h" />
+    <ClInclude Include="include\DrFPrint.h" />
+    <ClInclude Include="include\DrFPrint_polynomials.h" />
+    <ClInclude Include="include\DrFunctions.h" />
+    <ClInclude Include="include\DrGuid.h" />
+    <ClInclude Include="include\DrHash.h" />
+    <ClInclude Include="include\DrHeap.h" />
+    <ClInclude Include="include\DrList.h" />
+    <ClInclude Include="include\DrLogging.h" />
+    <ClInclude Include="include\DrMemory.h" />
+    <ClInclude Include="include\DrMemoryStream.h" />
+    <ClInclude Include="include\DrNodeAddress.h" />
+    <ClInclude Include="include\DrProperties.h" />
+    <ClInclude Include="include\DrPropertiesDef.h" />
+    <ClInclude Include="include\DrPropertyDumper.h" />
+    <ClInclude Include="include\DrPropertyType.h" />
+    <ClInclude Include="include\DrRefCounter.h" />
+    <ClInclude Include="include\DrStringUtil.h" />
+    <ClInclude Include="include\DrTags.h" />
+    <ClInclude Include="include\DrTagsDef.h" />
+    <ClInclude Include="include\DrThread.h" />
+    <ClInclude Include="include\DrTypes.h" />
+    <ClInclude Include="include\Dryad.h" />
+    <ClInclude Include="include\DryadTags.h" />
+    <ClInclude Include="include\DryadTagsDef.h" />
+    <ClInclude Include="include\fingerprint.h" />
+    <ClInclude Include="include\Interlocked.h" />
+    <ClInclude Include="include\LogIds.h" />
+    <ClInclude Include="include\LogIdsCustomized.h" />
+    <ClInclude Include="include\LogTagIds.h" />
+    <ClInclude Include="include\ms_fprint.h" />
+    <ClInclude Include="include\MSMutex.h" />
+    <ClInclude Include="include\PropertyIds.h" />
+    <ClInclude Include="include\RefCount.h" />
+    <ClInclude Include="include\XCompute.h" />
+    <ClInclude Include="include\XComputeTypes.h" />
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="src\DrCriticalSection.cpp" />
+    <ClCompile Include="src\DrError.cpp" />
+    <ClCompile Include="src\DrExecution.cpp" />
+    <ClCompile Include="src\DrExitCodes.cpp" />
+    <ClCompile Include="src\DrFPrint.cpp" />
+    <ClCompile Include="src\DrFunctions.cpp" />
+    <ClCompile Include="src\DrGuid.cpp" />
+    <ClCompile Include="src\DrHash.cpp" />
+    <ClCompile Include="src\DrHeap.cpp" />
+    <ClCompile Include="src\DrLogging.cpp" />
+    <ClCompile Include="src\DrMemory.cpp" />
+    <ClCompile Include="src\DrMemoryStream.cpp" />
+    <ClCompile Include="src\DrNodeAddress.cpp" />
+    <ClCompile Include="src\DrRefCounter.cpp" />
+    <ClCompile Include="src\DrStringUtil.cpp" />
+    <ClCompile Include="src\DrThread.cpp" />
+    <ClCompile Include="src\fingerprint.cpp" />
+    <ClCompile Include="src\ms_fprint.cpp" />
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>

DryadVertex/VertexHost/system/classlib/include/DrBList.h

@@ -0,0 +1,411 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+/*
+ * DrBList
+ *
+ * Declares a utility class for managing bi-linked listed
+ */
+ 
+#ifndef __DRYADBLIST_H__
+#define __DRYADBLIST_H__
+
+
+/*
+ * DrBListEntry
+ * The list entry class is intentionally opaque. Operations should always
+ * be done using the list object. The list entry class should be embedded
+ * in the object you want to store, and the DR_GET_CONTAINER macro
+ * used to map back to the actual object
+ */
+
+class DrBListEntry
+{
+public:
+
+    //Standard c'tor
+    inline DrBListEntry();
+    
+    //Return TRUE if this entry is in a list.
+    //N.B. Remember to use the locking conventions of the list object!
+    inline BOOL IsInList();
+
+    //Remove this entry from whatever list its currently in
+    //N.B. Remember to use the locking conventions of the list object!
+    inline void Remove();
+        
+private:
+    friend class DrBList;
+    DrBListEntry * m_pNext, * m_pPrev;
+};
+
+/*
+ * DrBList
+ * Manages a bi-linked list of DrBListEntry's
+ */
+
+class DrBList
+{
+public:
+
+    /*
+     * Lifecycle Management
+     */
+
+    //Standard c'tor
+    inline DrBList();
+
+
+    /*
+     * Inserting entries to list
+     */
+
+    //Insert specified entry as head of list
+    //Asserts if pEntry is already in an existing list
+    inline void InsertAsHead(DrBListEntry * pEntry);
+    
+    //Insert specified entry as tail of list
+    //Asserts if pEntry is already in an existing list
+    inline void InsertAsTail(DrBListEntry * pEntry);
+
+    //Insert pNewEntry as the next entry after pCurrentEntry
+    //Asserts if pNewEntry is already in an existing list
+    inline void InsertAsNext(DrBListEntry * pCurrentEntry, DrBListEntry * pNewEntry);
+    
+    //Insert pNewEntry as the previoud entry before pCurrentEntry
+    //Asserts if pNewEntry is already in an existing list
+    inline void InsertAsPrev(DrBListEntry * pCurrentEntry, DrBListEntry * pNewEntry);
+    
+    //Remove pEntry from a list and transition it to the head of this one
+    //pEntry can either be in this list or an unrelated one
+    //Asserts if pEntry is not already in an existing list
+    inline void TransitionToHead(DrBListEntry * pEntry);
+    
+    //Remove pEntry from a list and transition it to the tail of this one
+    //pEntry can either be in this list or an unrelated one
+    //Asserts if pEntry is not already in an existing list
+    inline void TransitionToTail(DrBListEntry * pEntry);
+    
+    //Remove all the entries from pList and transition them to the
+    //head of this one. pList must not be this list
+    inline void TransitionToHead(DrBList * pList);
+    
+    //Remove all the entries from pList and transition them to the
+    //tail of this one. pList must not be this list
+    inline void TransitionToTail(DrBList * pList);
+
+
+    /*
+     * Removing entries from list
+     */
+    
+    //Remove entry from head of list and return it
+    //Returns NULL if list is empty
+    inline DrBListEntry * RemoveHead();
+    
+    //Remove entry from tail of list and return it
+    //Returns NULL if list is empty
+    inline DrBListEntry * RemoveTail();
+    
+    //Remove specified entry from list
+    inline DrBListEntry * Remove(DrBListEntry * pEntry);
+
+
+    /*
+     * Accessing entries in the list
+     */
+    
+    //Scan the list and if pEntry is found return it
+    //Returns NULL if pEntry is not found
+    inline DrBListEntry * Find(DrBListEntry * pEntry);
+    
+    //Return the head entry in the list or NULL if list is empty
+    inline DrBListEntry * GetHead();
+    
+    //Return the tail entry in the list or NULL if list is empty
+    inline DrBListEntry * GetTail();
+    
+    //Return the next entry after pEntry, or NULL if pEntry is the tail
+    inline DrBListEntry * GetNext(DrBListEntry * pEntry);
+    
+    //Return the previous entry before pEntry, or NULL if pEntry is the head
+    inline DrBListEntry * GetPrev(DrBListEntry * pEntry);
+    
+    
+    /*
+     * Retrieving state of list
+     */
+    
+    //Scan the list and count the total number of entries
+    inline DWORD CountLinks();
+    
+    //Return TRUE if the list is empty
+    inline BOOL IsEmpty();
+    
+private:
+
+    DrBListEntry m_dummyEntry;
+    
+};
+
+
+
+/*
+ * Inline methods for DrBListEntry 
+ */
+ 
+ 
+DrBListEntry::DrBListEntry()
+{
+    m_pNext=this;
+    m_pPrev=this;
+}
+
+BOOL DrBListEntry::IsInList()
+{
+    return (m_pNext!=this);
+}
+
+void DrBListEntry::Remove()
+{
+    LogAssert(IsInList());
+
+    m_pNext->m_pPrev = m_pPrev;
+    m_pPrev->m_pNext = m_pNext;
+    m_pNext=this;
+    m_pPrev=this;
+}
+
+/*
+ * Inline methods for DrBList 
+ */
+ 
+
+DrBList::DrBList()
+{
+    //this space intentionally left blank
+}
+
+void DrBList::InsertAsHead(DrBListEntry * pEntry)
+{
+    LogAssert(pEntry->IsInList()==FALSE);
+    
+    pEntry->m_pNext=m_dummyEntry.m_pNext;
+    pEntry->m_pPrev=&m_dummyEntry;
+    m_dummyEntry.m_pNext->m_pPrev=pEntry;
+    m_dummyEntry.m_pNext=pEntry;
+}
+
+void DrBList::TransitionToHead(DrBListEntry * pEntry)
+{
+    LogAssert(pEntry->IsInList());
+    
+    //Pull entry from existing list
+    pEntry->m_pNext->m_pPrev = pEntry->m_pPrev;
+    pEntry->m_pPrev->m_pNext = pEntry->m_pNext;
+    //Insert into this list
+    pEntry->m_pNext=m_dummyEntry.m_pNext;
+    pEntry->m_pPrev=&m_dummyEntry;
+    m_dummyEntry.m_pNext->m_pPrev=pEntry;
+    m_dummyEntry.m_pNext=pEntry;
+}
+
+void DrBList::TransitionToHead(DrBList * pList)
+{
+    LogAssert(pList != this);
+
+    if (!pList->IsEmpty())
+    {
+        DrBListEntry* otherHead = pList->GetHead();
+        DrBListEntry* otherTail = pList->GetTail();
+        otherTail->m_pNext=m_dummyEntry.m_pNext;
+        otherHead->m_pPrev=&m_dummyEntry;
+        m_dummyEntry.m_pNext->m_pPrev=otherTail;
+        m_dummyEntry.m_pNext=otherHead;
+        pList->m_dummyEntry.m_pNext=&(pList->m_dummyEntry);
+        pList->m_dummyEntry.m_pPrev=&(pList->m_dummyEntry);
+    }
+}
+
+void DrBList::InsertAsTail(DrBListEntry * pEntry)
+{
+    LogAssert(pEntry->IsInList()==FALSE);
+
+    pEntry->m_pNext=&m_dummyEntry;
+    pEntry->m_pPrev=m_dummyEntry.m_pPrev;
+    m_dummyEntry.m_pPrev->m_pNext=pEntry;
+    m_dummyEntry.m_pPrev=pEntry;
+}
+
+void DrBList::TransitionToTail(DrBListEntry * pEntry)
+{
+    LogAssert(pEntry->IsInList());
+    //Pull entry from existing list
+    pEntry->m_pNext->m_pPrev = pEntry->m_pPrev;
+    pEntry->m_pPrev->m_pNext = pEntry->m_pNext;
+    //Insert into this list
+    pEntry->m_pNext=&m_dummyEntry;
+    pEntry->m_pPrev=m_dummyEntry.m_pPrev;
+    m_dummyEntry.m_pPrev->m_pNext=pEntry;
+    m_dummyEntry.m_pPrev=pEntry;
+}
+
+//
+// Append list to existing list's tail (uses m_dummyEntry to move links)
+//
+void DrBList::TransitionToTail(DrBList * pList)
+{
+    LogAssert(pList != this);
+
+    if (!pList->IsEmpty())
+    {
+        DrBListEntry* otherHead = pList->GetHead();
+        DrBListEntry* otherTail = pList->GetTail();
+        
+        //
+        // link tail's next and head's prev to placeholder's prev
+        //
+        otherTail->m_pNext=&m_dummyEntry;
+        otherHead->m_pPrev=m_dummyEntry.m_pPrev;
+
+        //
+        // Link placeholder's prev's next to head
+        //
+        m_dummyEntry.m_pPrev->m_pNext=otherHead;
+
+        //
+        // Link placeholder's prev to tail
+        //
+        m_dummyEntry.m_pPrev=otherTail;
+
+        //
+        // Update placeholder next and previous to self
+        //
+        pList->m_dummyEntry.m_pNext=&(pList->m_dummyEntry);
+        pList->m_dummyEntry.m_pPrev=&(pList->m_dummyEntry);
+    }
+}
+
+
+void DrBList::InsertAsNext(DrBListEntry * pCurrentEntry, DrBListEntry * pNewEntry)
+{
+    LogAssert(pCurrentEntry->IsInList());
+    LogAssert(pNewEntry->IsInList()==FALSE);
+
+    pNewEntry->m_pNext=pCurrentEntry->m_pNext;
+    pNewEntry->m_pPrev=pCurrentEntry;
+    pCurrentEntry->m_pNext->m_pPrev=pNewEntry;
+    pCurrentEntry->m_pNext=pNewEntry;
+}
+
+void DrBList::InsertAsPrev(DrBListEntry * pCurrentEntry, DrBListEntry * pNewEntry)
+{
+    LogAssert(pCurrentEntry->IsInList());
+    LogAssert(pNewEntry->IsInList()==FALSE);
+    
+    pNewEntry->m_pNext=pCurrentEntry;
+    pNewEntry->m_pPrev=pCurrentEntry->m_pPrev;
+    pCurrentEntry->m_pPrev->m_pNext=pNewEntry;
+    pCurrentEntry->m_pPrev=pNewEntry;
+}
+
+DrBListEntry * DrBList::RemoveHead()
+{
+    if (m_dummyEntry.m_pNext==&m_dummyEntry)
+        return NULL;
+    else
+        return Remove(m_dummyEntry.m_pNext);
+}
+
+DrBListEntry * DrBList::RemoveTail()
+{
+    if (m_dummyEntry.m_pPrev==&m_dummyEntry)
+        return NULL;
+    else
+        return Remove(m_dummyEntry.m_pPrev);
+}
+
+DrBListEntry * DrBList::Remove(DrBListEntry * pEntry)
+{
+    LogAssert(pEntry->IsInList());
+    LogAssert(pEntry!=&m_dummyEntry);
+
+    pEntry->m_pNext->m_pPrev = pEntry->m_pPrev;
+    pEntry->m_pPrev->m_pNext = pEntry->m_pNext;
+    pEntry->m_pNext=pEntry;
+    pEntry->m_pPrev=pEntry;
+    return pEntry;
+}
+
+
+DrBListEntry * DrBList::Find(DrBListEntry * pEntry)
+{
+    LogAssert(pEntry);
+    DrBListEntry * pScan=m_dummyEntry.m_pNext;
+    while (pScan!=&m_dummyEntry)
+    {
+        if (pScan==pEntry)
+            return pEntry;
+        pScan=pScan->m_pNext;
+    }
+    return NULL;
+}
+
+DrBListEntry * DrBList::GetHead()
+{
+    return (m_dummyEntry.m_pNext==&m_dummyEntry) ? NULL : m_dummyEntry.m_pNext;
+}
+
+DrBListEntry * DrBList::GetTail()
+{
+    return (m_dummyEntry.m_pPrev==&m_dummyEntry) ? NULL : m_dummyEntry.m_pPrev;
+}
+
+DrBListEntry * DrBList::GetNext(DrBListEntry * pEntry)
+{
+    return (pEntry->m_pNext==&m_dummyEntry) ? NULL : pEntry->m_pNext;
+}
+
+DrBListEntry * DrBList::GetPrev(DrBListEntry * pEntry)
+{
+    return (pEntry->m_pPrev==&m_dummyEntry) ? NULL : pEntry->m_pPrev;
+}
+    
+//
+// Count the number of links in the list and return count
+//
+DWORD DrBList::CountLinks()
+{
+    DWORD dwCount=0;
+    DrBListEntry * pScan=m_dummyEntry.m_pNext;
+    while (pScan!=&m_dummyEntry)
+    {
+        dwCount++;
+        pScan=pScan->m_pNext;
+    }
+    return dwCount;
+}
+
+BOOL DrBList::IsEmpty()
+{
+    return (m_dummyEntry.m_pNext==&m_dummyEntry);
+}
+
+#endif //end if not defined __DRYADBLIST_H__

DryadVertex/VertexHost/system/classlib/include/DrCommon.h

@@ -0,0 +1,47 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#pragma warning (disable: 4100) // unreferenced formal parameter
+#pragma warning (disable: 4995) // deprecated function
+
+#include "DrTypes.h"
+#include "Dryad.h"
+#include "DrErrorDef.h"
+#include "DrLogging.h"
+#include "DrFunctions.h"
+#include "DrHash.h"
+#include "MSMutex.h"
+#include "DrGuid.h"
+#include "DrStringUtil.h"
+#include "DrExitCodesDef.h"
+#include "DrRefCounter.h"
+#include "DrMemory.h"
+#include "DrMemoryStream.h"
+#include "DrCriticalSection.h"
+#include "DrHeap.h"
+#include "DrThread.h"
+#include "propertyids.h"
+#include "DrPropertiesDef.h"
+#include "DrMemoryStream.h"
+#include "DrNodeAddress.h"
+#include "DrTagsDef.h"
+#include "DrPropertyDumper.h"

DryadVertex/VertexHost/system/classlib/include/DrCriticalSection.h

@@ -0,0 +1,520 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+/*
+ * DrCriticalSection
+ *
+ * Defines a utility class for wrapping a critical section
+ */
+#pragma once
+
+
+#define DEFAULT_SPIN_COUNT 4000u
+#define DEFAULT_DR_LOG_HELD_TOO_LONG_TIMEOUT (500 * DrTimeInterval_Millisecond)
+
+class DrCriticalSectionBase : public CRITICAL_SECTION
+{
+public:
+    void Init(
+        __in PCSTR name = NULL,
+        DWORD spinCount = DEFAULT_SPIN_COUNT,
+        bool logUsage = false,
+        DrTimeInterval logHeldTooLongTimeout = DEFAULT_DR_LOG_HELD_TOO_LONG_TIMEOUT
+        );
+
+    void Uninit()
+    {
+        DebugLogAssert( RecursionCount==0 && OwningThread == 0);
+        DeleteCriticalSection( this );
+    }
+
+    void Enter( PCSTR functionName = NULL, PCSTR fileName = NULL, UINT lineNumber = 0 );
+
+    void Leave( PCSTR functionName = NULL, PCSTR fileName = NULL, UINT lineNumber = 0 );
+
+    bool Acquired() const
+    {
+        return RecursionCount > 0 && (DWORD)(DWORD_PTR)OwningThread == GetCurrentThreadId();
+    }
+
+    // TODO: deprecate this, misspelled
+    bool Aquired() const
+    {
+        return RecursionCount > 0 && (DWORD)(DWORD_PTR)OwningThread == GetCurrentThreadId();
+    }
+
+    // Returns 0 if this thread does not own the critical section
+    UInt32 GetRecursionCountIfAcquired()
+    {
+        return (RecursionCount > 0 && (DWORD)(DWORD_PTR)OwningThread == GetCurrentThreadId()) ? (UInt32)RecursionCount : 0 ;
+    }
+
+    // also returns true if n is 0 and the thread does not own the critical section
+    bool AcquiredExactNumberOfTimes(UInt32 n)
+    {
+        return (GetRecursionCountIfAcquired() == n);
+    }
+
+    bool AcquiredExactlyOnce()
+    {
+        return RecursionCount == 1 && (DWORD)(DWORD_PTR)OwningThread == GetCurrentThreadId();
+    }
+    
+    bool NotAcquired() const
+    {
+        return (DWORD)(DWORD_PTR)OwningThread != GetCurrentThreadId();
+    }
+
+    // TODO: deprecate this, misspelled
+    bool NotAquired() const
+    {
+        return (DWORD)(DWORD_PTR)OwningThread != GetCurrentThreadId();
+    }
+
+    // returns the old spin count
+    DWORD SetSpinCount(DWORD spinCount)
+    {
+        return SetCriticalSectionSpinCount(this, spinCount);
+    }
+
+    // Could cause inconsistent settings if called concurrently. If you might be calling this
+    // method concurrently, you should claim the lock before calling. Since in most cases
+    // this is called before the lock is actually used, this precaution is
+    // not taken by default.
+    void SetCriticalSectionLoggingParameters(
+        bool logUsage = true,
+        DrTimeInterval logHeldTooLongTimeout = DEFAULT_DR_LOG_HELD_TOO_LONG_TIMEOUT
+    );
+
+    void SetCriticalSectionLogging(
+        bool logUsage = true
+    );
+
+    void SetCriticalSectionLogHeldTooLongTimeout(
+        DrTimeInterval logHeldTooLongTimeout = DEFAULT_DR_LOG_HELD_TOO_LONG_TIMEOUT
+    );
+
+
+    void SetCriticalSectionName(__in   PCSTR name)
+    {
+        _name = name;
+    }
+    
+    bool TryEnter( PCSTR functionName = NULL, PCSTR fileName = NULL, UINT lineNumber = 0)
+    {
+        BOOL entered = TryEnterCriticalSection(this);
+
+        if ( entered )
+        {
+            _lastFunctionName = functionName;
+            _lastFileName = fileName;
+            _lastLineNumber = lineNumber;
+        }
+
+        if ( _logUsage )
+        {
+            if ( entered )
+            {
+                if (fileName != NULL) {
+                    DrLogD( "CritSect TRY ENTER OK, %s at %s %s(%u), addr=%08Ix",
+                        _name, functionName, fileName, lineNumber, this );
+                } else {
+                    DrLogD( "CritSect TRY ENTER OK, %s, addr=%08Ix",
+                        _name, this );
+                }
+            }
+            else
+            {
+                if (fileName != NULL) {
+                    DrLogW( "CritSect TRY ENTER FAILED, %s at %s %s(%u), addr=%08Ix",
+                        _name, functionName, fileName, lineNumber, this );
+                } else {
+                    DrLogW( "CritSect TRY ENTER FAILED, %s, addr=%08Ix",
+                        _name, this );
+                }
+            }
+        }
+
+        return entered != FALSE;
+    }
+
+    PCSTR   _name;
+    bool    _logUsage;
+    DWORD   _logHeldTooLongTimeoutMs;
+    PCSTR   _lastFunctionName;
+    PCSTR   _lastFileName;
+    UINT    _lastLineNumber;
+    DWORD   _enterTimeMs;
+};
+
+
+class DrCriticalSection : public DrCriticalSectionBase
+{
+public:
+
+    DrCriticalSection(
+        PCSTR name = NULL,
+        DWORD spinCount = DEFAULT_SPIN_COUNT,
+        bool logUsage = false,
+        UINT logHeldTooLongTimeout = DEFAULT_DR_LOG_HELD_TOO_LONG_TIMEOUT
+        )
+    {
+        Init( name, spinCount, logUsage, logHeldTooLongTimeout );
+    }
+
+    ~DrCriticalSection()
+    {
+        Uninit();
+    }
+
+};
+
+
+class AutoLock
+{
+    friend class AutoLockLogged;
+
+public:
+
+    AutoLock()
+    {
+        _lock = NULL;
+    }
+
+    AutoLock( DrCriticalSectionBase* lock )
+    {
+        _lock = lock;
+        _lock->Enter();
+    }
+
+    AutoLock( DrCriticalSectionBase& lock )
+    {
+        _lock = &lock;
+        _lock->Enter();
+    }
+
+    AutoLock( DrCriticalSectionBase* lock, PCSTR functionName, PCSTR fileName, UINT lineCount )
+    {
+        _lock = lock;
+        _lock->Enter( functionName, fileName, lineCount );
+    }
+
+    ~AutoLock()
+    {
+        if ( _lock != NULL )
+        {
+            _lock->Leave();
+        }
+    }
+
+    void Enter( DrCriticalSectionBase* newLock = NULL )
+    {
+        if ( newLock != NULL )
+        {
+            LogAssert( _lock == NULL );
+            _lock = newLock;
+        }
+        LogAssert( _lock != NULL );
+        _lock->Enter();
+    }
+
+    void Leave()
+    {
+        LogAssert( _lock != NULL );
+        _lock->Leave();
+        _lock = NULL;
+    }
+
+private:
+
+    AutoLock( const AutoLock& );
+    AutoLock& operator=( const AutoLock& );
+
+    DrCriticalSectionBase* _lock;
+
+};
+
+typedef AutoLock DrScopedCritSec;
+typedef AutoLock DrAutoCriticalSection;
+
+#define DR_ENTER(lock) (lock).Enter( __FUNCTION__, __FILE__, __LINE__ )
+#define DR_LEAVE(lock) (lock).Leave( __FUNCTION__, __FILE__, __LINE__ )
+#define DR_ENTER_NOLOG(lock) (lock).Enter()
+#define DR_LEAVE_NOLOG(lock) (lock).Leave()
+
+#define EnterAndLog() Enter( __FUNCTION__, __FILE__, __LINE__ )
+#define LeaveAndLog() Leave( __FUNCTION__, __FILE__, __LINE__ )
+
+#define LOCK_JOIN_NAME2(a, b, c) a##_line_##b##_counter_##c
+#define LOCK_JOIN_NAME(a, b, c) LOCK_JOIN_NAME2(a, b, c)
+#define LOCK(lock) AutoLock LOCK_JOIN_NAME(autoLock, __LINE__, __COUNTER__) (&lock, __FUNCTION__, __FILE__, __LINE__);
+#define LOCK_NOLOG(lock) AutoLock LOCK_JOIN_NAME(autoLock, __LINE__, __COUNTER__) (&lock);
+
+#define DR_IN_CRITSEC(cs) LOCK(cs)
+
+#define DRLOCKABLENOIMPL \
+    public: \
+        virtual void Lock() override = 0; \
+        virtual void Lock() const override = 0; \
+        virtual void Unlock() override = 0; \
+        virtual void Unlock() const override = 0; \
+
+#define DRLOCKABLEIMPL_NO_LOCK \
+    public: \
+        virtual void Lock() override\
+        { \
+        } \
+        void Lock() const override \
+        { \
+        } \
+        virtual void Unlock() override \
+        { \
+        } \
+        virtual void Unlock() const override \
+        { \
+        }
+
+#define DRLOCKABLEIMPL_PROTO \
+    public: \
+        virtual void Lock(); \
+        virtual void Lock() const; \
+        virtual void Unlock(); \
+        virtual void Unlock() const; \
+
+#define DRLOCKABLEIMPL_BASE(InterfaceClass) \
+    protected: \
+        mutable DrCriticalSection m_cs; \
+    public: \
+        virtual void Lock() override \
+        { \
+            m_cs.Enter(); \
+        } \
+        virtual void Lock() const override \
+        { \
+            m_cs.Enter(); \
+        } \
+        virtual void Unlock() override \
+        { \
+            m_cs.Leave(); \
+        } \
+        virtual void Unlock() const override \
+        { \
+            m_cs.Leave(); \
+        }
+
+#define DRLOCKABLEIMPL_DELEGATE(pImpl) \
+    public: \
+        virtual void Lock() override \
+        { \
+            pImpl->Lock(); \
+        } \
+        virtual void Lock() const override \
+        { \
+            pImpl->Lock(); \
+        } \
+        virtual void Unlock() override \
+        { \
+            pImpl->Unlock(); \
+        } \
+        virtual void Unlock() const override \
+        { \
+            pImpl->Unlock(); \
+        }
+
+#define DRLOCKABLEIMPL_DELEGATE_OUTSIDE_CLASS(classname, pImpl) \
+    void classname::Lock() \
+    { \
+        pImpl->Lock(); \
+    } \
+    void classname::Lock() const \
+    { \
+        pImpl->Lock(); \
+    } \
+    void classname::Unlock() \
+    { \
+        pImpl->Unlock(); \
+    } \
+    void classname::Unlock() const \
+    { \
+        pImpl->Unlock(); \
+    }
+
+
+#define DRLOCKABLEIMPL DRLOCKABLEIMPL_BASE(IDrLockable)
+
+class IDrLockable
+{
+public:
+    virtual void Lock() = 0;
+    virtual void Lock() const = 0;
+    virtual void Unlock() = 0;
+    virtual void Unlock() const = 0;
+    virtual ~IDrLockable()
+    {
+    }
+};
+
+class DrLockable : public IDrLockable
+{
+    DRLOCKABLEIMPL
+};
+
+template<class T> class DrScopedLock
+{
+public:
+    DrScopedLock(T *pT = NULL)
+    {
+        m_pT = pT;
+        if (pT != NULL) {
+            pT->Lock();
+        }
+    }
+
+    ~DrScopedLock()
+    {
+        if (m_pT != NULL) {
+            m_pT->Unlock();
+        }
+    }
+
+private:
+    T *m_pT;
+
+};
+
+typedef DrScopedLock<const IDrLockable> DrScopedLockable;
+
+extern DrCriticalSection *g_pDrGlobalCritSec; // A general shared critical section that is intialized on first use or at static constructor time
+
+extern DrCriticalSection *DrInitializeGlobalCritSec();
+
+__inline DrCriticalSection *DrGetGlobalCritSec()
+{
+    if (g_pDrGlobalCritSec == NULL) {
+        DrInitializeGlobalCritSec();
+    }
+    return g_pDrGlobalCritSec;
+}
+
+__inline void DrEnterGlobalCritSec()
+{
+    if (g_pDrGlobalCritSec == NULL) {
+        DrInitializeGlobalCritSec();
+    }
+#pragma prefast(push)
+#pragma prefast(disable:6011)
+    g_pDrGlobalCritSec->Enter();
+#pragma prefast(pop)    
+}
+
+__inline void DrLeaveGlobalCritSec()
+{
+    g_pDrGlobalCritSec->Leave();
+}
+
+// A scoped lock of the singleton global cosmos critical section
+class DrScopedGlobalLock
+{
+public:
+    DrScopedGlobalLock()
+    {
+        DrEnterGlobalCritSec();
+    }
+
+    ~DrScopedGlobalLock()
+    {
+        DrLeaveGlobalCritSec();
+    }
+};
+
+// put this in your header file to declare a static singleton lock that is never destructed and can be
+// used within static initializers
+#define DECLARE_STATIC_LOCK(name) \
+    class name \
+    { \
+    public: \
+        static DrCriticalSection *InitialGetCritSec() \
+        { \
+            DrScopedGlobalLock glock; \
+            if (s_pCritSec == NULL) { \
+                s_pCritSec = new DrCriticalSection(); \
+            } \
+            return s_pCritSec; \
+        } \
+        __forceinline static DrCriticalSection *GetCritSec() \
+        { \
+            if (s_pCritSec == NULL) { \
+                InitialGetCritSec(); \
+            } \
+            return s_pCritSec; \
+        } \
+        static void Lock() \
+        { \
+            GetCritSec()->Enter(); \
+        } \
+        static void Unlock() \
+        { \
+            GetCritSec()->Enter(); \
+        } \
+    private: \
+        static DrCriticalSection *s_pCritSec; \
+    }; \
+
+// Put this in your cpp file to provide a definition for a static singleton lock that is never destructed and can be
+// used within static initializers.
+#define DEFINE_STATIC_LOCK(name) \
+    DrCriticalSection * name::s_pCritSec = NULL;
+
+// A scoped lock of a singleton static critical section that is never destructed and can be
+// used within static initializers.
+template<class LockName> class DrScopedStaticLock
+{
+public:
+    DrScopedStaticLock()
+    {
+        LockName::Lock();
+    }
+
+    ~DrScopedStaticLock()
+    {
+        LockName::Unlock();
+    }
+};
+
+#define DRLOCKABLEIMPL_DELEGATE_TO_STATIC(LockName) \
+    public: \
+        virtual void Lock() override \
+        { \
+            LockName::Lock(); \
+        } \
+        virtual void Lock() const override \
+        { \
+            LockName::Lock(); \
+        } \
+        virtual void Unlock() override \
+        { \
+            LockName::Unlock(); \
+        } \
+        virtual void Unlock() const override \
+        { \
+            LockName::Unlock(); \
+        }
+
+

DryadVertex/VertexHost/system/classlib/include/DrError.h

@@ -0,0 +1,557 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//JC Check this file for redundant information
+
+// DrError.h
+//
+// This file must contain *only* DEFINE_DR_ERROR directives!
+//
+//
+// It is included multiple times with different macro definitions.
+
+#ifndef COMMON_DR_ERRORS_DEFINED
+
+// The operation succeeded
+DEFINE_DR_ERROR (DrError_OK,                        S_OK,                   "The operation succeeded")
+
+// The operation failed
+DEFINE_DR_ERROR (DrError_Fail,                      E_FAIL,                 "The operation failed")
+
+#endif
+
+// Out of memory
+DEFINE_DR_ERROR (DrError_OutOfMemory,               DR_ERROR (0x0001), "Out of memory")
+
+// The operation has not yet completed
+DEFINE_DR_ERROR (DrError_IncompleteOperation,       DR_ERROR (0x0002), "The operation has not yet completed")
+
+// Out of Range
+DEFINE_DR_ERROR (DrError_OutOfRange,                DR_ERROR (0x0003), "Out of Range")
+
+#ifndef COMMON_DR_ERRORS_DEFINED
+
+// Invalid Parameter
+DEFINE_DR_ERROR (DrError_InvalidParameter,          DR_ERROR (0x0004), "Invalid Parameter")
+
+#endif
+
+// Operation timed out
+DEFINE_DR_ERROR (DrError_Timeout,                   DR_ERROR (0x0005), "Operation timed out")
+
+// Remote end of the connection disconnected
+DEFINE_DR_ERROR (DrError_RemoteDisconnected,        DR_ERROR (0x0006), "Remote end of the connection disconnected")
+
+// Local end of the connection disconnected
+DEFINE_DR_ERROR (DrError_LocalDisconnected,         DR_ERROR (0x0007), "Local end of the connection disconnected")
+
+// Listen operation was stopped
+DEFINE_DR_ERROR (DrError_ListenStopped,             DR_ERROR (0x0008), "Listen operation was stopped ")
+
+// Stream already exists
+DEFINE_DR_ERROR (DrError_StreamAlreadyExists,       DR_ERROR (0x0009), "Stream already exists")
+
+// Stream not found
+DEFINE_DR_ERROR (DrError_StreamNotFound,            DR_ERROR (0x000a), "Stream not found")
+
+#ifndef COMMON_DR_ERRORS_DEFINED
+
+// End of stream
+DEFINE_DR_ERROR (DrError_EndOfStream,               DR_ERROR (0x000b), "End of stream")
+
+// Invalid property
+DEFINE_DR_ERROR (DrError_InvalidProperty,           DR_ERROR (0x000c), "Invalid property")
+
+#endif
+
+// Extent not found
+DEFINE_DR_ERROR (DrError_ExtentNotFound,            DR_ERROR (0x000d), "Extent not found")
+
+#ifndef COMMON_DR_ERRORS_DEFINED
+
+// Unable to peek this far forward
+DEFINE_DR_ERROR (DrError_PeekTooFar,                DR_ERROR (0x000e), "Unable to peek this far forward")
+
+// String is too long
+DEFINE_DR_ERROR (DrError_StringTooLong,             DR_ERROR (0x000f), "String is too long")
+
+#endif
+
+// operation aborted
+DEFINE_DR_ERROR (DrError_Aborted,                   DR_ERROR (0x0010), "The operation was aborted")
+
+// Directory does not exist
+DEFINE_DR_ERROR (DrError_DirectoryDoesNotExist,     DR_ERROR (0x0011), "Directory does not exist")
+
+// Invalid version
+DEFINE_DR_ERROR (DrError_InvalidVersion,            DR_ERROR (0x0012), "Invalid version")
+
+// Host not found or name resolution error
+DEFINE_DR_ERROR (DrError_HostNotFound,              DR_ERROR (0x0013), "Host not found")
+
+// Parameter not found
+DEFINE_DR_ERROR (DrError_ParameterNotFound,         DR_ERROR (0x0014), "Parameter not found")
+
+// Bad offset
+DEFINE_DR_ERROR (DrError_BadOffset,                 DR_ERROR (0x0015), "Bad offset")
+
+// No available extent instances
+DEFINE_DR_ERROR (DrError_NoExtentInstances,         DR_ERROR (0x0016), "No available extent instances")
+
+// Invalid pathname
+DEFINE_DR_ERROR (DrError_InvalidPathname,           DR_ERROR (0x0017), "Invalid pathname")
+
+// Not implemented
+DEFINE_DR_ERROR (DrError_NotImplemented,            DR_ERROR (0x0018), "Not implemented")
+
+// Improper protocol response received
+DEFINE_DR_ERROR (DrError_ImproperProtocolResponse,  DR_ERROR (0x0019), "Improper Protocol Response Received")
+
+// The connection failed
+DEFINE_DR_ERROR (DrError_ConnectionFailed,          DR_ERROR (0x001a), "The connection failed")
+
+// Extent cannot be appended as it has reached its size limit
+DEFINE_DR_ERROR(DrError_ExtentFull,                 DR_ERROR (0x001b), "The extent is full")
+
+// Extent has been sealed
+DEFINE_DR_ERROR(DrError_ExtentSealed,               DR_ERROR (0x001c), "The extent is sealed")
+
+// Extent failed to seal after reaching its size limit on an append
+DEFINE_DR_ERROR(DrError_ExtentSealFailOnFull,       DR_ERROR (0x001d), "The extent failed to seal as full")
+
+// Extent operation that can only be done on primary attempted on a non-primary EN
+DEFINE_DR_ERROR(DrError_EnPrimaryOnly,               DR_ERROR (0x001e), "The extent instance is not the primary instance")
+
+// Timeout waiting for a protocol response
+DEFINE_DR_ERROR(DrError_ResponseTimeout,            DR_ERROR (0x001f), "The Dryad server did not respond to the request")
+
+// Timeout waiting for a protocol send
+DEFINE_DR_ERROR (DrError_SendTimeout,               DR_ERROR (0x0020), "Unable to reach the Dryad node")
+
+// Name server unavailable
+DEFINE_DR_ERROR (DrError_NameServerUnavailable,     DR_ERROR (0x0021), "Dryad Name server unavailable")
+
+#ifndef COMMON_DR_ERRORS_DEFINED
+
+// Line Too Long
+DEFINE_DR_ERROR (DrError_LineTooLong,               DR_ERROR (0x0022), "Line is too long")
+
+#endif
+
+// Process Already Exists
+DEFINE_DR_ERROR (DrError_ProcessAlreadyExists,      DR_ERROR (0x0023), "Process already exists")
+
+// Extent operation that can only be done only on non-primary attempted on primary
+DEFINE_DR_ERROR (DrError_EnNonPrimaryOnly,          DR_ERROR (0x0024), "The extent instance is the primary instance")
+
+// Process Already Exists
+DEFINE_DR_ERROR (DrError_UnknownProcess,            DR_ERROR (0x0025), "Unknown process")
+
+// Extent is incomplete (and therefore whatever operation was being atttempted is invalid)
+DEFINE_DR_ERROR (DrError_ExtentIncomplete,          DR_ERROR (0x0026), "The extent is incomplete")
+
+// Length specified is invalid
+DEFINE_DR_ERROR (DrError_InvalidLength,             DR_ERROR (0x0028), "Invalid length")
+
+// Retry limit reached (client lib append)
+DEFINE_DR_ERROR (DrError_RetryLimit,                DR_ERROR (0x0029), "Retry limit reached")
+
+// Previous operation in queue failed (client lib fixed extent read/write queues)
+DEFINE_DR_ERROR (DrError_PreviousFailed,            DR_ERROR (0x002a), "Previous operation in queue failed")
+
+// This handle is not open for read
+DEFINE_DR_ERROR (DrError_HandleInvalidModeRead,     DR_ERROR (0x002b), "This handle is not open for read")
+
+// This handle is not open for append
+DEFINE_DR_ERROR (DrError_HandleInvalidModeAppend,   DR_ERROR (0x002c), "This handle is not open for append")
+
+// client lib Initialization failed
+DEFINE_DR_ERROR (DrError_InitializationFailed,      DR_ERROR (0x002d), "Initialization failed")
+
+// invalid handle
+DEFINE_DR_ERROR (DrError_InvalidHandle,             DR_ERROR (0x002e), "Invalid handle value")
+
+// Disconnect waiting for a protocol response
+DEFINE_DR_ERROR(DrError_ResponseDisconnect,         DR_ERROR (0x002f), "Connection lost while waiting for a response")
+
+// File-based stream allows only 0 extent index.
+DEFINE_DR_ERROR(DrError_InvalidFileExtentIndex,     DR_ERROR (0x0030), "Local file-based Dryad stream allows only 0 extent index")
+
+// Attempt to append beyond an unsealed extent in a stream
+DEFINE_DR_ERROR (DrError_OffsetIsUnsealed,          DR_ERROR (0x0031), "Offset is unsealed")
+
+// Request/call/operation unexpected
+DEFINE_DR_ERROR(DrError_Unexpected,                 DR_ERROR (0x0032), "Unexpected call or request")
+
+#ifndef COMMON_DR_ERRORS_DEFINED
+
+// Invalid time interval
+DEFINE_DR_ERROR(DrError_InvalidTimeInterval,        DR_ERROR (0x0033), "Invalid time interval")
+
+#endif
+
+// Too many redirects
+DEFINE_DR_ERROR(DrError_TooManyRedirects,           DR_ERROR (0x0034), "Too many redirects")
+
+// Incompatible rendezvous parties
+DEFINE_DR_ERROR(DrError_IncompatibleRendezvousParties, DR_ERROR (0x0035), "Incompatible rendezvous parties")
+
+// No matching rendezvous part
+DEFINE_DR_ERROR(DrError_NoMatchingRendezvousParty,  DR_ERROR (0x0036), "No matching rendezvous party")
+
+// Failed to seal an extent (used in failure mode sealing)
+DEFINE_DR_ERROR (DrError_FailedToSeal,              DR_ERROR (0x0037), "Failed to seal extent")
+
+//Stream option id is invalid
+DEFINE_DR_ERROR(DrError_InvalidOption,              DR_ERROR (0x0038), "Stream option id is invalid")
+
+//Stream option value size is invalid
+DEFINE_DR_ERROR(DrError_InvalidOptionSize,          DR_ERROR (0x0039), "Stream option value size is invalid")
+
+//Item not found in lookup table, typically converted to specific item code
+DEFINE_DR_ERROR(DrError_ItemNotFound,               DR_ERROR (0x003a), "Item not found")
+
+// The pipe was redirected
+DEFINE_DR_ERROR(DrError_InternalPipeRedirected,     DR_ERROR (0x003b), "The pipe was redirected (internal)")
+
+// request did not succeed, please talk to the primary Server
+DEFINE_DR_ERROR(DrError_TalkToPrimaryServer,        DR_ERROR (0x003c), "Please talk to primary server for this request")
+
+// Invalid pipe URI
+DEFINE_DR_ERROR(DrError_InvalidPipeUri,             DR_ERROR (0x003d), "Invalid Pipe URI")
+
+// Artificial fault error
+DEFINE_DR_ERROR(DrError_ArtificialError,            DR_ERROR (0x003e), "Artificial test error; Please test again")
+
+// Both streams should be on the same volume.
+DEFINE_DR_ERROR(DrError_SameVolumeRequired,         DR_ERROR (0x003f), "Both streams should be on the same volume")
+
+// Append block cannot be larger than the extent size
+DEFINE_DR_ERROR(DrError_AppendBlockTooLarge,        DR_ERROR (0x0040), "Append block is larger then the local limit for append or extent size")
+
+// Message was inconsistent
+DEFINE_DR_ERROR(DrError_InconsistentMessage,        DR_ERROR (0x0041), "Inconsistent message")
+
+// Message was inconsistent
+DEFINE_DR_ERROR(DrError_OutOfLimits,                DR_ERROR (0x0042), "The value was out of given limits")
+
+// Client specified a volume ID on the DRM that does not match the DRM's volume ID
+DEFINE_DR_ERROR(DrError_InvalidVolumeId,            DR_ERROR (0x0043), "The volume id was invalid")
+
+// Unsupported version
+DEFINE_DR_ERROR(DrError_UnsupportedVersion,         DR_ERROR (0x0044), "This version is not supported")
+
+// Client lib is not initizalized
+DEFINE_DR_ERROR(DrError_ClientNotInit,              DR_ERROR (0x0045), "Dryad Client Library is not initialized")
+
+// Inconsistent configuration
+DEFINE_DR_ERROR(DrError_InconsistentConfig,         DR_ERROR (0x0046), "Inconsistent INI Configuration")
+
+// One of hosts in chain failed with network error
+DEFINE_DR_ERROR(DrError_ChainedNetworkError,        DR_ERROR (0x0047), "One of hosts in chain failed with network error")
+
+// Process Property Version Mismatch
+DEFINE_DR_ERROR(DrError_ProcessPropertyVersionMismatch,        DR_ERROR (0x0048), "Process property version mismatch")
+
+// Current unsealed extent read from primary failed
+DEFINE_DR_ERROR(DrError_PrimaryFailed,              DR_ERROR (0x0049), "Current unsealed extent read from primary failed")
+
+// Async operation already completed
+DEFINE_DR_ERROR(DrError_AlreadyCompleted,           DR_ERROR (0x004a), "Async operation already completed")
+
+// Response message is invalid.
+DEFINE_DR_ERROR(DrError_InvalidReply,               DR_ERROR (0x004b), "Response message is invalid")
+
+// Internal error: required metadata is missing.
+DEFINE_DR_ERROR(DrError_MissingMetadata,            DR_ERROR (0x004c), "Internal error: required metadata is missing")
+
+// Internal error: required metadata is in inconsistent state.
+DEFINE_DR_ERROR(DrError_CorruptMetadata,            DR_ERROR (0x004d), "Internal error: required metadata is in inconsistent state")
+
+// Unable to seek over unsealed extent.
+DEFINE_DR_ERROR(DrError_NoSeekOverUnsealed,         DR_ERROR (0x004e), "Unable to seek over unsealed extent")
+
+// Too many files being retrieved by the cache manager
+DEFINE_DR_ERROR(DrError_TooManyFilesBeingRetrieved, DR_ERROR (0x004f), "Too many files being retrieved by the cache manager")
+
+// Primary extent instance is not present in retrieved metadata
+DEFINE_DR_ERROR(DrError_NoPrimary,                  DR_ERROR (0x0050), "Primary extent instance is not present in retrieved metadata")
+
+// Stream is read-only
+DEFINE_DR_ERROR(DrError_StreamReadOnly,             DR_ERROR (0x0051), "Stream is read-only")
+
+// Async operation was queued up successfully and it's now pending execution
+DEFINE_DR_ERROR(DrError_Pending,                    DR_ERROR (0x0052), "Operation is in progress")
+
+// Attempted operation is invalid in the given context
+DEFINE_DR_ERROR(DrError_InvalidOperation,           DR_ERROR (0x0053), "Executing operation is invalid in the given context")
+
+// Extent is not sealed
+DEFINE_DR_ERROR(DrError_ExtentNotSealed,            DR_ERROR (0x0054), "Extent is not sealed")
+
+// Duplicated resources
+DEFINE_DR_ERROR(DrError_DuplicatedResources,        DR_ERROR (0x0055), "This file exists; possible duplicated resources specified")
+
+// Unhandled STL exception
+DEFINE_DR_ERROR(DrError_InternalException,          DR_ERROR (0x0056), "Unhandled internal exception")
+
+// Pn Queue is full and can not admit a new process
+DEFINE_DR_ERROR(DrError_PnQueueFull,                DR_ERROR (0x0057), "Pn Queue Full")
+
+// Append request not at end of data
+DEFINE_DR_ERROR(DrError_AppendNotAtEnd,             DR_ERROR (0x0058), "Append request not at end of data")
+
+// Supplied password for Pn accounts is not correct
+DEFINE_DR_ERROR(DrError_IncorrectPnPassword,        DR_ERROR (0x0059), "Incorrect PN password")
+
+// Extent is not in active state. It's either deleted, recycled, incomplete or corrupt.
+DEFINE_DR_ERROR(DrError_ExtentNotActive,            DR_ERROR (0x0060), "Extent is not in active state; it is either deleted, recycled, incomplete or corrupt")
+
+// Seal operation is in progress.
+DEFINE_DR_ERROR(DrError_SealInProgress,             DR_ERROR (0x0061), "Seal operation is in progress")
+
+// Extent needs failure mode seal.
+DEFINE_DR_ERROR(DrError_ExtentNeedsSeal,            DR_ERROR (0x0062), "Extent needs failure mode seal")
+
+// Extent needs failure mode seal.
+DEFINE_DR_ERROR(DrError_UnknownMethod,              DR_ERROR (0x0063), "Unknown request or method")
+
+// not authorized
+DEFINE_DR_ERROR(DrError_NotAuthorized,              DR_ERROR (0x0064), "Client request cannot be authorized")
+
+// Syntax error
+DEFINE_DR_ERROR(DrError_SyntaxError,                DR_ERROR (0x0065), "Syntax error")
+
+// Attempt to suspend a task that was already suspended
+DEFINE_DR_ERROR(DrError_TaskAlreadySuspended,       DR_ERROR (0x0066), "Task already suspended")
+
+// Attempt to unsuspend a task that was already unsuspended
+DEFINE_DR_ERROR(DrError_TaskAlreadyUnsuspended,     DR_ERROR (0x0067), "Task already unsuspended")
+
+// Task does not exist
+DEFINE_DR_ERROR(DrError_TaskNotFound,               DR_ERROR (0x0068), "Task does not exist")
+
+// Task exists but it is in the deleted state
+DEFINE_DR_ERROR(DrError_TaskDeleted,                DR_ERROR (0x0069), "Task is in the deleted state")
+
+// Task already completed (e.g. you tried to suspend it when it was already completed)
+DEFINE_DR_ERROR(DrError_TaskAlreadyCompleted,       DR_ERROR (0x006A), "Task already completed")
+
+// Scheduler already paused
+DEFINE_DR_ERROR(DrError_SchedulerAlreadyPaused,     DR_ERROR (0x006B), "Scheduler already paused")
+
+// Scheduler already unpaused
+DEFINE_DR_ERROR(DrError_SchedulerAlreadyUnpaused,   DR_ERROR (0x006C), "Scheduler already unpaused")
+
+// DrServiceDescriptor name resolving errors
+DEFINE_DR_ERROR(DrError_ClusterNotFound,            DR_ERROR (0x006D), "Cluster not found")
+DEFINE_DR_ERROR(DrError_ServiceTypeNotFound,        DR_ERROR (0x006E), "Service type not found")
+DEFINE_DR_ERROR(DrError_NamespaceNotFound,          DR_ERROR (0x006F), "Namespace/volume not found")
+DEFINE_DR_ERROR(DrError_ServiceInstanceNotFound,    DR_ERROR (0x0070), "Service instance not found")
+
+// Job ticket is invalid
+DEFINE_DR_ERROR(DrError_JobTicketInvalid,           DR_ERROR (0x0071), "Job ticket is invalid")
+
+// Job ticket has expired
+DEFINE_DR_ERROR(DrError_JobTicketExpired,           DR_ERROR (0x0072), "Job ticket has expired")
+
+// Another authentication is in process
+DEFINE_DR_ERROR(DrError_AuthenticationInProcess,    DR_ERROR (0x0073), "Another authentication is in process")
+
+
+// i-th readahead request was satisfied with (i-1)-th
+DEFINE_DR_ERROR(DrError_ReadaheadAleadySatisfied,   DR_ERROR (0x0074), "Readahead satisfied by previous request")
+
+//
+// Cache service errors
+//
+DEFINE_DR_ERROR(DrError_CacheItemFetchIncomplete,   DR_ERROR (0x0075), "Cache Item is still being fetched from source")
+DEFINE_DR_ERROR(DrError_CacheItemFetchFailed,       DR_ERROR (0x0076), "Failed to fetch the cache item from source")
+DEFINE_DR_ERROR(DrError_CacheItemSourceInvalid,     DR_ERROR (0x0077), "The source of cache item is invalid")
+
+// PN process states
+DEFINE_DR_ERROR(DrError_PnProcessInitializing,     DR_ERROR (0x0078), "The process is being initialized")
+DEFINE_DR_ERROR(DrError_PnProcessCreated,     DR_ERROR (0x0079), "The process has been created")
+DEFINE_DR_ERROR(DrError_PnProcessRunning,     DR_ERROR (0x007A), "The process is running")
+DEFINE_DR_ERROR(DrError_PnProcessCreateFailed,     DR_ERROR (0x007B), "Process create failed")
+
+// VCWS errors
+DEFINE_DR_ERROR(DrError_VcwsVirtualClusterNotFound, DR_ERROR (0x007C), "Virtual cluster not found")
+DEFINE_DR_ERROR(DrError_VcwsMountPointNotFound,     DR_ERROR (0x007D), "Mount point not found")
+DEFINE_DR_ERROR(DrError_VcwsPermissionDenied,        DR_ERROR (0x007E), "Permission denied")
+DEFINE_DR_ERROR(DrError_VcwsInvalidMountPoint,        DR_ERROR (0x007F), "Invalid mount point")
+DEFINE_DR_ERROR(DrError_VcwsInvalidVirtualCluster,    DR_ERROR (0x0080), "Invalid virtual cluster")
+DEFINE_DR_ERROR(DrError_VcwsIncompleteMountPoint,  DR_ERROR (0x0081), "Incomplete mount point")
+DEFINE_DR_ERROR(DrError_DmRequestFailed,  DR_ERROR (0x0082), "Failed to send command to DM")
+DEFINE_DR_ERROR(DrError_CjsNoMaster,  DR_ERROR (0x0083), "No master of CJS machines from DM")
+
+//PN process states continued
+DEFINE_DR_ERROR(DrError_PnProcessSystemProcessRunning,     DR_ERROR (0x0084), "Could not create process due to System priority process.")
+
+DEFINE_DR_ERROR(DrError_XmlAttributeNotFound,     DR_ERROR (0x0085), "Attribute cannot be found in XML")
+DEFINE_DR_ERROR(DrError_VcwsShareNameAlreadyExists,     DR_ERROR (0x0086), "Share name has already exists in the virtual cluster")
+DEFINE_DR_ERROR(DrError_VcwsShareNotFound,     DR_ERROR (0x0087), "Share not found")
+DEFINE_DR_ERROR(DrError_VcwsMultipleShareNames,     DR_ERROR (0x0088), "This directory has already been shared with another share name")
+DEFINE_DR_ERROR(DrError_VcwsJobResourceTooLarge,     DR_ERROR (0x0089), "The size of some job resource is too large")
+DEFINE_DR_ERROR(DrError_VcwsTooManyLogAppendRequestsInProgress,     DR_ERROR (0x008a), "There are too many log append requests in progress and please re-try later.")
+DEFINE_DR_ERROR(DrError_VcwsInvalidVcPath,     DR_ERROR (0x008b), "The VC path format is invalid.")
+
+//PN process states continued
+DEFINE_DR_ERROR(DrError_PnDeploymentInProgress,     DR_ERROR (0x008C), "Could not create process due to deployment of cosmos code.")
+
+// Mirroring
+DEFINE_DR_ERROR (DrError_MirrorPolicyNotFound,     DR_ERROR (0x0090), "MirrorPolicyNotFound")
+
+// Returned when you try to perform some operation on a mirrored object that is not allowed; e.g. deleting a mirrored
+// stream or directory, or setting up mirroring for a directory that is already mirrored.
+DEFINE_DR_ERROR (DrError_ObjectIsMirrored, DR_ERROR (0x0091), "ObjectIsMirrored")
+
+DEFINE_DR_ERROR (DrError_DirectoryAlreadyMirrored, DR_ERROR (0x0092), "DirectoryAlreadyMirrored")
+DEFINE_DR_ERROR (DrError_MirrorInstanceNotFound, DR_ERROR (0x0093), "MirrorInstanceNotFound")
+
+// Returned when you try to perform an illegal operation on a mirror instance that is the master; e.g. trying to delete it
+DEFINE_DR_ERROR (DrError_MirrorInstanceIsMaster, DR_ERROR (0x0094), "MirrorInstanceIsMaster")
+
+DEFINE_DR_ERROR (DrError_MirrorInstanceAlreadyExists, DR_ERROR (0x0095), "MirrorInstanceAlreadyExists")
+DEFINE_DR_ERROR (DrError_DirectoryNotMirrored, DR_ERROR (0x0096), "DirectoryNotMirrored")
+
+// Returned when mirror policy name does not start with the "cluster.volume." prefix of the master
+DEFINE_DR_ERROR (DrError_InvalidMirrorPolicyName, DR_ERROR (0x0097), "InvalidMirrorPolicyName")
+
+DEFINE_DR_ERROR (DrError_MirrorPolicyAlreadyExists, DR_ERROR (0x0098), "MirrorPolicyAlreadyExists")
+
+DEFINE_DR_ERROR (DrError_UnalbeToAllocateExtentInstances,         DR_ERROR (0x0099), "Unable to allocate extent instances")
+
+// VCWS errors continued
+DEFINE_DR_ERROR(DrError_VcwsDrJobTimeout,  DR_ERROR (0x009a), "Your stream request could not be completed in time because of unusually high latency to the volume of your requested resource. Please try again later.")
+DEFINE_DR_ERROR(DrError_VcwsDrJobReject,  DR_ERROR (0x009b), "Your stream request is being rejected because too many timeouts to the volume of your requested resource have been observed. Please try again later.")
+
+// network send abort: map the same error in DrNetlib
+DEFINE_DR_ERROR (DrError_SendAborted,         DR_ERROR (0x009c), "Send to cosmos node aborted")
+
+// PN memory, execution time and max execution time exceeded, speculative process killed
+DEFINE_DR_ERROR (DrError_ExecutionQuotaExceeded,         DR_ERROR (0x009d), "Process exceeded execution time lease")
+DEFINE_DR_ERROR (DrError_MaxExecutionQuotaExceeded,         DR_ERROR (0x009e), "Process execution time exceeded upper threshold")
+DEFINE_DR_ERROR (DrError_MemoryQuotaExceeded,         DR_ERROR (0x009f), "Process exceeded memory quota")
+DEFINE_DR_ERROR (DrError_SpeculativeProcessAborted,         DR_ERROR (0x0110), "Speculative process aborted")
+DEFINE_DR_ERROR (DrError_ProcessTerminated,         DR_ERROR (0x0111), "Process terminated by client")
+
+// service is too busy processing requests on the stream
+DEFINE_DR_ERROR (DrError_StreamTooBusy,             DR_ERROR (0x0112), "Service is too busy processing requests of this stream")
+
+// VCWS errors continued
+DEFINE_DR_ERROR(DrError_VcwsInvalidShareName,     DR_ERROR (0x0120), "There are invalid characters in share name and the list of all invalid characters is \"ASCII 1-31, \", *, :, <, >, ?, \\, <space>, %%, [, ], (, ), &, ;, /, {, }, #\"")
+DEFINE_DR_ERROR(DrError_VcwsPermissionDeniedByMountPointSecurityGroups,     DR_ERROR (0x0121), "You are not in the security groups associated with this mount point. Please ask your VC admin for specific information.")
+
+// StreamSet errors
+DEFINE_DR_ERROR(DrError_StreamSetTemplateEmpty,     DR_ERROR (0x0130), "[StreamSet:Template] in StreamSet.ini is of zero length.")
+DEFINE_DR_ERROR(DrError_InvalidStreamSetHotSpot,     DR_ERROR (0x0131), "[StreamSet:HotSpot] in StreamSet.ini is invalid.")
+DEFINE_DR_ERROR(DrError_InvalidStreamSetExpireAfter,     DR_ERROR (0x0132), "[StreamSet:ExpireAfter] in StreamSet.ini is invalid.")
+DEFINE_DR_ERROR(DrError_MultipleStreamsForAppend,     DR_ERROR (0x0133), "A streamset being mapped to multiple streams is specified for appending.")
+
+// OpenFromBinary error (clientlib)
+DEFINE_DR_ERROR(DrError_CannotCreateFromBinary,     DR_ERROR (0x0134), "A stream cannot be created from binary stream info.")
+
+// APX Client errors
+DEFINE_DR_ERROR(DrError_ApxAddressNotFound,     DR_ERROR (0x0140), "No available Autopilot authentication proxy IP address could be found.")
+DEFINE_DR_ERROR(DrError_ApxUnexpectedHttpStatusCode,     DR_ERROR (0x0141), "An unexpected HTTP status code received from Autopilot authentication proxy.")
+
+// Dryad Partition Manager
+DEFINE_DR_ERROR(DrError_InvalidPartitionEntry,      DR_ERROR (0x0150), "Invalid Dryad partition entry.")
+DEFINE_DR_ERROR(DrError_InvalidPartitionTable,      DR_ERROR (0x0151), "Invalid Dryad partition table.")
+DEFINE_DR_ERROR(DrError_PartitionNotFound,          DR_ERROR (0x0152), "Dryad partition not found.")
+DEFINE_DR_ERROR(DrError_PartitionKeyNotFound,       DR_ERROR (0x0153), "Dryad partition key not found.")
+DEFINE_DR_ERROR(DrError_PartitionServerUnavailable, DR_ERROR (0x0154), "Dryad Partition Manager unavailable.")
+
+//
+// Cache service errors
+//
+DEFINE_DR_ERROR(DrError_CacheItemSizeExceedCacheSize,   DR_ERROR (0x0160), "The size of the cache item requested is greater than the stable size of the cache")
+
+//
+// HPC Errors
+//
+DEFINE_DR_ERROR(DrError_InvalidJob,                 DR_ERROR (0x0170), "Invalid HPC Job ID.")
+
+
+// The error that should never be returned!
+// This is useful for initializing error variables so we can test if they
+// got set correctly at some point.
+// ! DONT RETURN THIS AS AN ERROR EVER !
+
+// An error code was not initialized
+DEFINE_DR_ERROR(DrError_Impossible,                 DR_ERROR (0xFFFF), "An error code was not initialized")
+
+
+
+
+
+/* ------------------------------------ DrIo errors (should eventually go away) --------------------------------- */
+
+
+// Internal Error
+DEFINE_DR_ERROR (DrError_IoInternalError,           DR_ERROR (0x0100), "Internal Error")
+
+// Extent already Exists
+DEFINE_DR_ERROR (DrError_IoExtentAlreadyExists,     DR_ERROR (0x0101), "Extent already Exists")
+
+// Extent instance already deleted
+DEFINE_DR_ERROR (DrError_IoExtentAlreadyDeleted,    DR_ERROR (0x0102), "Extent instance already deleted")
+
+// Extent instance is corrupted
+DEFINE_DR_ERROR (DrError_IoExtentCorrupted,         DR_ERROR (0x0103), "Extent instance is corrupted")
+
+// Cannot open extent instance
+DEFINE_DR_ERROR (DrError_IoExtentCannotOpen,        DR_ERROR (0x0104), "Cannot open extent instance")
+
+// The volume is full
+DEFINE_DR_ERROR (DrError_IoVolumeIsFull,            DR_ERROR (0x0105), "The volume is full")
+
+// Dryad volume is corrupted
+DEFINE_DR_ERROR (DrError_IoVolumeCorrupted,         DR_ERROR (0x0106), "Dryad volume is corrupted")
+
+// Unknown Dryad volume
+DEFINE_DR_ERROR (DrError_IoVolumeUnknown,           DR_ERROR (0x0107), "Unknown Dryad volume")
+
+// An I/O operation is pending
+DEFINE_DR_ERROR (DrError_IoPending,                 DR_ERROR (0x0108), "An I/O operation is pending")
+
+// An I/O error occurred
+DEFINE_DR_ERROR (DrError_IoReadWriteError,          DR_ERROR (0x0109), "An I/O error occurred")
+
+// CRC mismatch
+DEFINE_DR_ERROR (DrError_IoCorruptedData,           DR_ERROR (0x010a), "CRC mismatch")
+
+#ifndef COMMON_DR_ERRORS_DEFINED
+// Extent instance not found
+DEFINE_DR_ERROR (DrError_IoExtentNotFound,          DR_ERROR (0x010b), "Extent instance not found")
+#endif
+
+// Stream is sealed
+DEFINE_DR_ERROR(DrError_StreamSealed,               DR_ERROR (0x0200), "Stream is sealed")
+
+// Stream has changed
+DEFINE_DR_ERROR(DrError_StreamChanged,          DR_ERROR (0x0201), "Stream ID or length has changed")
+
+
+#ifndef COMMON_DR_ERRORS_DEFINED
+#define COMMON_DR_ERRORS_DEFINED
+#endif
+

DryadVertex/VertexHost/system/classlib/include/DrErrorDef.h

@@ -0,0 +1,77 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+typedef HRESULT DrError;
+
+class DrStr;
+
+extern DrStr& DrAppendErrorDescription(DrStr& strOut, DrError err);
+//JC extern DrWStr& DrAppendErrorDescription(DrWStr& strOut, DrError err);
+
+extern void DrInitErrorTable(void);
+extern DrError DrAddErrorDescription(DrError code, const char *pszDescription);
+
+// The returned error string should be freed with free();
+//
+// If the error code is unknown, a generic error description is returned.
+extern char *DrGetErrorText(DrError err);
+
+// The returned error string should be freed with free();
+//
+// If the error code is unknown, a generic error description is returned.
+//JC extern WCHAR *DrGetErrorTextW(DrError err);
+
+// The buffer must be at least 64 bytes long to guarantee a result. If the result won't fit in the buffer, a generic
+// error message is generated.
+extern const char *DrGetErrorDescription(DrError err, char *pBuffer, int buffLen);
+
+// The buffer must be at least 64 chars long to guarantee a result. If the result won't fit in the buffer, a generic
+// error message is generated.
+//JC extern const WCHAR *DrGetErrorDescription(DrError err, WCHAR *pBuffer, int buffLen);
+
+static const int k_DrMaxErrorLength = 256;
+
+#ifndef FACILITY_DR
+#define FACILITY_DR   0x309
+#endif
+
+#ifndef DR_ERROR
+#define DR_ERROR(n) MAKE_HRESULT(SEVERITY_ERROR, FACILITY_DR, n)
+#endif
+
+#ifdef DEFINE_DR_ERROR
+#undef DEFINE_DR_ERROR
+#endif
+
+#define DEFINE_DR_ERROR(name, number, description) static const DrError name = number;
+
+#include "DrError.h"
+
+#undef DEFINE_DR_ERROR
+
+#if defined(__cplusplus)
+}
+#endif

DryadVertex/VertexHost/system/classlib/include/DrExecution.h

@@ -0,0 +1,103 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#ifndef _DrExecution_h_
+#define  _DrExecution_h_
+
+#pragma once
+
+
+/*
+    DrExecution.h - main header file for Dryad execution
+*/
+
+#include "DrCommon.h"
+
+//JC#include "DrFileCache.h"
+//JC#include "DrPipe.h"
+//JC#include "DrPipeLoopback.h"
+//JC#include "DrNativeHandlePipe.h"
+//JC#include "DrTcpPipe.h"
+//JC#include "DrPipeRendezvous.h"
+//JC#include "DrJobStatistics.h"
+
+
+//JC#define DR_ENVSTR_PN_QUOTA "PN_QUOTA"
+
+DrError DrInitExecution();
+
+class DrExecutionEnvironment
+{
+    friend DrError DrInitExecution();
+
+public:
+
+    void Lock()
+    {
+        m_cs.Enter();
+    }
+
+    void Unlock()
+    {
+        m_cs.Leave();
+    }
+
+	/* JC
+    void GetCurrentJobDescriptor(DrJobDescriptorEx& jdOut)
+    {
+        Lock();
+        jdOut = g_pDryadConfig->CurrentJobDescriptor();
+        Unlock();
+    }
+
+    const DrProcessDescriptor& GetCurrentProcessDescriptor()
+    {
+        return g_pDryadConfig->GetCurrentProcessDescriptor();
+    }
+
+    const DrProcessDescriptor& GetRootProcessDescriptor()
+    {
+        return g_pDryadConfig->GetRootProcessDescriptor();
+    }
+
+    bool IsUnderProcessNode()
+    {
+        return g_pDryadConfig->IsUnderProcessNode();
+    }
+
+    bool JobWasInherited()
+    {
+        return g_pDryadConfig->JobWasInherited();
+    }
+*/
+private:
+    DrExecutionEnvironment()
+    {
+    }
+
+    DrError Initialize();
+
+private:
+    DrCriticalSection m_cs;
+};
+
+extern DrExecutionEnvironment *g_pDryadExecution;
+
+#endif // end if not defined  _DrExecution_h_

DryadVertex/VertexHost/system/classlib/include/DrExitCodes.h

@@ -0,0 +1,72 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+// DrExitCodes.h
+//
+// This file must contain *only* declarions of the form:
+//
+// DEFINE_DREXITCODE(name, value, description)
+// DEFINE_DREXITCODE_NO_DESC(name, value)
+// DECLARE_DREXITCODE(valname, description)
+// DECLARE_DREXITCODE_NO_DESC(valname)
+//
+// It is included multiple times with different macro definitions.
+
+// Values from ntstatus.h:
+#define STATUS_NOT_SUPPORTED             ((DWORD)0xC00000BBL)
+#define STATUS_UNHANDLED_EXCEPTION       ((DWORD)0xC0000144L)
+//#define STATUS_ASSERTION_FAILURE         ((NTSTATUS)0xC0000420L)
+
+
+DEFINE_DREXITCODE_NO_DESC(DrExitCode_OK, 0)
+DEFINE_DREXITCODE_NO_DESC(DrExitCode_Fail, 1)
+DEFINE_DREXITCODE_NO_DESC(DrExitCode_StillActive, STILL_ACTIVE)
+DEFINE_DREXITCODE_NO_DESC(DrExitCode_Killed, (DrExitCode) STATUS_CONTROL_C_EXIT)
+DEFINE_DREXITCODE_NO_DESC(DrExitCode_JobQuotaExceeded, ERROR_NOT_ENOUGH_QUOTA)
+
+DECLARE_DREXITCODE_NO_DESC(STATUS_INVALID_HANDLE)
+DECLARE_DREXITCODE_NO_DESC(STATUS_ACCESS_VIOLATION)
+DECLARE_DREXITCODE_NO_DESC(STATUS_ARRAY_BOUNDS_EXCEEDED)
+DECLARE_DREXITCODE_NO_DESC(STATUS_BREAKPOINT)
+DECLARE_DREXITCODE_NO_DESC(STATUS_DATATYPE_MISALIGNMENT)
+DECLARE_DREXITCODE_NO_DESC(STATUS_FLOAT_DENORMAL_OPERAND)
+DECLARE_DREXITCODE_NO_DESC(STATUS_FLOAT_DIVIDE_BY_ZERO)
+DECLARE_DREXITCODE_NO_DESC(STATUS_FLOAT_INEXACT_RESULT)
+DECLARE_DREXITCODE_NO_DESC(STATUS_FLOAT_INVALID_OPERATION)
+DECLARE_DREXITCODE_NO_DESC(STATUS_FLOAT_MULTIPLE_FAULTS)
+DECLARE_DREXITCODE_NO_DESC(STATUS_FLOAT_MULTIPLE_TRAPS)
+DECLARE_DREXITCODE_NO_DESC(STATUS_FLOAT_OVERFLOW)
+DECLARE_DREXITCODE_NO_DESC(STATUS_FLOAT_STACK_CHECK)
+DECLARE_DREXITCODE_NO_DESC(STATUS_FLOAT_UNDERFLOW)
+DECLARE_DREXITCODE_NO_DESC(STATUS_GUARD_PAGE_VIOLATION)
+DECLARE_DREXITCODE_NO_DESC(STATUS_ILLEGAL_INSTRUCTION)
+DECLARE_DREXITCODE_NO_DESC(STATUS_IN_PAGE_ERROR)
+DECLARE_DREXITCODE_NO_DESC(STATUS_INVALID_DISPOSITION)
+DECLARE_DREXITCODE_NO_DESC(STATUS_INTEGER_DIVIDE_BY_ZERO)
+DECLARE_DREXITCODE_NO_DESC(STATUS_INTEGER_OVERFLOW)
+DECLARE_DREXITCODE_NO_DESC(STATUS_NONCONTINUABLE_EXCEPTION)
+DECLARE_DREXITCODE_NO_DESC(STATUS_NOT_SUPPORTED)
+DECLARE_DREXITCODE_NO_DESC(STATUS_PRIVILEGED_INSTRUCTION)
+DECLARE_DREXITCODE_NO_DESC(STATUS_REG_NAT_CONSUMPTION)
+DECLARE_DREXITCODE_NO_DESC(STATUS_SINGLE_STEP)
+DECLARE_DREXITCODE_NO_DESC(STATUS_STACK_OVERFLOW)
+DECLARE_DREXITCODE_NO_DESC(STATUS_ASSERTION_FAILURE)
+DECLARE_DREXITCODE_NO_DESC(STATUS_UNHANDLED_EXCEPTION)
+

DryadVertex/VertexHost/system/classlib/include/DrExitCodesDef.h

@@ -0,0 +1,64 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+typedef UInt32 DrExitCode;
+
+void DrInitExitCodeTable();
+DrError DrAddExitCodeDescription(DrExitCode code, const char *pszDescription);
+DrStr& DrAppendExitCodeDescription(DrStr& strOut, DrExitCode code);
+DrStr& DrGetExitCodeDescription(DrStr& strOut, DrExitCode code);
+
+class DrExitCodeString : public DrStr128
+{
+public:
+    DrExitCodeString(DrExitCode code)
+    {
+        DrGetExitCodeDescription(*this, code);
+    }
+};
+
+// This macro can be used to obtain a temporary "const char *" error description for an exit code. It can be used
+// as the parameter to a method call; the pointer will become invalid after the function returns
+#define DREXITCODESTRING(code)    (DrExitCodeString(code).GetString())
+#define DREXITCODEWSTRING(code)    (DRUTF8TOWSTRING(DREXITCODESTRING(code)))
+
+
+
+#define DEFINE_DREXITCODE_NO_DESC(name, value) DEFINE_DREXITCODE(name, (DrExitCode)(value), #name)
+#define DECLARE_DREXITCODE_NO_DESC(valname) DECLARE_DREXITCODE(valname, #valname)
+
+#ifdef DEFINE_DREXITCODE
+#undef DEFINE_DREXITCODE
+#endif
+#ifdef DECLARE_DREXITCODE
+#undef DECLARE_DREXITCODE
+#endif
+#define DEFINE_DREXITCODE(name, value, description) const DrExitCode name = (DrExitCode)(value);
+#define DECLARE_DREXITCODE(valname, description)
+
+#include "DrExitCodes.h"
+
+#undef DEFINE_DREXITCODE
+#undef DECLARE_DREXITCODE
+
+
+

DryadVertex/VertexHost/system/classlib/include/DrFPrint.h

@@ -0,0 +1,193 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+/* (c) Microsoft Corporation.  All rights reserved. */
+#pragma once
+#include <string.h>
+#include "basic_types.h"
+
+#undef IndexAssert
+#define IndexAssert(expr) LogAssert(expr)
+typedef __int64 Dryad_dupelim_fprint_int64_t;
+typedef UInt64 Dryad_dupelim_fprint_uint64_t;
+
+/* the type of a 64-bit fingerprint */
+typedef Dryad_dupelim_fprint_uint64_t Dryad_dupelim_fprint_t;
+
+/* an opaque type used to keep the data structures need to compute
+   fingerprints.  */
+
+typedef struct Dryad_dupelim_fprint_data_s *Dryad_dupelim_fprint_data_t;
+typedef const struct Dryad_dupelim_fprint_data_s *Dryad_dupelim_fprint_data_tc;
+
+/* hash lengths */
+enum HashPolyLength{
+    Poly8bit, 
+    Poly16bit,
+    Poly32bit,
+    Poly64bit} ;
+
+/* Allocate and return a new Rabin fingerprint function.
+   Rabin fingerprint belongs to the family of CRC hashes
+   Its collusion is bounded by a very small number
+   Since it employs polynomials in a galois field, it is very 
+   efficient in calculating recursive hashes
+
+   
+   for straight-forward applications, use Dryad_dupelim_rabinhash_create() and 
+   Dryad_dupelim_rabinhash_process() in your applications. unless you 
+   understand what the other functions exactly do, refrain from using them.
+
+   Dryad_dupelim_rabinhash_new()----------------------
+   returns true if a hash function is created, 
+   fprint data structure should already been allocated
+   returns a pointer to the hash function created on pHashfunction
+   hashLen is the order of polynomials to be used for the hash function
+   seed is the index of the polynomial to be used in the hash function
+   seed has to be less than or equal to cbPolysN {N = 8 , 16, 32, 64}
+        otherwise, Dryad_dupelim_rabinhash_new will return false
+*/
+bool Dryad_dupelim_rabinhash_init (Dryad_dupelim_fprint_data_s* pHashData, 
+                                HashPolyLength hashLen,
+				UInt32 seed);
+/* if fp was generated with polynomial P,bytes "data[0, ..., len-1]" 
+   contain string A, return the fingerprint under P of A.
+   Strings are treated as polynomials.  The low-order bit in the first
+   byte is the highest degree coefficient in the polynomial. 
+*/
+Dryad_dupelim_fprint_t Dryad_dupelim_rabinhash_process(Dryad_dupelim_fprint_data_s* pHashData,
+                                                   const unsigned char *data, unsigned len);
+
+/* if fp was generated with polynomial P,bytes "data[0, ..., len-1]" 
+   contain string B, and initialHash contains the hash value for string A
+   return the fingerprint under P of A added to initialHash.
+   the output value is merely the hash of string A concat string B.
+   Strings are treated as polynomials.  The low-order bit in the first
+   byte is the highest degree coefficient in the polynomial. 
+*/
+
+Dryad_dupelim_fprint_t Dryad_dupelim_rabinhash_add(Dryad_dupelim_fprint_data_s* pHashFunction, Dryad_dupelim_fprint_t initialHash,
+                                               const unsigned char *data, unsigned len);
+
+
+/* Allocate and return a new fingerprint function.
+
+   Computes the tables needed for fingerprint manipulations.
+   Requires that "poly" be the binary representation
+   of an irreducible polynomial in GF(2) of degree 64.   The X^64 term
+   is not represented.  The X^0 term is the high order bit, and the
+   X^63 term is the low-order bit.
+
+   span is used in later calls to Dryad_dupelim_fprint_slide_word().
+   If Dryad_dupelim_fprint_slide_word() is not to be called, span
+   should be set to zero. */
+Dryad_dupelim_fprint_data_t Dryad_dupelim_fprint_new (Dryad_dupelim_fprint_t poly,
+						  unsigned span);
+
+/* Like "new" above, except that the degree can be any value between 1
+   and 64.  Return 0 if that's not true.
+
+   The X^(degree-1) term is in the low-order bit of poly.
+*/
+Dryad_dupelim_fprint_data_t Dryad_dupelim_fprint_new2 (Dryad_dupelim_fprint_t poly,
+						   unsigned span, int degree);
+
+/* returns the seeded polynomial ie. fingerprint of an empty element under this fp */
+Dryad_dupelim_fprint_t Dryad_dupelim_fprint_empty (Dryad_dupelim_fprint_data_tc fp);
+
+/* if fp was generated with polynomial P, "a" is the fingerprint under
+   P of string A, and bytes "data[0, ..., len-1]" contain string B,
+   return the fingerprint under P of the concatenation of A and B.
+   Strings are treated as polynomials.  The low-order bit in the first
+   byte is the highest degree coefficient in the polynomial. This
+   routine differs from Dryad_dupelim_fprint_extend_word() in that it
+   will read bytes in increasing address order, regardless of the
+   endianness of the  machine.  
+   data's length is the number of unsigned chars
+*/
+Dryad_dupelim_fprint_t Dryad_dupelim_fprint_extend (Dryad_dupelim_fprint_data_tc fp,
+						Dryad_dupelim_fprint_t a,
+                                                const unsigned char *data, unsigned len);
+
+/* If fp was generated with polynomial P, "a" is the fingerprint under
+   P of string A, and 64-bit words "data[0, ..., len-1]" contain
+   string B, return the fingerprint under P of the concatenation of A
+   and B.  Arrays of words are treated as polynomials.  The low-order
+   bit in the first word is the highest degree coefficient in the
+   polynomial. This routine differs from Dryad_dupelim_fprint_extend()
+   on bigendian machines, where the byte order within each word is
+   backwards. */
+Dryad_dupelim_fprint_t
+Dryad_dupelim_fprint_extend_word (Dryad_dupelim_fprint_data_tc fp,
+				Dryad_dupelim_fprint_t a,
+                const Dryad_dupelim_fprint_uint64_t *data,
+				unsigned len);
+
+/* if fp was generated with polynomial P, "a" is the fingerprint under
+   P of string A, and "b" is the fingerprint under P of string B,
+   which has length "blen" bytes, return the fingerprint under P of
+   the concatenation of A and B */
+Dryad_dupelim_fprint_t
+Dryad_dupelim_fprint_concat(Dryad_dupelim_fprint_data_tc fp,
+			  Dryad_dupelim_fprint_t a, Dryad_dupelim_fprint_t
+			  b, Dryad_dupelim_fprint_t blen);
+
+
+/* Turn fingerprint "f" into a hexadecimal, ascii-zero-filled
+   printable string S of length 16, and place the characters in
+   buf[0,...,15].  No null terminator is written by the routine. */
+void Dryad_dupelim_fprint_toascii (Dryad_dupelim_fprint_t f, char *buf);
+
+/* if "fp" was generated with polynomial P, X is some string of length
+   "(span-1)*sizeof (Dryad_dupelim_fprint_uint64_t)" bytes  (see
+   Dryad_dupelim_fprint_new()), "f" is the fingerprint under P of word
+   "a" concatenated with X, return the fingerprint under P of X
+   concatenated with word "b".  The words "a" and "b" represent
+   polynomials whose X^0 term is in the high-order bit, and whose X^63
+   term is in the low order bit.  */
+Dryad_dupelim_fprint_t
+Dryad_dupelim_fprint_slideword (Dryad_dupelim_fprint_data_tc fp,
+			      Dryad_dupelim_fprint_t f,
+			      Dryad_dupelim_fprint_uint64_t a,
+			      Dryad_dupelim_fprint_uint64_t b);
+
+/* discard the data associated with "fp" */
+void Dryad_dupelim_fprint_close (Dryad_dupelim_fprint_data_t fp);
+
+/* fprint struct */
+struct Dryad_dupelim_fprint_data_s {
+    Dryad_dupelim_fprint_t poly[2];
+    /* poly[0] = 0; poly[1] = polynomial */
+    Dryad_dupelim_fprint_t empty;
+    /* fingerprint of the empty string */
+    Dryad_dupelim_fprint_t bybyte[8][256];
+    /* bybyte[b][i] is i*X^(degree+8*b) mod poly[1] */
+    Dryad_dupelim_fprint_t powers[64];
+    /* extend[i] is X^(8*2^i) mod poly[1] */
+    static const UInt32 LOGZEROBLOCK = 8;
+    static const UInt32 ZEROBLOCK = (1 << LOGZEROBLOCK); 
+    union {
+        double align;
+        unsigned char zeroes[ZEROBLOCK];
+    } zeroes;
+    Dryad_dupelim_fprint_t bybyte_out[8][256];
+    /* bybyte_out[b][i] is i*X^(degree+8*(b+span)) mod poly[1] */
+    unsigned span;
+};

DryadVertex/VertexHost/system/classlib/include/DrFPrint_polynomials.h

@@ -0,0 +1,190 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+// This file defines 4 sets of GF[2] irreducible polynomials
+// These polynomials are of the order 8, 16, 32 and 64
+// You can use these to initialize fprint_data 
+
+static const Dryad_dupelim_fprint_t polys8[] = {
+  (Dryad_dupelim_fprint_t)0x00d4UL,
+  (Dryad_dupelim_fprint_t)0x00b4UL,
+  (Dryad_dupelim_fprint_t)0x00b1UL,
+  (Dryad_dupelim_fprint_t)0x00b2UL,
+  (Dryad_dupelim_fprint_t)0x0095UL,
+  (Dryad_dupelim_fprint_t)0x00afUL,
+  (Dryad_dupelim_fprint_t)0x00b2UL,
+  (Dryad_dupelim_fprint_t)0x00a6UL,
+  (Dryad_dupelim_fprint_t)0x0096UL
+};
+
+static const Dryad_dupelim_fprint_t polys16[] = {
+  (Dryad_dupelim_fprint_t)0x00009ee6UL,
+  (Dryad_dupelim_fprint_t)0x0000dfb5UL,
+  (Dryad_dupelim_fprint_t)0x0000e95dUL,
+  (Dryad_dupelim_fprint_t)0x0000ab23UL,
+  (Dryad_dupelim_fprint_t)0x00009566UL,
+  (Dryad_dupelim_fprint_t)0x0000d5e9UL,
+  (Dryad_dupelim_fprint_t)0x000086c1UL,
+  (Dryad_dupelim_fprint_t)0x000082c3UL,
+  (Dryad_dupelim_fprint_t)0x0000a485UL,
+  (Dryad_dupelim_fprint_t)0x00008b55UL,
+  (Dryad_dupelim_fprint_t)0x00008b4dUL,
+  (Dryad_dupelim_fprint_t)0x0000883fUL,
+  (Dryad_dupelim_fprint_t)0x0000bd5cUL,
+  (Dryad_dupelim_fprint_t)0x0000e87dUL,
+  (Dryad_dupelim_fprint_t)0x000082b4UL,
+  (Dryad_dupelim_fprint_t)0x0000c036UL,
+  (Dryad_dupelim_fprint_t)0x000097e9UL,
+  (Dryad_dupelim_fprint_t)0x00009e98UL,
+  (Dryad_dupelim_fprint_t)0x000099f9UL,
+  (Dryad_dupelim_fprint_t)0x0000fa93UL,
+};
+
+static const Dryad_dupelim_fprint_t polys32[] = {
+  (Dryad_dupelim_fprint_t)0x8b950699UL,
+  (Dryad_dupelim_fprint_t)0xf8c45e2aUL,
+  (Dryad_dupelim_fprint_t)0xdfdac578UL,
+  (Dryad_dupelim_fprint_t)0x896f6717UL,
+  (Dryad_dupelim_fprint_t)0xb2ab5f5dUL,
+  (Dryad_dupelim_fprint_t)0xece51013UL,
+  (Dryad_dupelim_fprint_t)0xc9ed9c7bUL,
+  (Dryad_dupelim_fprint_t)0xb2b28a80UL,
+  (Dryad_dupelim_fprint_t)0xb03c9ed2UL,
+  (Dryad_dupelim_fprint_t)0x85cd5087UL,
+  (Dryad_dupelim_fprint_t)0xcb7d544eUL,
+  (Dryad_dupelim_fprint_t)0xf090b664UL,
+  (Dryad_dupelim_fprint_t)0xfe442fe2UL,
+  (Dryad_dupelim_fprint_t)0x80a0adc0UL,
+  (Dryad_dupelim_fprint_t)0x9132521fUL,
+  (Dryad_dupelim_fprint_t)0xeca10123UL,
+  (Dryad_dupelim_fprint_t)0xf06b52c3UL,
+  (Dryad_dupelim_fprint_t)0x87b146b5UL,
+  (Dryad_dupelim_fprint_t)0xc6b63122UL,
+  (Dryad_dupelim_fprint_t)0xaa109fabUL
+};
+
+const Dryad_dupelim_fprint_t polys64[] = {
+  (((Dryad_dupelim_fprint_t)0xb40ab24eUL) << 32) | (Dryad_dupelim_fprint_t)0x49737109UL, 
+  (((Dryad_dupelim_fprint_t)0xc0398760UL) << 32) | (Dryad_dupelim_fprint_t)0xd3108fd6UL, 
+  (((Dryad_dupelim_fprint_t)0xd869093fUL) << 32) | (Dryad_dupelim_fprint_t)0x2ebec587UL, 
+  (((Dryad_dupelim_fprint_t)0xa6ab08f8UL) << 32) | (Dryad_dupelim_fprint_t)0x00c128c9UL, 
+  (((Dryad_dupelim_fprint_t)0xa629a9c4UL) << 32) | (Dryad_dupelim_fprint_t)0x60a8edfbUL, 
+  (((Dryad_dupelim_fprint_t)0xd422e286UL) << 32) | (Dryad_dupelim_fprint_t)0x78b47614UL, 
+  (((Dryad_dupelim_fprint_t)0x93facdf9UL) << 32) | (Dryad_dupelim_fprint_t)0xbc1363a2UL, 
+  (((Dryad_dupelim_fprint_t)0x93caa3c5UL) << 32) | (Dryad_dupelim_fprint_t)0xdd40d768UL, 
+  (((Dryad_dupelim_fprint_t)0xaa53204aUL) << 32) | (Dryad_dupelim_fprint_t)0x7969914eUL, 
+  (((Dryad_dupelim_fprint_t)0xe2415fb3UL) << 32) | (Dryad_dupelim_fprint_t)0x440a16bbUL, 
+  (((Dryad_dupelim_fprint_t)0xa05f3d02UL) << 32) | (Dryad_dupelim_fprint_t)0x95be208fUL, 
+  (((Dryad_dupelim_fprint_t)0xb1e61188UL) << 32) | (Dryad_dupelim_fprint_t)0x6ec27c88UL, 
+  (((Dryad_dupelim_fprint_t)0xd6d2bc63UL) << 32) | (Dryad_dupelim_fprint_t)0xc91d290eUL, 
+  (((Dryad_dupelim_fprint_t)0xf80f25b8UL) << 32) | (Dryad_dupelim_fprint_t)0xc1930eccUL, 
+  (((Dryad_dupelim_fprint_t)0x97dc1fd1UL) << 32) | (Dryad_dupelim_fprint_t)0x15e0e70eUL, 
+  (((Dryad_dupelim_fprint_t)0xe17f23cdUL) << 32) | (Dryad_dupelim_fprint_t)0x55fe08aeUL, 
+  (((Dryad_dupelim_fprint_t)0xd309c54aUL) << 32) | (Dryad_dupelim_fprint_t)0xe0d66600UL, 
+  (((Dryad_dupelim_fprint_t)0xb55bd691UL) << 32) | (Dryad_dupelim_fprint_t)0x17e20f21UL, 
+  (((Dryad_dupelim_fprint_t)0x9b19a5d4UL) << 32) | (Dryad_dupelim_fprint_t)0xd4f5ccbeUL, 
+  (((Dryad_dupelim_fprint_t)0xcbca35d9UL) << 32) | (Dryad_dupelim_fprint_t)0xab901b9bUL, 
+  (((Dryad_dupelim_fprint_t)0x889417edUL) << 32) | (Dryad_dupelim_fprint_t)0x965534ddUL, 
+  (((Dryad_dupelim_fprint_t)0x8f27c100UL) << 32) | (Dryad_dupelim_fprint_t)0xbd898837UL, 
+  (((Dryad_dupelim_fprint_t)0x930fc2d3UL) << 32) | (Dryad_dupelim_fprint_t)0x4cc207e3UL, 
+  (((Dryad_dupelim_fprint_t)0xba0920c3UL) << 32) | (Dryad_dupelim_fprint_t)0xf1c7b364UL, 
+  (((Dryad_dupelim_fprint_t)0x80d46b49UL) << 32) | (Dryad_dupelim_fprint_t)0xcfadf5ccUL, 
+  (((Dryad_dupelim_fprint_t)0xb45b9d25UL) << 32) | (Dryad_dupelim_fprint_t)0x2b5d6071UL, 
+  (((Dryad_dupelim_fprint_t)0x9fe4d82fUL) << 32) | (Dryad_dupelim_fprint_t)0x5fd432d2UL, 
+  (((Dryad_dupelim_fprint_t)0xa97d6763UL) << 32) | (Dryad_dupelim_fprint_t)0xd5f818b3UL, 
+  (((Dryad_dupelim_fprint_t)0xe8d6b0beUL) << 32) | (Dryad_dupelim_fprint_t)0x7c43649dUL, 
+  (((Dryad_dupelim_fprint_t)0xbc673c33UL) << 32) | (Dryad_dupelim_fprint_t)0xfbe55129UL, 
+  (((Dryad_dupelim_fprint_t)0xec03ce27UL) << 32) | (Dryad_dupelim_fprint_t)0xf7509ae5UL, 
+  (((Dryad_dupelim_fprint_t)0x808401d4UL) << 32) | (Dryad_dupelim_fprint_t)0x40abf627UL, 
+  (((Dryad_dupelim_fprint_t)0x95c51b3dUL) << 32) | (Dryad_dupelim_fprint_t)0x387ce64bUL, 
+  (((Dryad_dupelim_fprint_t)0xa5a59bd2UL) << 32) | (Dryad_dupelim_fprint_t)0x7d3f452dUL, 
+  (((Dryad_dupelim_fprint_t)0xe429f8beUL) << 32) | (Dryad_dupelim_fprint_t)0x22291027UL, 
+  (((Dryad_dupelim_fprint_t)0xe4764c26UL) << 32) | (Dryad_dupelim_fprint_t)0x913308e0UL, 
+  (((Dryad_dupelim_fprint_t)0xafd52ea1UL) << 32) | (Dryad_dupelim_fprint_t)0x35797bdaUL, 
+  (((Dryad_dupelim_fprint_t)0xeb04bdfeUL) << 32) | (Dryad_dupelim_fprint_t)0xa0163482UL, 
+  (((Dryad_dupelim_fprint_t)0x9e81f8b8UL) << 32) | (Dryad_dupelim_fprint_t)0xd63a6b87UL, 
+  (((Dryad_dupelim_fprint_t)0xd320f803UL) << 32) | (Dryad_dupelim_fprint_t)0x485563aeUL, 
+  (((Dryad_dupelim_fprint_t)0x8af88fe4UL) << 32) | (Dryad_dupelim_fprint_t)0x09983363UL, 
+  (((Dryad_dupelim_fprint_t)0xd66102feUL) << 32) | (Dryad_dupelim_fprint_t)0xf6ccfe37UL, 
+  (((Dryad_dupelim_fprint_t)0xa93e4704UL) << 32) | (Dryad_dupelim_fprint_t)0x3985cda0UL, 
+  (((Dryad_dupelim_fprint_t)0x88bf43afUL) << 32) | (Dryad_dupelim_fprint_t)0x43565fa7UL, 
+  (((Dryad_dupelim_fprint_t)0xbebb7241UL) << 32) | (Dryad_dupelim_fprint_t)0x360adb47UL, 
+  (((Dryad_dupelim_fprint_t)0xd399e12dUL) << 32) | (Dryad_dupelim_fprint_t)0xea25d131UL, 
+  (((Dryad_dupelim_fprint_t)0xd03a3d3cUL) << 32) | (Dryad_dupelim_fprint_t)0x20aa87f4UL, 
+  (((Dryad_dupelim_fprint_t)0x8111202dUL) << 32) | (Dryad_dupelim_fprint_t)0x77c4b0a8UL, 
+  (((Dryad_dupelim_fprint_t)0xc62d960fUL) << 32) | (Dryad_dupelim_fprint_t)0xccc5ba7fUL, 
+  (((Dryad_dupelim_fprint_t)0x9d94edd9UL) << 32) | (Dryad_dupelim_fprint_t)0xe31c0833UL, 
+  (((Dryad_dupelim_fprint_t)0xa926bc80UL) << 32) | (Dryad_dupelim_fprint_t)0x10d838e0UL, 
+  (((Dryad_dupelim_fprint_t)0xf3c8b809UL) << 32) | (Dryad_dupelim_fprint_t)0x89f6395aUL, 
+  (((Dryad_dupelim_fprint_t)0x99824e83UL) << 32) | (Dryad_dupelim_fprint_t)0xb5562fbaUL, 
+  (((Dryad_dupelim_fprint_t)0xd87d11f3UL) << 32) | (Dryad_dupelim_fprint_t)0xa1ae7f31UL, 
+  (((Dryad_dupelim_fprint_t)0xadb9b99eUL) << 32) | (Dryad_dupelim_fprint_t)0x5d44d4eaUL, 
+  (((Dryad_dupelim_fprint_t)0xaef654bbUL) << 32) | (Dryad_dupelim_fprint_t)0x644fe26aUL, 
+  (((Dryad_dupelim_fprint_t)0xcbf16d7aUL) << 32) | (Dryad_dupelim_fprint_t)0xc4a259e8UL, 
+  (((Dryad_dupelim_fprint_t)0x8a1a38ceUL) << 32) | (Dryad_dupelim_fprint_t)0x068a8e79UL, 
+  (((Dryad_dupelim_fprint_t)0xfc5207dcUL) << 32) | (Dryad_dupelim_fprint_t)0x711c0a9fUL, 
+  (((Dryad_dupelim_fprint_t)0xd30ddb1bUL) << 32) | (Dryad_dupelim_fprint_t)0xa0f02884UL, 
+  (((Dryad_dupelim_fprint_t)0xd48fc688UL) << 32) | (Dryad_dupelim_fprint_t)0x376f2998UL, 
+  (((Dryad_dupelim_fprint_t)0xa79f0024UL) << 32) | (Dryad_dupelim_fprint_t)0xe168fb6eUL, 
+  (((Dryad_dupelim_fprint_t)0x80709fe6UL) << 32) | (Dryad_dupelim_fprint_t)0xa7dd8d6fUL, 
+  (((Dryad_dupelim_fprint_t)0xc8771453UL) << 32) | (Dryad_dupelim_fprint_t)0xabb9e8e3UL, 
+  (((Dryad_dupelim_fprint_t)0xc9e8268eUL) << 32) | (Dryad_dupelim_fprint_t)0xfb9fd8a3UL, 
+  (((Dryad_dupelim_fprint_t)0xc994dbf7UL) << 32) | (Dryad_dupelim_fprint_t)0xc566278eUL, 
+  (((Dryad_dupelim_fprint_t)0xddd80109UL) << 32) | (Dryad_dupelim_fprint_t)0xc37bd67bUL, 
+  (((Dryad_dupelim_fprint_t)0xa9cc5534UL) << 32) | (Dryad_dupelim_fprint_t)0x8f13c673UL, 
+  (((Dryad_dupelim_fprint_t)0xa36d7a45UL) << 32) | (Dryad_dupelim_fprint_t)0xd27bc907UL, 
+  (((Dryad_dupelim_fprint_t)0xd7e2a78cUL) << 32) | (Dryad_dupelim_fprint_t)0x66663257UL, 
+  (((Dryad_dupelim_fprint_t)0xdd426ee6UL) << 32) | (Dryad_dupelim_fprint_t)0x7c908039UL, 
+  (((Dryad_dupelim_fprint_t)0xc80996c7UL) << 32) | (Dryad_dupelim_fprint_t)0x916f5fc8UL, 
+  (((Dryad_dupelim_fprint_t)0xf9a6c515UL) << 32) | (Dryad_dupelim_fprint_t)0x3d62dc96UL, 
+  (((Dryad_dupelim_fprint_t)0x8267aaa0UL) << 32) | (Dryad_dupelim_fprint_t)0xc80b20a6UL, 
+  (((Dryad_dupelim_fprint_t)0xdeb59e2dUL) << 32) | (Dryad_dupelim_fprint_t)0xb3e430a8UL, 
+  (((Dryad_dupelim_fprint_t)0xa03fa280UL) << 32) | (Dryad_dupelim_fprint_t)0x2d0318a9UL, 
+  (((Dryad_dupelim_fprint_t)0x83b7afb5UL) << 32) | (Dryad_dupelim_fprint_t)0xc47e0dfcUL, 
+  (((Dryad_dupelim_fprint_t)0x8752b710UL) << 32) | (Dryad_dupelim_fprint_t)0xe740bfa9UL, 
+  (((Dryad_dupelim_fprint_t)0xa6ee843cUL) << 32) | (Dryad_dupelim_fprint_t)0x1df1006eUL, 
+  (((Dryad_dupelim_fprint_t)0x814705bfUL) << 32) | (Dryad_dupelim_fprint_t)0x21a7a80eUL, 
+  (((Dryad_dupelim_fprint_t)0xf3feedbaUL) << 32) | (Dryad_dupelim_fprint_t)0x611a554dUL, 
+  (((Dryad_dupelim_fprint_t)0xdbe78addUL) << 32) | (Dryad_dupelim_fprint_t)0xf2daa748UL, 
+  (((Dryad_dupelim_fprint_t)0x961e7a41UL) << 32) | (Dryad_dupelim_fprint_t)0x615851ccUL, 
+  (((Dryad_dupelim_fprint_t)0xdb85afd5UL) << 32) | (Dryad_dupelim_fprint_t)0x496a1c1dUL, 
+  (((Dryad_dupelim_fprint_t)0xbadd6e78UL) << 32) | (Dryad_dupelim_fprint_t)0x2e2ba8ceUL, 
+  (((Dryad_dupelim_fprint_t)0xaf93ef6dUL) << 32) | (Dryad_dupelim_fprint_t)0x2abed356UL, 
+  (((Dryad_dupelim_fprint_t)0xc645141aUL) << 32) | (Dryad_dupelim_fprint_t)0xd5794d6cUL, 
+  (((Dryad_dupelim_fprint_t)0xd86e9600UL) << 32) | (Dryad_dupelim_fprint_t)0x582cb555UL, 
+  (((Dryad_dupelim_fprint_t)0xc39d12b4UL) << 32) | (Dryad_dupelim_fprint_t)0x25fe98a3UL, 
+  (((Dryad_dupelim_fprint_t)0x8c346762UL) << 32) | (Dryad_dupelim_fprint_t)0x9a5f7296UL, 
+  (((Dryad_dupelim_fprint_t)0x9f373e3cUL) << 32) | (Dryad_dupelim_fprint_t)0x90100d71UL, 
+  (((Dryad_dupelim_fprint_t)0xb00c9e7bUL) << 32) | (Dryad_dupelim_fprint_t)0x68d20287UL, 
+  (((Dryad_dupelim_fprint_t)0x9f6f838bUL) << 32) | (Dryad_dupelim_fprint_t)0x293b2e4aUL, 
+  (((Dryad_dupelim_fprint_t)0xcbd55e6bUL) << 32) | (Dryad_dupelim_fprint_t)0xb5990fdcUL, 
+  (((Dryad_dupelim_fprint_t)0xc9ca494cUL) << 32) | (Dryad_dupelim_fprint_t)0x50fcc7c8UL, 
+  (((Dryad_dupelim_fprint_t)0xe7e36ad9UL) << 32) | (Dryad_dupelim_fprint_t)0x68b357d0UL, 
+  (((Dryad_dupelim_fprint_t)0x88f27f83UL) << 32) | (Dryad_dupelim_fprint_t)0xc0204576UL, 
+  (((Dryad_dupelim_fprint_t)0x9b17ad6fUL) << 32) | (Dryad_dupelim_fprint_t)0x4c8a74b2UL, 
+  (((Dryad_dupelim_fprint_t)0xe0cfbf08UL) << 32) | (Dryad_dupelim_fprint_t)0x5660db1cUL, 
+  (((Dryad_dupelim_fprint_t)0x982f1507UL) << 32) | (Dryad_dupelim_fprint_t)0x9f214ce0UL
+}; 
+  
+const UInt64 cbPolys8 =(sizeof(polys8) / sizeof(polys8[0]));
+const UInt64 cbPolys16 =(sizeof(polys16) / sizeof(polys16[0]));
+const UInt64 cbPolys32 =(sizeof(polys32) / sizeof(polys32[0]));
+const UInt64 cbPolys64 = sizeof (polys64) / sizeof (polys64[0]);
+

DryadVertex/VertexHost/system/classlib/include/DrFunctions.h

@@ -0,0 +1,119 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+#define _ATL_CSTRING_EXPLICIT_CONSTRUCTORS      // some CString constructors will be explicit
+
+
+#include <math.h>
+#include <strsafe.h>
+
+DrError DrStringToSignedOrUnsignedInt64(const char *psz, UInt64 *pResult, bool fSigned);
+DrError DrStringToFloat(const char *psz, float *pResult);
+DrError DrStringToInt64(const char *psz, Int64 *pResult);
+DrError DrStringToInt32(const char *psz, Int32 *pResult);
+DrError DrStringToUInt64(const char *psz, UInt64 *pResult);
+DrError DrStringToUInt32(const char *psz, UInt32 *pResult);
+DrError DrStringToUInt16(const char *psz, UInt16 *pResult);
+DrError DrStringToPortNumber(const char *psz, DrPortNumber *pResult);
+DrError DrStringToUInt(const char *psz, unsigned int *pResult);
+DrError DrStringToInt(const char *psz, int *pResult);
+DrError DrStringToDouble(const char *psz, double *pResult);
+DrError DrStringToBool(const char *psz, bool *pResult);
+
+// Convert string to size
+// String may contain KB, MB, GB, TB, PB at the end, i.e. 12KB. It's case-insensitive.
+// If suffix is present then fractions are allowed , i.e. 20.5MB
+DrError DrStringToSize(PCSTR psz, UInt64* result);
+DrError DrStringToIntegerSize(PCSTR psz, Int64* result); //parses negative sizes as well
+DrError DrStringToSizeEx(PCSTR psz, UInt64* result, bool allowNegative);
+
+/* Returns a timestamp representing the current date and time (UTC time). Note that this time may change
+   suddenly due to system clock updates and may even move backwards. */
+DrTimeStamp DrGetCurrentTimeStamp();
+
+// Returns the elapsed time between two Dryad timestamps. May be negative.
+inline DrTimeInterval DrGetElapsedTime(DrTimeStamp tStart, DrTimeStamp tEnd)
+{
+    return (DrTimeInterval)(tEnd - tStart);
+}
+
+// Reads an environment variable
+// the returned value must be freed with free()
+// Note that the varname and returned value are both UTF-8 encoded
+DrError DrGetEnvironmentVariable(const char *pszVarName, /* out */ const char **ppszValue);
+
+//
+// Get the environment variable using WCHARs
+//
+DrError DrGetEnvironmentVariable(const WCHAR *pszVarName, WCHAR ppszValue[]);
+
+// 
+// Get the SID for a user
+//
+DrError DrGetSidForUser(LPCWSTR domainUserName, PSID* ppSid); 
+
+//
+// Get the computer name whether it's on-premises or in Azure
+//
+DrError DrGetComputerName(WCHAR ppszValue[]);
+
+__inline DWORD DrGetTimerMsFromInterval(DrTimeInterval t)
+{
+    if (t == DrTimeInterval_Infinite) {
+        return INFINITE;
+    }
+    LogAssert (t >= DrTimeInterval_Zero && t < (DrTimeInterval_Millisecond * 0x7fffffff));
+
+    // round up since timers are a minimum time
+    return (DWORD)((t + DrTimeInterval_Millisecond - DrTimeInterval_Quantum) / DrTimeInterval_Millisecond);
+}
+
+// Same as ExitProcess, but flushes logging and stdout/stderr first...
+void DrExitProcess(UInt32 exitCode);
+
+// Converts a Win32 SYSTEMTIME structure, either in UTC or local time, to a Dryad timestamp
+DrError DrSystemTimeToTimeStamp(const SYSTEMTIME *pSystemTime, /* out */ DrTimeStamp *pTimeStamp, bool fFromLocalTimeZone = false);
+
+// Generates a string to append to a timestamp string to identify a local time zone ("form "Z" or "L+7h")
+DrError DrGenerateTimeZoneBiasSuffix(DrTimeInterval bias, char *szBuff, size_t nbBuff);
+
+// Converts a time interval string to a DrTimeInterval.
+// If len is -1, it is computed with strlen.
+// Strings must include units; e.g., "105.42s" or "12d5h10m".
+DrError DrStringToTimeInterval(const char *pszString, DrTimeInterval *pTimeInterval, int len = -1);
+
+// Converts a Dryad timestamp to a human-readable string, either in UTC  (if bias is 0) or local time (according to bias). If bias is given the
+// special value DrTimeInterval_Infinite, the default local bias is used.
+// nFracDig Is the number of fractional second digits to include. If -1, either 0 or 3 digits will be included depending on whether the time represents an integral second.
+DrError DrTimeStampToString(DrTimeStamp timeStamp, char *pBuffer, int buffLen, DrTimeInterval bias = DrTimeInterval_Zero, Int32 nFracDig=-1);
+DrError DrTimeStampToString(DrTimeStamp timeStamp, char *pBuffer, int buffLen, bool fToLocalTimeZone, Int32 nFracDig=-1);
+
+static const size_t k_DrTimeIntervalStringBufferSize = 64; // this size guarrantees successful completion of DrTimeIntervalToString
+
+// Converts a cosmos timeinterval to a human-readable string.
+// The generated string may be fed back into DrStringToTimeInterval
+DrError DrTimeIntervalToString(DrTimeInterval timeInterval, char *pBuffer, size_t buffLen);
+
+DrError DrStringToTimeStamp(const char *pszTime, DrTimeStamp *pTimeStampOut, bool fDefaultLocalTimeZone = true);
+DrError DrStringToTimeStamp(const char *pszTime, DrTimeStamp *pTimeStampOut, DrTimeInterval defaultTimeZoneBias);
+
+// Converts a Dryad timestamp to a Win32 SYSTEMTIME structure, either in UTC or local time
+DrError DrTimeStampToSystemTime(DrTimeStamp timeStamp, /* out */ SYSTEMTIME *pSystemTime, bool fToLocalTimeZone = false);

DryadVertex/VertexHost/system/classlib/include/DrGuid.h

@@ -0,0 +1,155 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#include <objbase.h>
+#include "DrHash.h"
+
+#pragma once
+
+class DrStr;
+//JC class DrWStr;
+
+extern const GUID g_DrInvalidGuid;
+extern const GUID g_DrNullGuid;
+
+#define DrInvalidGuid ((const DrGuid &)g_DrInvalidGuid)
+#define DrNullGuid ((const DrGuid &)g_DrNullGuid)
+
+// A class that encapsulates guids used within Dryad
+//
+// NOTE: This class has no constructor, so that it can be used in unions. Use
+// DrInitializedGuid if you want the constructor version.
+//
+// This class must add no new member state since we cast from arbitrary GUIDs to this class.
+class DrGuid : public GUID
+{
+public:
+    DrGuid& Set(const GUID& other)
+    {
+        *(GUID *)this = other;
+        return *this;
+    }
+
+    DrGuid& operator=(const GUID& other)
+    {
+        return Set(other);
+    }
+
+    bool operator==(const GUID& other) const
+    {
+        return memcmp(this, &other, sizeof(DrGuid)) == 0;
+    }
+
+    bool operator!=(const GUID& other) const
+    {
+        return memcmp(this, &other, sizeof(DrGuid)) != 0;
+    }
+
+    bool operator>(const DrGuid& other) const
+    {
+        const unsigned int *pCur = (const unsigned int*) this;
+        const unsigned int *pOther = (const unsigned int*) &other;
+
+        for(int i = 0; i < sizeof(DrGuid)/sizeof(int); i++) {
+            if(pCur[i] > pOther[i])
+                return true;
+            else if(pCur[i] < pOther[i])
+                return false;
+        }
+
+        return false;  // They are equal
+    }
+
+    // Return a 32-bit hash for this guid
+    DWORD Hash() const
+    {
+        return DrHash32::Guid( this );
+    }
+
+    // Store the invalid guid as our guid
+    void Invalidate()
+    {
+        Set( g_DrInvalidGuid );
+    }
+
+    // Returns whether this is a valid guid
+    bool IsValid() const
+    {
+        return (*this != g_DrInvalidGuid) && !IsNull();
+    }
+
+    // Store the null guid as our guid
+    void SetToNull()
+    {
+        Set( g_DrNullGuid );
+    }
+
+    // Returns whether this is a valid guid
+    bool IsNull() const
+    {
+        return *this == g_DrNullGuid;
+    }
+
+    // Generate a guid
+    void Generate();
+
+    // Parse a guid from a string. Acceptes guids either with or without braces
+    BOOL Parse(const char *string);
+
+    // Parses guid from a string, return NULL on failure or pointer to next char after GUID in given string on success
+    // allowBraces - string may contain optional braces 
+    // requireBraces - braces are required (allowBraces should be true)
+    // requireEOL - string should contain nothing after GUID (i.e. zero terminator should be present right after GUID)
+    const char* Parse(const char *string, bool allowBraces, bool requireBraces, bool requireEOL);
+    
+    const static size_t GuidStringLength = 39; //38 + null terminator
+
+    // Output the guid in string form; {EFF6744C-7143-11cf-A51B-080036F12502}. If fBraces
+    // is false, the braces are omitted.
+    // String must be able to hold DrGuid::GuidStringLength (39 characters = 38 + null terminator)
+    char *ToString(char *string, bool fBraces=true) const;
+
+    // appends the guid in string form; {EFF6744C-7143-11cf-A51B-080036F12502}. If fBraces
+    // is false, the braces are omitted.
+    DrStr& AppendToString(DrStr& strOut, bool fBraces=true) const;
+
+    // appends the guid in string form; {EFF6744C-7143-11cf-A51B-080036F12502}. If fBraces
+    // is false, the braces are omitted.
+//JC    DrWStr& AppendToString(DrWStr& strOut, bool fBraces=true) const;
+
+protected:
+    // Helper function to convert a byte into 2 hex digits
+    static void ByteToHex(BYTE input, char *output);
+};
+
+class DrInitializedGuid : public DrGuid
+{
+public:
+    DrInitializedGuid()
+    {
+        Set(g_DrInvalidGuid);
+    }
+
+    DrInitializedGuid(const GUID& other)
+    {
+        Set(other);
+    }
+};
+

DryadVertex/VertexHost/system/classlib/include/DrHash.h

@@ -0,0 +1,231 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//
+// Fast, thorough hash functions returning 32 or 64 bit results.
+//
+// You can use hash table sizes that are a power of 2, and use & to 
+// trim values, for example:
+//   LOGSIZE=10;
+//   Buckets *hash_table[(1<<LOGSIZE)];
+//   bucket = hash_table[ hash(key) & ((1<<LOGSIZE)-1) ];
+// It doesn't matter which bits of the hash you use, they are all well
+// mixed.  It doesn't matter what patterns there are in your keys, the
+// hash does equally well on all key formats.
+//
+// Usually the function you want here is DrHash32::Compute().
+//
+
+
+#pragma once
+
+#include "basic_types.h"
+
+#pragma pack (push, 8)
+
+//JCnamespace apsdk
+//JC{
+
+class DrHash32
+{
+private:
+    DrHash32 ();                               // no NEW allowed 
+    DrHash32 ( const DrHash32& );              // no copy allowed
+    DrHash32& operator= ( const DrHash32& );   // no assignment allowed
+
+public:
+ 
+    //
+    // Mix(): cause every bit of a,b,c to affect 32 bits of a,b,c
+    // both forwards and in reverse.  Same for pairs of bits in a,b,c.
+    // This can be used along with Final() to hash a fixed number of
+    // 4-byte integers, for example see the implementation of Guid().
+    //
+    inline static void Mix( UInt32& a, UInt32& b, UInt32& c)
+    {
+        a -= c;  a ^= _rotl(c, 4);  c += b;
+        b -= a;  b ^= _rotl(a, 6);  a += c;
+        c -= b;  c ^= _rotl(b, 8);  b += a;
+        a -= c;  a ^= _rotl(c,16);  c += b;
+        b -= a;  b ^= _rotl(a,19);  a += c;
+        c -= b;  c ^= _rotl(b, 4);  b += a;
+    }
+
+    //
+    // Final: cause every bit of a,b,c to affect every bit of c, only forward.
+    // Same for pairs of bits in a,b,c.  It also causes b to be an OK hash.
+    // This is a good way to hash 1 or 2 or 3 integers:
+    //   a = k1; b = k2; c = 0;
+    //   DrHash32::Final(a,b,c);
+    // Use c (and maybe b) as the hash value
+    //
+    inline static void Final( UInt32& a, UInt32& b, UInt32& c)
+    {
+        c ^= b; c -= _rotl(b,14);
+        a ^= c; a -= _rotl(c,11);
+        b ^= a; b -= _rotl(a,25);
+        c ^= b; c -= _rotl(b,16);
+        a ^= c; a -= _rotl(c,4);
+        b ^= a; b -= _rotl(a,14);
+        c ^= b; c -= _rotl(b,24);
+    }
+
+    //
+    // Compute2: Compute two hash values for a byte array of known length
+    //
+    static void Compute2 (
+        const void *pData,    // byte array of known length
+        Size_t      uSize,    // size of pData
+        UInt32      uSeed1,   // first seed
+        UInt32      uSeed2,   // second seed
+        UInt32     *uHash1,   // OUT: first hash value (may not be null)
+        UInt32     *uHash2);  // OUT: second hash value (may not be null)
+
+    //
+    // Compute: Compute a hash values for a byte array of known length
+    //
+    static const UInt32 Compute (
+        const void *pData,      // byte array of known length
+        Size_t      uSize,      // size of pData
+        UInt32      uSeed = 0)  // seed for hash function
+    {
+        UInt32 uHash2 = 0;
+        DrHash32::Compute2(pData, uSize, uSeed, uHash2, &uSeed, &uHash2);
+        return uSeed;
+    }
+
+    //
+    // String: hash of string of unknown length
+    //
+    static const UInt32 String (
+        const char *pString,     // ASCII string to hash case-insensitive
+        UInt32      uSeed = 0)   // optional seed for hash
+    {
+        UInt32      uHash2 = 0;
+        Size_t      uSize;
+
+        uSize = strlen(pString);
+        DrHash32::Compute2(pString, uSize, uSeed, uHash2, &uSeed, &uHash2);
+        return uSeed;
+    }
+
+    //
+    // StringI2: Produce two case-insensitive 32-bit hashes of an ASCII string
+    // The results are identical to Compute2() on an uppercased string
+    //
+    static void StringI2 (
+        const char *pString,   // ASCII string to hash case-insensitive
+        Size_t      uSize,     // length of string (required)
+        UInt32      uSeed1,    // first seed
+        UInt32      uSeed2,    // second seed
+        UInt32     *uHash1,    // OUT: first hash
+        UInt32     *uHash2);   // OUT: second hash
+
+    //
+    // StringI: case insensitive hash of string of unknown length
+    //
+    static const UInt32 StringI (
+        const char *pString,     // ASCII string to hash case-insensitive
+        size_t      len = (size_t)-1,
+        UInt32      uSeed = 0)   // optional seed for hash
+    {
+        UInt32      uHash2 = 0;
+        Size_t      uSize;
+
+        uSize = (len == (size_t)-1) ? strlen(pString) : len;
+        DrHash32::StringI2(pString, uSize, uSeed, uHash2, &uSeed, &uHash2);
+        return uSeed;
+    }
+
+    //
+    // Find a 32-bit hash of a GUID
+    //
+    static const UInt32 Guid ( const GUID *pGuid )
+    {
+        UInt32 a = ((UInt32 *)pGuid)[0];
+        UInt32 b = ((UInt32 *)pGuid)[1];
+        UInt32 c = ((UInt32 *)pGuid)[2];
+        DrHash32::Mix(a,b,c);
+        a ^= ((UInt32 *)pGuid)[3];
+        DrHash32::Final(a,b,c);
+        return c;
+    }
+};
+
+
+
+
+class DrHash64
+{
+private:
+    DrHash64 ();                               // no NEW allowed 
+    DrHash64 ( const DrHash64& );              // no copy allowed
+    DrHash64& operator= ( const DrHash64& );   // no assignment allowed
+
+public:
+
+
+    //
+    // Compute hash for a byte array of known length.
+    //
+    static const UInt64 Compute ( 
+        const void *pData,         // byte array to hash
+        Size_t      uSize,          // length of pData
+        UInt64      uSeed = 0 )     // seed to hash function; 0 is an OK value
+    {
+        UInt32 uHash1, uHash2;
+        DrHash32::Compute2( pData, uSize, (UInt32) uSeed, (UInt32) (uSeed >> 32), &uHash1, &uHash2);
+        return uHash1 | (((UInt64)uHash2) << 32);
+    }
+
+    //
+    // case-insensitive hash of null terminated string
+    // produce the same hash as Compute on an uppercased string
+    //
+    static const UInt64 StringI (
+        const char *pString,
+        Size_t      uSize,
+        UInt64      uSeed)
+    {
+        UInt32 uHash1, uHash2;
+        DrHash32::StringI2( pString, uSize, (UInt32) uSeed, (UInt32) (uSeed >> 32), &uHash1, &uHash2);
+        return uHash1 | (((UInt64)uHash2) << 32);
+    }
+
+    // Compute a 64-bit hash of a GUID
+    static const UInt64 Guid ( const GUID *pGuid )
+    {
+        UInt32 a = ((UInt32 *)pGuid)[0];
+        UInt32 b = ((UInt32 *)pGuid)[1];
+        UInt32 c = ((UInt32 *)pGuid)[2];
+        DrHash32::Mix(a,b,c);
+        a ^= ((UInt32 *)pGuid)[3];
+        DrHash32::Final(a,b,c);
+        return c | (((UInt64)b) << 32);
+    }
+};
+
+#pragma pack (pop)
+
+//JC} // namespace apsdk
+
+#ifdef USING_APSDK_NAMESPACE
+using namespace apsdk;
+#endif

DryadVertex/VertexHost/system/classlib/include/DrHeap.h

@@ -0,0 +1,104 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#ifndef __DRYADHEAP_H__
+#define __DRYADHEAP_H__
+
+
+// Heap grows by 2*oldsize + c_heapGrowAmount
+static const int c_heapGrowAmount = 100;
+
+class DryadHeapItem
+{
+public:
+    // This is where the item is in the index
+    DWORD   m_heapIndex;
+
+    // Returns TRUE if we are supposed to be dequeued before "other"
+    virtual bool IsHigherPriorityThan(DryadHeapItem *other) = NULL;
+};
+
+// Implements a binary heap priority queue
+class DryadHeap
+{
+public:
+    DryadHeap()
+    {
+        m_entries = NULL;
+        m_numEntries = 0;
+        m_heapAllocSize = 0;
+    }
+
+    // Initialize heap with the initial count of elements
+    void    Initialize(int initialCount);
+
+    // Dequeue the heap root; returns NULL if empty
+    DryadHeapItem* DequeueHeapRoot();
+
+    // Peek at the heap root without dequeueing; returns NULL if empty
+    DryadHeapItem* PeekHeapRoot();
+
+    // Insert a new entry into the heap
+    void    InsertHeapEntry(DryadHeapItem *entry);
+
+    // Remove the heap item at the given index
+    void    RemoveHeapEntry(DWORD index);
+
+protected:
+    void    DownHeapify(DWORD index);
+    void    UpHeapify(DWORD index);
+    void    Heapify(DWORD index);
+
+    // Increases the size of the heap, when it needs more memory
+    void    GrowHeap();
+
+    inline void HeapSwap(DWORD index1, DWORD index2);
+
+    // Return whether an index exists
+    // Indices are 1-based, so 1 is the root and m_numEntries is the last entry
+    // e.g. ParentOf(1) will return FALSE
+    bool Exists(DWORD index)
+    {
+        return (index > 0 && index <= m_numEntries);
+    }
+
+    // Note, ParentOf(root) will return root, so your code may need to check for this
+    static DWORD ParentOf(DWORD index)
+        {   return ((index) >> 1);  };
+
+    static DWORD LeftChild(DWORD index) 
+        {   return ((index) << 1);  };
+
+    static DWORD RightChild(DWORD index) 
+        {   return (((index) << 1)+1);  };
+
+
+public:
+    DryadHeapItem**  m_entries;
+
+    // Current # entries
+    DWORD   m_numEntries;
+
+    // # entries allocated
+    size_t  m_heapAllocSize;
+};
+
+
+#endif  //end if not defined __DRYADHEAP_H__

DryadVertex/VertexHost/system/classlib/include/DrList.h

@@ -0,0 +1,606 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#ifndef __DRLIST_H__
+#define __DRLIST_H__
+
+#pragma once
+
+// Simple Growable vector of arbitrary assignable typed values
+// Insertions and deletions can be performed both at the head and at the tail of the list, making it suitable for queues.
+// The underlying type must either be a simple scalar, a struct/class that is clonable with memcpy, or a class that implements copy constructor and assignment operator
+// TODO: This version constructs all objects on reallocation, and does not destruct unused items until the list is destroyed.
+// TODO: That is OK for simple types, but can waste space for types that allocate additional storage.
+// TODO: to do it correctly, fix this class to use in-place contructor/destructor; construct object when it is added to list, destruct when it is removed.
+template<typename t> class DrValList sealed
+{
+public:    
+    DrValList(UInt32 uFirstAllocSize = 20)
+    {
+        m_nEntries = 0;
+        m_uFirstEntry = 0;
+        m_nAllocated = 0;
+        m_prgEntries = NULL;
+        m_uFirstAllocSize = uFirstAllocSize;
+    }
+
+    ~DrValList()
+    {
+        if (m_prgEntries != NULL) {
+            delete[] m_prgEntries;
+            m_prgEntries = NULL;
+        }
+        m_nEntries = 0;
+        m_nAllocated = 0;
+    }
+
+    // forces the buffer to be reallocated with the given size, even if it
+    // is the same as the current size.
+    // the requested size must be big enough to hold the valid entries.
+    // On exit, the valid entries are always contiguous starting at offset 0
+    // if n==0, frees the buffer
+    void ForceRealloc(UInt32 n)
+    {
+        LogAssert(n >= m_nEntries);
+        t *pnew = NULL;
+        if (n != 0) {
+            pnew = new t[n];
+            LogAssert(pnew != NULL);
+            UInt32 uFrom = m_uFirstEntry;
+            for (UInt32 i = 0; i < m_nEntries; i++) {
+                pnew[i]=m_prgEntries[uFrom++];
+                if (uFrom >= m_nAllocated) {
+                    uFrom = 0;
+                }
+            }
+        }
+        if (m_prgEntries != NULL) {
+            delete[] m_prgEntries;
+        }
+        m_prgEntries = pnew;
+        m_nAllocated = n;
+        m_uFirstEntry = 0;
+    }
+
+    // reallocates the buffer if there are not at least n elements allocated    
+    void GrowTo(UInt32 n)
+    {
+        if (n > m_nAllocated) {
+            if (n < 2 * m_nAllocated) {
+                n = 2 * m_nAllocated;
+            }
+            if (n < m_uFirstAllocSize) {
+                n = m_uFirstAllocSize;
+            }
+            ForceRealloc(n);
+        }
+    }
+
+    // Converts a potentially wrapped list (if you have moved the head) into a
+    // contiguous list, and returns a pointer to the first item in the contiguous list
+    // If possible, nothing is moved. If the list is wrapped, a new buffer is allocated (easier than moving everything in a full list)
+    // This operation is always cheap if you never remove from or add to the head.
+    t *MakeContiguous()
+    {
+        if (m_uFirstEntry + m_nEntries > m_nAllocated) {
+            ForceRealloc(m_nEntries);
+        }
+
+        return m_prgEntries + m_uFirstEntry;
+    }
+
+    UInt32 NumEntries() const
+    {
+        return m_nEntries;
+    }
+
+    bool IsEmpty()
+    {
+        return m_nEntries == 0;
+    }
+
+    t& EntryAt(UInt32 index)
+    {
+        LogAssert(index < m_nEntries);
+        return m_prgEntries[NormalizeEntryIndex(index)];
+    }
+
+    const t& EntryAt(UInt32 index) const
+    {
+        LogAssert(index < m_nEntries);
+        return m_prgEntries[NormalizeEntryIndex(index)];
+    }
+
+    t& operator[](UInt32 index)
+    {
+        return EntryAt(index);
+    }
+
+    const t& operator[](UInt32 index) const
+    {
+        return EntryAt(index);
+    }
+
+    t& Head()
+    {
+        LogAssert(m_nEntries != 0);
+        return m_prgEntries[m_uFirstEntry];
+    }
+
+    const t& Head() const
+    {
+        LogAssert(m_nEntries != 0);
+        return m_prgEntries[m_uFirstEntry];
+    }
+
+    t& Tail()
+    {
+        LogAssert(m_nEntries != 0);
+        return EntryAt(m_nEntries-1);
+    }
+
+    const t& Tail() const
+    {
+        LogAssert(m_nEntries != 0);
+        return EntryAt(m_nEntries-1);
+    }
+
+    // LIFO-style top of stack
+    t& TopOfStack()
+    {
+        return Tail();
+    }
+
+    const t& TopOfStack() const
+    {
+        return Tail();
+    }
+
+
+    // Invalidates all entry references previously returned
+    // This is the typical method used to implement Enqueue for FIFO queues, or Push for stacks
+    t& AddEntryToTail(const t& val)
+    {
+        GrowTo(m_nEntries+1);
+        t& newEntry = m_prgEntries[NormalizeEntryIndex(m_nEntries++)];
+        newEntry = val;
+        return newEntry;
+    }
+
+    // FIFO-style queueing
+    t& Enqueue(const t& val)
+    {
+        return AddEntryToTail(val);
+    }
+
+    // LIFO-style stack push
+    t& Push(const t& val)
+    {
+        return AddEntryToTail(val);
+    }
+
+    // Invalidates all entry references previously returned
+    t& AddEntryToHead(const t& val)
+    {
+        GrowTo(m_nEntries+1);
+        if (m_uFirstEntry == 0) {
+            m_uFirstEntry = m_nAllocated - 1;
+        } else {
+            m_uFirstEntry--;
+        }
+        m_nEntries++;
+        t& newEntry = m_prgEntries[m_uFirstEntry];
+        newEntry = val;
+        return newEntry;
+    }
+
+    // Invalidates all entry references previously returned
+    t& AddEntry(const t& val)
+    {
+        return AddEntryToTail(val);
+    }
+
+    // This is the typical method used to emplement Pop for stacks
+    t& RemoveEntryFromTail(__out t *pValOut)
+    {
+        LogAssert(m_nEntries != 0);
+        *pValOut = m_prgEntries[NormalizeEntryIndex(--m_nEntries)];
+    }
+
+    t RemoveEntryFromTail()
+    {
+        LogAssert(m_nEntries != 0);
+        // NOTE: following code depends on not destructing the returned value until after we return.
+        // if we add in-place destructors, this has to change
+        const t& retVal = m_prgEntries[NormalizeEntryIndex(--m_nEntries)];
+        return retVal;
+    }
+
+    // LIFO-style stack pop
+    t Pop()
+    {
+        return RemoveEntryFromTail();
+    }
+
+    t& Pop(__out t *pValOut)
+    {
+        return RemoveEntryFromTail(pValOut);
+    }
+
+    // This is the typical method used to emplement Pop for stacks
+    t& RemoveEntryFromHead(__out t *pValOut)
+    {
+        LogAssert(m_nEntries != 0);
+        *pValOut = m_prgEntries[m_uFirstEntry++];
+        if (m_uFirstEntry >= m_nAllocated) {
+            m_uFirstEntry = 0;
+        }
+        m_nEntries--;
+        return *pValOut;
+    }
+
+    t RemoveEntryFromHead()
+    {
+        LogAssert(m_nEntries != 0);
+        // NOTE: following code depends on not destructing the returned value until after we return.
+        // if we add in-place destructors, this has to change
+        const t& retVal = m_prgEntries[m_uFirstEntry++];
+        if (m_uFirstEntry >= m_nAllocated) {
+            m_uFirstEntry = 0;
+        }
+        m_nEntries--;
+        return retVal;
+    }
+
+    // FIFO-style dequeueing
+    t Dequeue()
+    {
+        return RemoveEntryFromHead();
+    }
+
+    t& Dequeue(__out t *pValOut)
+    {
+        return RemoveEntryFromHead(pValOut);
+    }
+
+
+    // does not shrink the allocated list or destruct existing entries. To do that, use ForceRealloc(0).
+    void Clear()
+    {
+        m_nEntries = 0;
+        m_uFirstEntry = 0;
+    }
+
+
+    // returns NULL if not in the list.
+    t *FindVal(const t& val)
+    {
+        for (UInt32 i = 0; i < m_nEntries; i++) {
+            t& entry = EntryAt(i);
+            if (entry == val) {
+                return &entry;
+            }
+        }
+        return NULL;
+    }
+    
+    // returns NULL if not in the list.
+    const t *FindVal(const t& val) const
+    {
+        for (UInt32 i = 0; i < m_nEntries; i++) {
+            const t& entry = EntryAt(i);
+            if (entry == val) {
+                return &entry;
+            }
+        }
+        return NULL;
+    }
+
+    bool ContainsVal(const t& val) const
+    {
+        return FindVal(val) != NULL;
+    }
+    
+    static int __cdecl InternalDrValListEntryPointerCompare(void *context, const void *p1, const void *p2)
+    {
+        if (**( const t**)p1 > **(const t**)p2) {
+            return 1;
+        } else if (**( const t**)p1 ==**(const t**)p2) {
+            return 0;
+        }
+        return -1;
+    }
+    
+    // performs a quicksort on the list.
+    // To sort, the base type must suport the ">" and "==" operators
+    // As a side-effect, truncates the allocated size to the actual size.
+    void Sort()
+    {
+        // We cannot sort directly with C's quicksort, since it uses memmove.
+        // So we will sort a pointer list, and then reallocate
+        if (m_nEntries != 0)
+        {
+            const t **rgpEntries = new const t *[m_nEntries];
+            LogAssert(rgpEntries != NULL);
+            for (UInt32 i = 0; i < m_nEntries; i++) {
+                rgpEntries[i] = &(EntryAt(i));
+            }
+            qsort(rgpEntries, m_nEntries, sizeof(rgpEntries[0]), InternalDrValListEntryPointerCompare);
+            // Generally, noone ever sorts when they plan to grow the list, so we can truncate to actual size.
+            t *pNew = new t[m_nEntries];
+            LogAssert(pNew != NULL);
+            for (UInt32 i = 0; i < m_nEntries; i++)
+            {
+                pNew[i] = *(rgpEntries[i]);
+            }
+            delete[] m_prgEntries;
+            m_prgEntries = pNew;
+            m_nAllocated = m_nEntries;
+            m_uFirstEntry = 0;
+        }
+        else
+        {
+            ForceRealloc(0);
+        }
+    }
+
+protected:
+    UInt32 NormalizeEntryIndex(UInt32 index) const
+    {
+        LogAssert(index < m_nAllocated);
+        if (m_uFirstEntry != 0) {
+            index += m_uFirstEntry;
+            if (index >= m_nAllocated) {
+                index -= m_nAllocated;
+            }
+        }
+        return index;
+    }
+
+private:
+    UInt32 m_uFirstEntry;             // Normally 0, the index of the first entry (for circular buffers)
+    
+    UInt32 m_uFirstAllocSize;
+    UInt32 m_nEntries;
+    UInt32 m_nAllocated;
+    t *m_prgEntries;
+};
+
+
+
+// a List template that only grows :)
+// You are recommended to use DrPtrList unless you are aware of the potential complicated issues behind it
+// when copying/assigning values. . :)
+
+template <typename T>
+class DrList{
+private:
+    //basically not supported
+    void Merge(DrList &other){
+        for(unsigned int i = 0; i <other.Size(); i ++){
+            Push(other[i]);
+        }
+    }
+
+public:
+    DrList(unsigned int numAlloc = 10){
+        Init(numAlloc);
+    }
+    DrList(DrList &other){
+        Init(other.Size());
+        Merge(other);
+    }
+    
+    
+    ~DrList(){
+        FreeElements();     // special processing for freeing each element
+        FreeMem();
+    }
+
+    bool IsEmpty() const {
+        return m_numEntries == 0;
+    }
+
+    unsigned int Size() const {
+        return m_numEntries;
+    }
+
+    void Push(const T &entry){
+        AddEntry(entry);
+    }
+
+    /*
+    T& Pop() const{
+        LogAssert(Size());
+        m_numEntries --;
+        return m_p[m_numEntries];
+    }
+
+    T& Last() const{
+        LogAssert(Size());
+        return m_p[m_numEntries - 1];
+    }
+
+    T& First() const{
+        LogAssert(Size());
+        return m_p[0];
+    }
+    */
+
+    T& GetEntry(unsigned int i) const {
+        LogAssert(i < m_numEntries);
+
+        return m_p[i];
+    }
+
+    T& operator[](unsigned int i) const{
+        return GetEntry(i);
+    }
+
+    /* sample compare function
+        int __cdecl compare(const void *p1, const void *p2){
+            const T &v1 = *(T *)p1;
+            const T &v2 = *(T *)p2;
+
+            return v1 - v2;
+        }
+      */
+      
+    void Sort(int (__cdecl *compare )(const void *, const void *)){
+
+        qsort(m_p, m_numEntries, sizeof(T), compare);
+    }
+    
+    void Clear(){
+        if(m_numAllocated == m_initNumAllocated && IsEmpty()) return;
+
+        unsigned int numAlloc = m_initNumAllocated;
+        FreeElements();
+        FreeMem();
+        Init(numAlloc);
+    }
+
+    // don't call me !!! This function violates some design principle, will be removed shortly
+    void MakeAllAllocatedElementsValid(){
+        m_numEntries = m_numAllocated;
+    }
+    
+protected:
+
+    unsigned int m_numEntries;
+    unsigned int m_numAllocated;
+    unsigned int m_initNumAllocated;
+
+    T           *m_p;
+
+    bool IsFull() const {
+        return m_numEntries == m_numAllocated;
+    }
+
+    void Grow(){
+        m_numAllocated *= 2;
+
+        T  *newp = new T[m_numAllocated];
+
+        LogAssert(newp);
+
+        for(unsigned int i = 0; i < m_numEntries; i ++){
+            newp[i] = m_p[i];
+        }
+
+        FreeMem();      
+
+        m_p = newp;
+    }
+
+    void Init(unsigned int numAlloc){
+        m_numEntries = 0;
+        m_initNumAllocated = m_numAllocated = numAlloc;
+        m_p = new T[numAlloc];
+        LogAssert(m_p);
+    }
+
+    void FreeMem(){
+        if(m_p){
+            delete[] m_p;
+            m_p = 0;
+        }
+    }
+
+    virtual void FreeElements(){
+        /* do nothing */
+    }
+
+    void AddEntry(const T &entry){
+        if(IsFull()){
+            Grow();
+        }
+        m_p[m_numEntries] = entry;
+        m_numEntries ++;
+    }
+
+    unsigned int NumAllocated() const {
+        return m_numAllocated;
+    }
+
+};
+
+template <typename T>
+class DrInternalPtrList : public DrList<T *>{
+public:
+    DrInternalPtrList(unsigned int numAlloc) : DrList<T *>(numAlloc) {}
+    // allow pointer expressions to be passed as argument
+    void Push(T *ptr){
+        DrList<T *>::Push(ptr);          
+    }
+};
+
+// managed means the pointers give to the list are owned by the list and therefore they will be freed by the list.
+
+template <typename T>
+class DrPtrList : public DrInternalPtrList<T>{
+public:
+    DrPtrList(unsigned int numAlloc = 10) : DrInternalPtrList<T>(numAlloc) {}
+};
+
+template <typename T>
+class DrUnmanagedPtrList : public DrInternalPtrList<T>{
+public:
+    DrUnmanagedPtrList(unsigned int numAlloc = 10) : DrInternalPtrList<T>(numAlloc) {}
+};
+
+template <typename T>
+class DrUnmanagedArrList : public DrInternalPtrList<T>{
+public:
+    DrUnmanagedArrList(unsigned int numAlloc = 10) : DrInternalPtrList<T>(numAlloc) {}
+};
+
+template <typename T>
+class DrManagedPtrList : public DrInternalPtrList<T>{
+public:
+    DrManagedPtrList(unsigned int numAlloc = 10) : DrInternalPtrList<T>(numAlloc) {}
+protected:
+    virtual void FreeElements(){
+        if(m_p){
+            for(unsigned int i = 0; i < Size(); i ++){
+                delete GetEntry(i);
+            }
+        }
+    }
+};
+
+template <typename T>
+class DrManagedArrList: public DrInternalPtrList<T>{
+public:
+    DrManagedArrList(unsigned int numAlloc = 10) : DrInternalPtrList<T>(numAlloc) {}
+protected:
+    virtual void FreeElements(){
+        if(m_p){
+            for(unsigned int i = 0; i < Size(); i ++){
+                delete[] GetEntry(i);
+            }
+        }
+    }
+};
+
+typedef DrManagedArrList<const char> DrManagedStrList;
+typedef DrUnmanagedPtrList<const char> DrUnmanagedStrList;
+
+#endif

DryadVertex/VertexHost/system/classlib/include/DrLogging.h

@@ -0,0 +1,105 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include <assert.h>
+#include <stdio.h>
+#include <share.h>
+
+#define DrLogWithType(_x) DrLogHelper(_x,__FILE__,__FUNCTION__,__LINE__)
+
+#define DRMAKELOGTYPE(_type,_initial) \
+    
+
+//
+// Create logging calls at various levels and allow them to complete iff logging is enabled
+// at the selected level
+//
+#define DrLogD if (DrLogging::Enabled(LogLevel_Debug)) DrLogWithType(LogLevel_Debug)
+#define DrLogI if (DrLogging::Enabled(LogLevel_Info)) DrLogWithType(LogLevel_Info)
+#define DrLogW if (DrLogging::Enabled(LogLevel_Warning)) DrLogWithType(LogLevel_Warning)
+#define DrLogE if (DrLogging::Enabled(LogLevel_Error)) DrLogWithType(LogLevel_Error)
+#define DrLogA if (DrLogging::Enabled(LogLevel_Assert)) DrLogWithType(LogLevel_Assert)
+
+//
+// Define the logging levels
+//
+typedef enum 
+{
+    LogLevel_Off = 0,
+    LogLevel_Assert = 1,
+    LogLevel_Error = 3,
+    LogLevel_Warning = 7,
+    LogLevel_Info = 15,
+    LogLevel_Debug = 31
+} LogLevel;
+
+//
+// Expose functions to set the logging level, check the logging level, and flush the log
+//
+class DrLogging
+{   
+public:
+    static void SetLoggingLevel(LogLevel type);
+    static bool Enabled(LogLevel type);
+    static void FlushLog();
+    static FILE* GetLogFile();
+
+private:   
+    static FILE* CreateLogFile();
+    static FILE* m_logFile;
+};
+
+//
+// Class that defines the logging context for a particular log call
+//
+class DrLogHelper
+{
+public:
+    DrLogHelper(LogLevel type, const char* file, const char* function, int line)
+    {
+        m_type = type;
+        m_file = file;
+        m_function = function;
+        m_line = line;
+    }
+
+    void operator()(const char* format, ...);
+
+private:
+    LogLevel    m_type;
+    const char*  m_file;
+    const char*  m_function;
+    int          m_line;
+};
+
+//
+// Define a helper that logs at the assert level if any conditional fails 
+//
+#define LogAssert(exp, ...) \
+    do { \
+        if (!(exp)) { \
+            printf("Assert -- %s, %d: %s\n", __FILE__, __LINE__, #exp); \
+            DrLogA(#exp, __VA_ARGS__ ); \
+        } \
+    } while (0)
+
+#define DebugLogAssert(exp, ...)	LogAssert(#exp, __VA_ARGS__ )

DryadVertex/VertexHost/system/classlib/include/DrMemory.h

@@ -0,0 +1,362 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+class DrMemoryBuffer: public DrRefCounter
+{
+private:
+    Size_t m_uAvailableSize;
+    
+protected:
+    Size_t m_uAllocatedSize;
+    bool m_fIsGrowable;
+
+    // these two are anded together to determine writability    
+    bool m_fIsWritable;
+    bool m_fIsUserWritable;
+
+    void Init()
+    {
+        m_uAllocatedSize   = 0;
+        m_uAvailableSize   = 0;
+        m_fIsGrowable = true;
+        m_fIsWritable = true;
+        m_fIsUserWritable = true;
+    };
+
+    DrMemoryBuffer()
+    {
+        Init();
+    };
+
+    virtual ~DrMemoryBuffer()
+    {
+    };
+
+    virtual void InternalSetAvailableSize(Size_t uSize)
+    {
+        m_uAvailableSize = uSize;
+    }
+
+
+/* --------------------- Functions that the actual implementation provides --------------------- */
+
+public:
+
+    //
+    // Retrieve pointer to the data stored in memory block at uOffset and max size available in this block
+    //
+    // Returns NULL if no data at this offset, valid pointer otherwise
+    //
+    virtual void *GetDataAddress(
+        Size_t uOffset,            // starting offset
+        Size_t *puSize,             // number of bytes available (0 in case of failure)
+        Size_t *puPriorSize         // optional; size of contigious memory area prior to (*GetDataAddress())
+    ) = 0;
+
+    //
+    // Preallocate enough memory buffers to fix uMaxSize bytes of data.
+    //
+    // If the buffer is not growable, it is a fatal error to attempt to grow the allocated size
+    // beyond the current value.
+    virtual void IncreaseAllocatedSize(
+        Size_t uSize              // preallocate memory blocks to fit at least uSize bytes of data
+    ) = 0;
+
+    //
+    // Updates the available size. Grows the allocated size if necessary.
+    //
+    // The default implementation just does the required updating, then updates m_uAvailableSize
+    //
+    virtual void SetAvailableSize(Size_t uSize)
+    {
+        if (uSize > m_uAllocatedSize) {
+            IncreaseAllocatedSize(uSize);
+            LogAssert(uSize <= m_uAllocatedSize);
+        }
+        InternalSetAvailableSize(uSize);
+    }
+
+
+/* ------------------------------- The rest are predefined ----------------------------------------- */
+
+
+public:
+
+    bool IsWritable()
+    {
+        return m_fIsWritable && m_fIsUserWritable;
+    }
+
+    bool IsGrowable()
+    {
+        return m_fIsGrowable;
+    }
+
+    //
+    // Get max size of data that may be stored in the buffer
+    //
+    Size_t GetAllocatedSize()
+    {
+        return m_uAllocatedSize;
+    };
+
+    //
+    // Get amount of data already stored in the buffer
+    //
+    Size_t GetAvailableSize()
+    {
+        return m_uAvailableSize;
+    };
+
+    //
+    // Get address and size of available memory chunk allocating more memory if necessary
+    //
+    // Note that this may return memory beyond the current available size; it is
+    // the caller's responsibility to SetAvailableSize() if necessary.
+    //
+    // Note also that the returned *puSize may be greater than uDataSize
+    //
+    // If the buffer is not growable, it is a fatal error to ask for data beyond the allocated size.
+    void *GetWriteAddress(
+        Size_t uOffset,    // offset at which to return a write pointer
+        Size_t uDataSize, // minimum number of bytes to ensure is available (not necessarily contiguous) starting at the specified offset
+        /* out */ Size_t *puSize,                                // Number of contiguous bytes starting at the returned pointer
+        /* out */ Size_t *puPreceedingSize = NULL   // Number of contiguous bytes that preceed the returned pointer
+    );
+
+    //
+    // This is the same as GetWriteAddress, except it is not a fatal error if the buffer is not growable grown to accomodate; in
+    // this case, NULL is returned.
+    void *GetWriteAddressIfPossible(
+        Size_t uOffset,    // offset at which to return a write pointer
+        Size_t uDataSize, // minimum number of bytes to ensure is available (not necessarily contiguous) starting at the specified offset
+        /* out */ Size_t *puSize,                                // Number of contiguous bytes starting at the returned pointer
+        /* out */ Size_t *puPreceedingSize = NULL   // Number of contiguous bytes that preceed the returned pointer
+        )
+    {
+        Size_t uEnd = uOffset + uDataSize;
+        if (!IsGrowable() && uEnd > m_uAllocatedSize) {
+            *puSize = 0;
+            if (puPreceedingSize != NULL) {
+                *puPreceedingSize = 0;
+            }
+            return NULL;
+        }
+        return GetWriteAddress(uOffset, uDataSize, puSize, puPreceedingSize);
+    }
+
+    //
+    // Copy uDataSize bytes from pData array into the buffer at the
+    // specified offset.  Grows the available data size if necessary to include the written data.
+    //
+    void Write(
+        Size_t uOffset,         // starting offset
+        const void *pData,    // data buffer
+        Size_t uDataSize      // number of bytes to copy
+    );
+
+    void Append(
+        const void *pData,
+        Size_t uDataSize
+    )
+    {
+        Write(GetAvailableSize(), pData, uDataSize);
+    }
+
+    //
+    // Zero uDataSize bytes in the buffer at the
+    // specified offset.  Grows the available data size if necessary to include the zeroed data.
+    //
+    void Zero(
+        Size_t uOffset,            // starting offset
+        Size_t uDataSize           // number of bytes to set to 0
+    );
+
+    //
+    // Get the address and size of contiguous available readable memory area at offset uOffset
+    // It is a fatal error to request readable memory at or beyond the current available buffer size.
+    //
+    const void *GetReadAddress(
+        Size_t uOffset,
+        /* out */ Size_t *puSize,                   // Number of contiguous available bytes beginning at uOffset
+        /* out */ Size_t *puPreceedingSize = NULL   // Number of contiguous readable bytes that preceed the returned pointer
+    );
+
+    //
+    // Read uDataSize bytes into pData into the buffer starting at uOffset.
+    //
+    // It is a fatal error to attempt to read beyond the available size of the buffer
+    //
+    void Read(
+        Size_t uOffset,
+        void  *pData,
+        Size_t uDataSize
+    );
+
+    //
+    // Compares uDataSize bytes from pData with buffer contents starting at uOffset.
+    //
+    // Return 0 on match, < 0 if contents of the buffer is less than contents of pData, > 0 otherwise
+    //
+    // It is a fatal error to attempt to read beyond the available size of the buffer
+    //
+    int Compare(
+        Size_t      uOffset,
+        const void *pData,
+        Size_t      uDataSize
+    );
+
+    //
+    // Copy data from one buffer to another
+    //
+    void CopyBuffer(
+        Size_t uDstOffset,     // starting offset
+        DrMemoryBuffer *pSrcBuffer,     // source buffer
+        Size_t uSrcOffset,     // starting offset in source buffer
+        Size_t uDataSize       // number of bytes to copy
+    );
+    
+};
+
+
+// A simple implementation of DrMemoryBuffer that is built on a single malloc'd block of memory
+class DrSimpleHeapBuffer : public DrMemoryBuffer
+{
+private:
+    BYTE *m_pData;
+
+public:
+    DrSimpleHeapBuffer();
+    DrSimpleHeapBuffer(Size_t allocedSize); // pregrows allocated size
+    virtual ~DrSimpleHeapBuffer();
+
+    // Returns the underlying heap object (if any). This buffer remains
+    // the owner of the heap object.
+    //
+    // returns NULL if there is no underlying heap object (allocedSize = 0)
+    //
+    void *GetHeapItem()
+    {
+        return m_pData;
+    }
+
+    // Detaches the underlying heap object (if any) and returns it to the caller, who
+    // must call free() on the memory when done with it.
+    //
+    // returns NULL if there is no underlying heap object (allocedSize = 0)
+    //
+    // After this call, the buffer is a new buffer with no data in it.
+    //
+    void *DetachHeapItem();
+
+    // Attaches an external heap item to the buffer. Any previous heap item is
+    // freed. The buffer becomes the owner of the heap item.
+    //
+    void AttachHeapItem(void *pHeapItem, Size_t allocedSize, Size_t dataSize);
+
+    // DrMemoryBuffer implementation:
+    
+public:
+
+    //
+    // Retrieve pointer to the data stored in memory block at uOffset and max size available in this block
+    //
+    // Returns NULL if no data at this offset, valid pointer otherwise
+    //
+    virtual void *GetDataAddress(
+        Size_t uOffset,            // starting offset
+        Size_t *puSize,             // number of bytes available (0 in case of failure)
+        Size_t *puPriorSize         // optional; size of contigious memory area prior to (*GetDataAddress())
+    );
+
+    //
+    // Preallocate enough memory buffers to fix uMaxSize bytes of data.
+    //
+    virtual void IncreaseAllocatedSize(
+        Size_t uSize              // preallocate memory blocks to fit at least uSize bytes of data
+    );
+};
+
+// A buffer that wraps a fixed contiguous block of memory. 
+//
+// The buffer allocated size is not growable. No memory is freed when the buffer is destroyed
+//
+class DrFixedMemoryBuffer : public DrMemoryBuffer
+{
+private:
+    const BYTE *m_pData;
+
+public:
+    DrFixedMemoryBuffer()
+    {
+        m_pData = NULL;
+        m_fIsGrowable = false;
+        m_fIsWritable = false;
+    }
+
+    virtual ~DrFixedMemoryBuffer()
+    {
+    }
+    
+    //
+    // Initialize fixed length buffer with byte array and sizes
+    //
+    void Init(const BYTE *pData, Size_t allocatedSize, Size_t availableSize = 0)
+    {
+        LogAssert(availableSize <= allocatedSize);
+        m_pData = pData;
+        m_uAllocatedSize = allocatedSize;
+        InternalSetAvailableSize(availableSize);
+        m_fIsWritable = true;
+    }
+
+    //
+    // Create new fixed length buffer with byte array and sizes
+    //
+    DrFixedMemoryBuffer(const BYTE *pData, Size_t allocatedSize, Size_t availableSize = 0)
+    {
+        Init(pData, allocatedSize, availableSize);
+    }
+
+    // DrMemoryBuffer implementation:
+    
+public:
+
+    //
+    // Retrieve pointer to the data stored in memory block at uOffset and max size available in this block
+    //
+    // Returns NULL if no data at this offset, valid pointer otherwise
+    //
+    virtual void *GetDataAddress(
+        Size_t uOffset,            // starting offset
+        Size_t *puSize,             // number of bytes available (0 in case of failure)
+        Size_t *puPriorSize         // optional; size of contigious memory area prior to (*GetDataAddress())
+    );
+
+    //
+    // Preallocate enough memory buffers to fix uMaxSize bytes of data.
+    //
+    virtual void IncreaseAllocatedSize(
+        Size_t uSize              // preallocate memory blocks to fit at least uSize bytes of data
+    );
+};
+

DryadVertex/VertexHost/system/classlib/include/DrNodeAddress.h

@@ -0,0 +1,897 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#ifndef __DRYADNODEADDRESS_H__
+#define __DRYADNODEADDRESS_H__
+
+#pragma once
+
+#pragma prefast(push)
+#pragma prefast(disable:24002, "struct sockaddr not ipv6 compatible")
+
+class DrMemoryReader;
+class DrMemoryWriter;
+class DrLastAccessTable;
+
+extern DrLastAccessTable *g_pDrLastAccessTable;
+
+void DrInitLastAccessTable();
+
+/*
+ * Dryad Node Address
+ *
+ * Used to represent the address of a cosmos node (EN, DRM, etc.) on the network
+ */
+
+
+static const int k_MaxHostNameLength = 255;
+static const int k_MaxIpAddressesPerHostName = 16;
+
+class DrNodeAddress
+{
+public:
+
+    //Default c'tor. Nulls out both the internet address and port
+    inline DrNodeAddress();
+
+    inline DrNodeAddress( const char *pszName, DrPortNumber defaultPort )
+    {
+        Set( pszName, defaultPort );
+    }
+
+    //Copy c'tor.
+    inline DrNodeAddress(const DrNodeAddress& addr);
+
+    inline DrNodeAddress& operator=(const DrNodeAddress& addr);
+
+    //Create address with specified internet address and port
+    inline DrNodeAddress(const IN_ADDR& ina, DrPortNumber wPort);
+
+    // sets using an IP address in network byte order
+    inline void Set(const IN_ADDR& ina, DrPortNumber wPort);
+
+    // returns false if not a valid ip/node address
+    bool Set(const struct sockaddr *pSockAddr, Size_t addrLen)
+    {
+        return Set((const struct sockaddr_in *)(const void *)pSockAddr, addrLen);
+    }
+
+    // returns false if not a valid ip/node address
+    bool Set(const struct sockaddr_in *pSockAddr, Size_t addrLen = sizeof(struct sockaddr_in));
+
+    // sets using an IP address in host byte order
+    inline void Set(DrIpAddress ipAddr, DrPortNumber wPort);
+
+    inline void Clear()
+    {
+        Set(DrInvalidIpAddress, DrInvalidPortNumber);
+    }
+
+    inline bool operator==(const DrNodeAddress& other) const
+    {
+        return (m_ina.S_un.S_addr == other.m_ina.S_un.S_addr && m_wPort == other.m_wPort);
+    }
+
+    inline bool operator!=(const DrNodeAddress& other) const
+    {
+        return (m_ina.S_un.S_addr != other.m_ina.S_un.S_addr || m_wPort != other.m_wPort);
+    }
+
+    inline void SetLocal(DrPortNumber wPort)
+    {
+        Set(DrLocalIpAddress, wPort);
+    }
+
+    void GetSockAddr(struct sockaddr_in *pAddr, Size_t len = sizeof(struct sockaddr_in)) const
+    {
+        LogAssert (len >= sizeof(struct sockaddr_in));
+        memset(pAddr, 0, len);
+        pAddr->sin_family = AF_INET;
+        pAddr->sin_port = htons(m_wPort);
+        pAddr->sin_addr = m_ina;
+    }
+        
+    
+    // Gets the IP address in host byte order
+    inline DrIpAddress GetIpAddress() const
+    {
+        return ntohl(m_ina.S_un.S_addr);
+    }
+
+    // Gets the IP address in network byte order
+    inline UInt32 GetIpAddressNetworkOrder() const
+    {
+        return m_ina.S_un.S_addr;
+    }
+
+    // Gets the IP address in network byte order
+    const IN_ADDR& GetInAddr() const
+    {
+        return m_ina;
+    }
+    
+    // Gets the port number in host byte order
+    inline DrPortNumber GetPort() const
+    {
+        return m_wPort;
+    }
+
+    // Sets the port number from a host byte ordered value
+    inline void SetPort(DrPortNumber port)
+    {
+        m_wPort = port;
+    }
+
+    inline bool IsNull() const
+    {
+        return m_ina.S_un.S_addr == 0 && m_wPort == 0;
+    }
+
+    inline void SetToNull()
+    {
+        m_ina.S_un.S_addr=0;
+        m_wPort=0;
+    }
+    
+    // Looks up a host name using DNS
+    // Note that this is a blocking request
+    // Returns up to addressBuffLen entries. If there are more entries than this, the list is truncated without error.
+    // Returns DrError_HostNotFound if no hosts match the name.
+    static DrError LookupHostName(
+        const char *pszHostName,
+        /* out */ DrIpAddress *pAddressBuff,
+        UInt32 addressBuffLen,
+        /* out */ UInt32 *pNumReturnedAddresses);
+    
+    // Parses a name in the form "#.#.#.#:port" or "dns-name:port" and splits out the host name and port.
+    // If ":port" is missing, uses the default port.
+    // Returns DrError_InvalidParameter if the string is malformed.
+    static DrError ParseHostPortName(/* out */ char *pHostNameBuffer, Size_t buffLen, /*out */ DrPortNumber *pPort, const char *pszName, DrPortNumber defaultPort = DrAnyPortNumber, UInt32 *pInstanceNumOut = NULL);
+
+    // Parses a name in the form "#.#.#.#:port" or "dns-name:port" and splits out the host name and port.
+    // If ":port" is missing, uses the default port.
+    // strOut is replaced with the parsed host name
+    // Returns DrError_InvalidParameter if the string is malformed.
+    static DrError ParseHostPortName(DrStr& strOut, /*out */ DrPortNumber *pPort, const char *pszName, DrPortNumber defaultPort = DrAnyPortNumber, UInt32 *pInstanceNumOut = NULL);
+
+    // Parses a stringified IP address in the form "#.#.#.#" into a host-order IP address.
+    // Returns DrError_InvalidParameter if the string is malformed.
+    static DrError ParseIpAddress(const char *pszIpAddress, /* out */ DrIpAddress *pIpAddr);
+
+    // Parses a name in the form "#.#.#.#:port" or "dns-name:port" and resolves it to an address.
+    // If ":port" is missing, uses the default port.
+    // If there is more than one address associated with a DNS name, uses the first one.
+    // Returns DrError_InvalidParameter if the string is malformed.
+    // Note that this method may block for DNS resolution if a DNS name is used.
+    DrError Set(const char *pszName, DrPortNumber defaultPort=DrAnyPortNumber);
+
+    // Converts the contained IP/port address to a string of the form "#.#.#.#:port". If the contained port number matches defaultPort, the
+    // port number is not included in the string.
+    // buffSize must be at least 22 or DrError_StringTooLong is returned.
+    DrError ToAddressPortString(char *pBuffer, Size_t buffSize, DrPortNumber defaultPort = DrAnyPortNumber) const;
+    DrError ToAddressPortString(WCHAR *pBuffer, Size_t buffSize, DrPortNumber defaultPort = DrAnyPortNumber) const;
+    DrStr& AppendToString(DrStr& strOut, DrPortNumber defaultPort = DrAnyPortNumber) const;
+    //JC DrWStr& AppendToString(DrWStr& strOut, DrPortNumber defaultPort = DrAnyPortNumber) const;
+
+    // Generates a 32-bit hash of the node address
+    inline UInt32 Hash() const
+    {
+        return (UInt32)GetIpAddress() + (UInt32)GetPort();
+    }
+    
+private:
+    IN_ADDR m_ina;               // IP address in network byte order
+    DrPortNumber m_wPort;   // Port number in host byte order
+
+};    
+
+class DrNodeAddressString : public DrStr32
+{
+public:
+    DrNodeAddressString(const DrNodeAddress& addr)
+    {
+        addr.AppendToString(*this, DrAnyPortNumber);
+    }
+};
+
+// This macro can be used to obtain a temporary "const char *" string equivalent for a node address. It can be used
+// as the parameter to a method call; the pointer will become invalid after the function returns
+#define DRNODEADDRESSSTRING(addr)    (DrNodeAddressString(addr).GetString())
+
+
+// DrNodeAddressList is a list of node addresses, as you might get from resolving a name. It is more efficient when there is only one name.
+class DrNodeAddressList
+{
+public:
+    DrNodeAddressList()
+    {
+        m_pMultipleAddresses = &m_singleAddress;
+        m_numAllocated = 1;
+        m_numEntries = 0;
+    }
+
+    ~DrNodeAddressList()
+    {
+        Clear();
+    }
+
+    // discards all the entries, but doesn't free memory
+    void Discard()
+    {
+        m_numEntries = 0;
+    }
+
+    // Discards all entries and frees allocated memory.
+    void Clear()
+    {
+        if (m_pMultipleAddresses != &m_singleAddress) {
+            delete[] m_pMultipleAddresses;
+            m_pMultipleAddresses = &m_singleAddress;
+            m_numAllocated = 1;
+        }
+        m_numEntries = 0;
+    }
+
+    void GrowTo(UInt32 numAllocated)
+    {
+        if (numAllocated > m_numAllocated) {
+            UInt32 nNew = 2 * m_numAllocated;
+            if (nNew < numAllocated) {
+                nNew = numAllocated;
+            }
+            if (nNew < 8) {
+                nNew = 8;
+            }
+            DrNodeAddress *pNew = new DrNodeAddress[nNew];
+            if (m_numEntries > 0) {
+                memcpy(pNew, m_pMultipleAddresses, m_numEntries * sizeof(DrNodeAddress));
+            }
+            if (m_pMultipleAddresses != &m_singleAddress) {
+                delete[] m_pMultipleAddresses;
+            }
+            m_pMultipleAddresses = pNew;
+            m_numAllocated = nNew;
+        }
+    }
+    
+    DrNodeAddress *AddEntry(const DrNodeAddress *pOther = NULL)
+    {
+        GrowTo(m_numEntries+1);
+        DrNodeAddress *pEntry = m_pMultipleAddresses + m_numEntries;
+        m_numEntries++;
+        
+        if (pOther != NULL) {
+            (*pEntry) = (*pOther);
+        } else {
+            pEntry->Clear();
+        }
+
+        return pEntry;
+    }
+
+    DrNodeAddress& operator[](UInt32 index)
+    {
+        LogAssert(index < m_numEntries);
+        return m_pMultipleAddresses[index];
+    }
+
+    const DrNodeAddress& operator[](UInt32 index) const
+    {
+        LogAssert(index < m_numEntries);
+        return m_pMultipleAddresses[index];
+    }
+
+    UInt32 GetLength() const
+    {
+        return m_numEntries;
+    }
+
+    // This call may block for DNS
+    // It resolves the specified host name (with optional ":port") to a list of IP addresses and *appends* those to this DrNodeAddressList, filling in
+    // the port number for each.
+    // Note that since this request appends to the existing list, you must Clear() or Discard() the list before you make this
+    // call if you want the results to replace the existing set.
+    // Returns DrError_HostNotFound if no hosts match the name.
+    DrError ResolveHostName(const char *pszHostName, DrPortNumber defaultPort=DrInvalidPortNumber);
+    
+private:
+    DrNodeAddress m_singleAddress;
+    DrNodeAddress *m_pMultipleAddresses; // Points to m_singleAddress if there is 1 entry; otherwise, a heap array.
+    UInt32 m_numAllocated;
+    UInt32 m_numEntries;
+};
+
+
+// After the first failure to send to a host, don't send to it again until Now + this interval
+const DrTimeInterval k_initialDelayedSendTimeInterval = DrTimeInterval_Second * 10;
+
+// Don't let the delayed send interval grow beyond this
+const DrTimeInterval k_maxDelayedSendInterval = DrTimeInterval_Second * 60; 
+
+// XStream-specific conversions between fabric fault domain and autopilot pod names
+//JC DrError DrPodNameToFaultDomain(__in PCSTR pszPodName, __out XsFaultDomain *pFaultDomainOut);
+
+// The return value is an internalized string of the for "pod%u";
+//JC PCSTR DrFaultDomainToPodName( __in XsFaultDomain faultDomain);
+
+    
+
+// Manages a host name and a port number
+// Also keeps track of a fault domain and an upgrade domain (for use in load balancing/load optimization)
+class DrHostAndPort
+{
+public:
+    DrHostAndPort()
+    {
+        m_pszHostName = NULL;
+        m_portNumber = DrInvalidPortNumber;
+        m_pszPodName = NULL;
+//JC        m_upgradeDomain = 0;
+//JC        m_faultDomain = 0;
+//JC        m_fValidFaultDomain = true;
+        m_fValidPod = true;
+    }
+
+    ~DrHostAndPort()
+    {
+    }
+
+    DrHostAndPort& Set(const DrHostAndPort& other)
+    {
+        m_pszHostName = other.m_pszHostName;
+        m_portNumber = other.m_portNumber;
+        m_pszPodName = other.m_pszPodName;
+//JC        m_upgradeDomain = other.m_upgradeDomain;
+//JC        m_faultDomain = other.m_faultDomain;
+//JC        m_fValidFaultDomain = other.m_fValidFaultDomain;
+        m_fValidPod = other.m_fValidPod;
+        return *this;
+    }
+
+    DrHostAndPort(const DrHostAndPort& other)
+    {
+        Set(other);
+    }
+
+    DrHostAndPort& operator=(const DrHostAndPort& other)
+    {
+        return Set(other);
+    }
+
+    // Note, doesn't compare pod and upgrade domain
+    bool operator==(const DrHostAndPort &other) const{
+        // only need to compare hostname addresses since they are internalized
+        return (m_portNumber == other.m_portNumber)  && 
+                   (m_pszHostName == other.m_pszHostName);  
+    }
+
+/*JC
+// Note: this replaces the fault domain with the one encoded in the pod name
+    void Set(const char *pszHostName, DrPortNumber port, const char *pszPodName, XsUpgradeDomain upgradeDomain)
+    {
+        m_pszHostName = g_DrInternalizedStrings.InternalizeStringLowerCase(pszHostName);
+        m_portNumber = port;
+        m_pszPodName = g_DrInternalizedStrings.InternalizeStringLowerCase(pszPodName);
+        m_fValidPod = true;
+//JC        ReplaceFaultDomainFromPod();
+//JC        m_upgradeDomain = upgradeDomain;
+    }
+
+    // Note: this replaces the fault domain with the one encoded in the pod name
+    void SetPodName(const char *pszPodName)
+    {
+        m_pszPodName = g_DrInternalizedStrings.InternalizeStringLowerCase(pszPodName);
+        m_fValidPod = true;
+        ReplaceFaultDomainFromPod();
+    }
+*/
+    // Sets the pod name without explicitly changing the fault domain.
+    void SetPodNameNoFaultDomainUpdate(const char *pszPodName)
+    {
+        m_pszPodName = g_DrInternalizedStrings.InternalizeStringLowerCase(pszPodName);
+        m_fValidPod = true;
+    }
+/*JC
+    // Note: this replaces the pod with "pod%u"
+    void SetFaultDomain(XsFaultDomain faultDomain)
+    {
+        m_faultDomain = faultDomain;
+        m_fValidFaultDomain = true;
+        ReplacePodFromFaultDomain();
+    }
+
+    // Sets the fault domain without explictly changing the pod name.
+    void SetFaultDomainNoPodUpdate(XsFaultDomain faultDomain)
+    {
+        m_faultDomain = faultDomain;
+        m_fValidFaultDomain = true;
+    }
+
+    void SetUpgradeDomain(XsUpgradeDomain upgradeDomain)
+    {
+        m_upgradeDomain = upgradeDomain;
+    }
+
+
+    // Note: this replaces the fault domain with the one encoded in the pod name
+    DrError SetWithDefaultPort(const char *pszHostName, DrPortNumber defaultPort, const char *pszPodName, XsUpgradeDomain upgradeDomain)
+    {
+        DrStr64 strHost;
+        DrError err = DrNodeAddress::ParseHostPortName(strHost, &m_portNumber, pszHostName, defaultPort);
+        if (err == DrError_OK) {
+            m_pszHostName = g_DrInternalizedStrings.InternalizeStringLowerCase(strHost.GetString());
+            m_pszPodName = g_DrInternalizedStrings.InternalizeStringLowerCase(pszPodName);
+            m_fValidPod = true;
+            ReplaceFaultDomainFromPod();
+            m_upgradeDomain = upgradeDomain;
+        }
+        return err;
+    }
+*/
+
+    void SetHostName(const char *pszHostName)
+    {
+        m_pszHostName = g_DrInternalizedStrings.InternalizeStringLowerCase(pszHostName);
+    }
+
+    void SetPort(DrPortNumber port)
+    {
+        m_portNumber = port;
+    }
+
+    const char *GetHostName() const
+    {
+        return m_pszHostName;
+    }
+
+    DrPortNumber GetPort() const
+    {
+        return m_portNumber;
+    }
+
+    // In XStream, the POD name is the fault domain in "pod%u" form
+    const char *GetPodName() const
+    {
+        return m_pszPodName;
+    }
+
+/* JC
+    // returns false if a POD was set that was not of the form "pod%u".
+    bool IsValidFaultDomain() const
+    {
+        return m_fValidFaultDomain;
+    }
+
+    XsFaultDomain GetFaultDomain() const
+    {
+        return m_faultDomain;
+    }
+
+    XsUpgradeDomain GetUpgradeDomain() const
+    {
+        return m_upgradeDomain;
+    }
+*/
+
+    void Clear()
+    {
+        m_pszHostName = NULL;
+        m_portNumber = DrInvalidPortNumber;
+        // TODO: should pod, faultdomain, upgrade domain, etc. be updated?
+    }
+
+    bool IsValid() const
+    {
+        return m_pszHostName != NULL;
+    }
+
+    bool IsInvalid() const
+    {
+        return m_pszHostName == NULL;
+    }
+    
+    // This call may block for DNS
+    // It resolves the host name to a list of IP addresses and *appends* those to the specified DrNodeAddressList, filling in
+    // the port number for each.
+    // Note that since this request appends to the existing list, you must Clear() or Discard() the list before you make this
+    // call if you want the results to replace the existing set.
+    // Returns DrError_HostNotFound if no hosts match the name.
+    DrError ResolveToAddresses(DrNodeAddressList *pAddresses);
+
+    DrError Unserialize(DrMemoryReader *pReader);
+    DrError Serialize(DrMemoryWriter *pWriter) const;
+    
+    DrStr& AppendToString(DrStr& strOut) const
+    {
+        strOut.AppendF("%s:%u", m_pszHostName, m_portNumber);
+        return strOut;
+    }
+
+    DrStr& ToString(DrStr& strOut) const
+    {
+        strOut.SetToEmptyString();
+        return AppendToString(strOut);
+    }
+
+/* JC
+private:
+    // Updates the fault domain from the pod name. Note that if the pod name is not of the form "pod%u", the fault domain
+    // will be set to 0, m_fValidFaultDomain will be set to false, and and DRError_InvalidParameter will be returned.
+    DrError ReplaceFaultDomainFromPod();
+
+    // Sets the pod name to "pod%u" from the fault domain.
+    void ReplacePodFromFaultDomain()
+    {
+        m_pszPodName = DrFaultDomainToPodName(m_faultDomain);
+        m_fValidPod = true;
+    }
+*/
+
+private:
+    const char * m_pszHostName;  // Internalized
+    DrPortNumber m_portNumber;
+    const char *m_pszPodName; // Internalized. String form of fault domain as "pod%u"
+    //JC XsUpgradeDomain m_upgradeDomain; // default = 0
+    //JC XsFaultDomain m_faultDomain; // Fault domain (default = 0). NUmeric form of pod;
+    //JC bool m_fValidFaultDomain; // Set to false if fault domain was not encountered in Unserialize(). After Unserialize, Always true unless pod is not "pod%u".
+    bool m_fValidPod; // Used only during serialize/Unserialize. Set to false if pod was not encountered in Unserialize(). Always true after Unserialize:
+};
+
+class DrHostAndPortString : public DrStr32
+{
+public:
+    DrHostAndPortString(const DrHostAndPort& host)
+    {
+        host.AppendToString(*this);
+    }
+};
+
+// For each (ip address, port), keeps track of the last time it was accessed
+class DrLastAccessEntry
+{
+public:
+    DrLastAccessEntry()
+    {
+        m_nextHash           = NULL;
+        m_nextAttemptAllowed = DrTimeStamp_LongAgo;
+        m_delayTime          = 0;
+        m_lastError          = DrError_OK;
+    }
+
+    UInt32 Hash()
+    {
+        return m_nodeAddress.Hash();
+    }
+
+public:
+    DrLastAccessEntry*  m_nextHash;
+    DrNodeAddress       m_nodeAddress;
+
+    // If this is not DrTimeStamp_LongAgo, don't send requests until this time
+    DrTimeStamp         m_nextAttemptAllowed;
+
+    // Amount of time to delay next protocol request
+    DrTimeInterval      m_delayTime;
+
+    // Last error code reported
+    DrError             m_lastError;
+};
+
+
+// This table keeps track of when we last accessed a particular IP/port
+// It is used for throttling when we are determining the primary DRM
+// Currently only DRM service descriptors go into this table, which is why it is small
+const UInt32 k_numLastAccessTableBuckets = 100;
+
+class DrLastAccessTable
+{
+public:
+    DrLastAccessTable();
+
+    // Successful send to this node address, reset throttling
+    void UpdateSuccess(const DrNodeAddress& nodeAddress);
+
+    // Send failure.
+    // Returns true if we were already at the maximum allowed delay value
+    bool UpdateFailure(const DrNodeAddress& nodeAddress, DrError error);
+
+    // When can we next send to this node address?
+    // If when is DrTimeStamp_LongAgo, it means there is no delay at all
+    DrError GetDelay(const DrNodeAddress& nodeAddress, DrTimeStamp* when);
+
+private:
+    void Lock()
+    {
+        m_lock.Enter();
+    }
+
+    void Unlock()
+    {
+        m_lock.Leave();
+    }
+
+    // You must have the lock to call these functions
+    DrLastAccessEntry*  FindOrCreate(const DrNodeAddress& nodeAddress);
+    DrLastAccessEntry*  Find(const DrNodeAddress& nodeAddress);
+
+public:
+    DrCriticalSection   m_lock;
+    DrLastAccessEntry*  m_head[k_numLastAccessTableBuckets];
+};
+
+
+#define DRHOSTANDPORTSTRING(host)    (DrHostAndPortString(host).GetString())
+
+// Manages a list of host names (with optional port #) strings
+class DrHostNameList : public DrPropertyParser{
+private:
+    const static UInt32 INVALID_PRIMARY_HOST = 0xFFFFFFFF;
+public:
+    DrHostNameList()
+    {
+        m_pMultipleHosts = &m_singleHost;
+        m_numAllocated = 1;
+        m_numEntries = 0;
+        m_primary = INVALID_PRIMARY_HOST;
+    }
+
+    ~DrHostNameList()
+    {
+        Clear();
+    }
+
+    // Set does a diff with what is already there, so this must be initialized
+    DrHostNameList& Set(const DrHostNameList& other, bool forceReordering = false);
+
+    DrHostNameList(const DrHostNameList& other)
+    {
+        m_pMultipleHosts = &m_singleHost;
+        m_numAllocated = 1;
+        m_numEntries = 0;
+        m_primary = INVALID_PRIMARY_HOST;
+
+        Set(other);
+    }
+
+    DrHostNameList& operator=(const DrHostNameList& other)
+    {
+        // Free previously allocated memory when and only when new name list will not fit
+        // into already allocated region
+        if ((other.m_numEntries > m_numAllocated) && (m_pMultipleHosts != &m_singleHost))
+            Clear();
+
+        return Set(other, true);
+    }
+    
+    void Clear()
+    {
+        if (m_pMultipleHosts != &m_singleHost) {
+            delete[] m_pMultipleHosts;
+            m_pMultipleHosts = &m_singleHost;
+            m_numAllocated = 1;
+        }
+        m_numEntries = 0;
+        m_primary = INVALID_PRIMARY_HOST;
+    }
+
+    void GrowTo(UInt32 numAllocated)
+    {
+        if (numAllocated > m_numAllocated) {
+            UInt32 nNew = 2 * m_numAllocated;
+            if (nNew < numAllocated) {
+                nNew = numAllocated;
+            }
+            if (nNew < 8) {
+                nNew = 8;
+            }
+            DrHostAndPort *pNew = new DrHostAndPort[nNew];
+            for (UInt32 i = 0; i < m_numEntries; i++) {
+                pNew[i] = m_pMultipleHosts[i];
+            }
+            if (m_pMultipleHosts != &m_singleHost) {
+                delete[] m_pMultipleHosts;
+            }
+            m_pMultipleHosts = pNew;
+            m_numAllocated = nNew;
+        }
+    }
+    
+    DrHostAndPort *AddEntry()
+    {
+        GrowTo(m_numEntries+1);
+        DrHostAndPort *pEntry = m_pMultipleHosts + m_numEntries;
+        m_numEntries++;
+        return pEntry;
+    }
+
+    const DrHostAndPort& operator[](UInt32 index) const
+    {
+        LogAssert(index < m_numEntries);
+        return m_pMultipleHosts[index];
+    }
+
+    DrHostAndPort& operator[](UInt32 index)
+    {
+        LogAssert(index < m_numEntries);
+        return m_pMultipleHosts[index];
+    }
+
+    UInt32 GetLength() const
+    {
+        return m_numEntries;
+    }
+
+    // Demote host to bottom of the list
+    // If the host was the primary, then invalidate the primary
+    void DemoteHost(DrHostAndPort &host){
+        DrHostAndPort saveHost;
+        saveHost.Set(host);
+       
+        LogAssert(m_numEntries > 0);
+
+        for(UInt32 i = 0; i < m_numEntries; i ++)
+        {
+            if(host == m_pMultipleHosts[i])
+            {
+                if (m_primary == i)
+                    m_primary = INVALID_PRIMARY_HOST;      // this host is deemed dead
+                
+                if (i < m_numEntries - 1)
+                {
+                    for(; i < m_numEntries - 1; i++)
+                    {
+                        m_pMultipleHosts[i].Set(m_pMultipleHosts[i + 1]);                   
+                    }
+                    
+                    m_pMultipleHosts[m_numEntries - 1].Set(saveHost);
+                }
+
+                break;
+            }
+        }
+    }
+    bool IsPrimaryValid(void) const
+    {
+        return m_primary != INVALID_PRIMARY_HOST;
+    }
+    
+    bool IsPrimaryInvalid(void) const
+    {
+        return m_primary == INVALID_PRIMARY_HOST;
+    }
+    
+    void SetPrimaryInvalid(void)
+    {
+        m_primary = INVALID_PRIMARY_HOST;
+    }
+
+    void SetPrimary(UInt32 primary)
+    {
+        m_primary = primary;
+    }
+
+    UInt32 GetPrimary(void) const{
+        return m_primary;
+    }
+
+    DrError Serialize(DrMemoryWriter *pWriter) const;
+    virtual DrError OnParseProperty(DrMemoryReader *reader, UInt16 property, UInt32 dataLen, void *cookie);
+    
+    // This call may block for DNS
+    // It resolves the list of host names to a list of IP addresses and *appends* those to the specified DrNodeAddressList, filling in
+    // the port number for each.
+    // Note that since this request appends to the existing list, you must Clear() or Discard() the list before you make this
+    // call if you want the results to replace the existing set.
+    // Returns DrError_HostNotFound if no hosts match the name.
+    DrError ResolveToAddresses(DrNodeAddressList *pAddresses);    
+
+    // return one IP/port pair in *pAddresses* and return one host name in *host*
+    DrError ResolveOneHostToAddresses(DrNodeAddressList *pAddressses, bool wantPrimary, DrHostAndPort &host);
+
+    DrStr& AppendToString(DrStr& strOut) const
+    {
+        if (m_numEntries == 0) {
+            strOut.Append("<NoHosts>");
+            
+        } else {
+            m_pMultipleHosts[0].AppendToString(strOut);
+            for (UInt32 i = 1; i < m_numEntries; i++) {
+                strOut.Append(';');
+                m_pMultipleHosts[i].AppendToString(strOut);
+            }
+        }
+        return strOut;
+    }
+
+    DrStr& ToString(DrStr& strOut) const
+    {
+        strOut.SetToEmptyString();
+        return AppendToString(strOut);
+    }
+    
+    
+protected:
+
+    void SelectOneHost(DrHostAndPort &host, bool wantPrimary = true);
+
+private:
+    DrHostAndPort m_singleHost;
+    DrHostAndPort *m_pMultipleHosts; // Points to m_singleHost if there is 1 entry; otherwise, a heap array.  If there is a primary it is always the first entry.
+    UInt32 m_numAllocated;
+    UInt32 m_numEntries;
+    UInt32 m_primary; // This is 0 if there is a primary, and INVALID_PRIMARY_HOST otherwise
+};
+
+class DrHostNameListString : public DrStr64
+{
+public:
+    DrHostNameListString(const DrHostNameList& hostList)
+    {
+        hostList.AppendToString(*this);
+    }
+};
+
+#define DRHOSTNAMELISTSTRING(hostList)    (DrHostNameListString(hostList).GetString())
+
+/*
+ * Inline methods for DrNodeAddress
+ */
+
+inline DrNodeAddress::DrNodeAddress()
+{
+    m_ina.S_un.S_addr=0;
+    m_wPort=0;
+}
+
+inline DrNodeAddress::DrNodeAddress(const DrNodeAddress& addr)
+{
+    m_ina=addr.m_ina;
+    m_wPort=addr.m_wPort;
+}
+
+inline DrNodeAddress& DrNodeAddress::operator=(const DrNodeAddress& addr)
+{
+    m_ina=addr.m_ina;
+    m_wPort=addr.m_wPort;
+    return *this;
+}
+
+inline DrNodeAddress::DrNodeAddress(const IN_ADDR& ina, DrPortNumber wPort)
+{
+    m_ina=ina;
+    m_wPort=wPort;
+}
+
+inline void DrNodeAddress::Set(const IN_ADDR& ina, DrPortNumber wPort)
+{
+    m_ina=ina;
+    m_wPort=wPort;
+}
+
+inline void DrNodeAddress::Set(DrIpAddress ipAddr, DrPortNumber wPort)
+{
+    m_ina.s_addr=htonl(ipAddr);
+    m_wPort=wPort;
+}
+
+#pragma prefast(pop)
+
+#endif
+

DryadVertex/VertexHost/system/classlib/include/DrProperties.h

@@ -0,0 +1,79 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//JC Check this file for redundant information.
+
+// Dryad properties (see propertybag.h)
+
+// General purpose parameters
+
+// (BeginTag, UInt16 tag) marks the beginning of a set of data
+// (EndTag, UInt16 tag) marks the end of the a set of data
+// See cstags.h for a description of known tag values
+
+// This file must consist only of DEFINE_DRPROPERTY statements
+
+// Current Allowed types are:
+//
+// UInt32
+// UInt64
+// String
+// Guid
+// TimeStamp
+// TimeInterval
+// BeginTag
+// EndTag
+// DrError
+// DrExitCode
+// Blob
+// EnvironmentBlock
+
+// ******************************************************************************************************
+//
+//              DO NOT RESERVE RANGES IN THIS FILE, USE NEXT AVAILABLE ID AND LEAVE NO HOLES
+//
+// ******************************************************************************************************
+
+// This is a BeginTag
+DEFINE_DRPROPERTY(Prop_Dryad_BeginTag, PROP_SHORTATOM(0x1200), BeginTag, "BeginTag")
+
+// This is an EndTag
+DEFINE_DRPROPERTY(Prop_Dryad_EndTag, PROP_SHORTATOM(0x1201), EndTag, "EndTag")
+
+// This is a Blob value - a win32 environment variable block
+DEFINE_DRPROPERTY(Prop_Dryad_EnvironmentBlock, PROP_LONGATOM(0x129A), EnvironmentBlock, "EnvironmentBlock")
+
+// This is a UInt16
+DEFINE_DRPROPERTY(Prop_Dryad_Port, PROP_SHORTATOM(0x1294), UInt16, "Port")
+
+DEFINE_DRPROPERTY(Prop_Dryad_ShortHostName, PROP_SHORTATOM(0x1293), String, "ShortHostName")
+DEFINE_DRPROPERTY(Prop_Dryad_LongHostName, PROP_LONGATOM(0x1037), String, "LongHostName")
+
+// This is a String
+DEFINE_DRPROPERTY(Prop_Dryad_PodName, PROP_LONGATOM(0x12C6), String, "PodName")
+
+// This is a UInt32 - gives number of entries that follow
+DEFINE_DRPROPERTY(Prop_Dryad_NumEntries, PROP_SHORTATOM(0x124C), UInt32, "NumEntries")
+
+// This is a UInt32 value - a pointer to the primary host in a host list
+DEFINE_DRPROPERTY(Prop_Dryad_PrimaryHost, PROP_SHORTATOM(0x12A8), UInt32, "PrimaryHost")
+
+// This is a UInt32 value - a pointer to the next host to return in a host list
+DEFINE_DRPROPERTY(Prop_Dryad_NextHost, PROP_SHORTATOM(0x12A9), UInt32, "NextHost")

DryadVertex/VertexHost/system/classlib/include/DrPropertiesDef.h

@@ -0,0 +1,60 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+class DrPropertyDumper;
+
+#ifdef DECLARE_DRPROPERTYTYPE
+#undef DECLARE_DRPROPERTYTYPE
+#endif
+
+#define DECLARE_DRPROPERTYTYPE(type) \
+    extern DrError DrPropertyToText_##type(DrPropertyDumper *pDumper, UInt16 enumId, const char *propertyName);
+
+#include "DrPropertyType.h"
+
+#undef DECLARE_DRPROPERTYTYPE
+
+
+#ifdef DEFINE_DRPROPERTY
+#undef DEFINE_DRPROPERTY
+#endif
+
+#define DEFINE_DRPROPERTY(var, value, type, propertyName) \
+    static const UInt16 var = value;
+
+#include "DrProperties.h"
+
+#undef DEFINE_DRPROPERTY
+
+
+// This is a special value for an offset meaning unknown
+const UInt64 DrStreamOffset_Unknown = 0xFFFFFFFFFFFFFFFF;
+
+// This is a special value for an extent offset meaning unknown
+// Note that -1 is used to mean an invalid offset
+const UInt64 DrExtentOffset_Unknown = (UInt64 ) -2;     //$TODO(DanielD) - for consistency (and logging, etc - see SamMck) swap values to have DrExtentOffset_Unknown==-1
+const UInt64 DrExtentOffset_Invalid = (UInt64 ) -1;
+
+const UInt64 DrExtentLength_Invalid = (UInt64) -1;
+
+//Flags for Prop_Dryad_PublishedCrc64
+const UInt64 DrExtentCrc64_Suspect = (UInt64)-1;

DryadVertex/VertexHost/system/classlib/include/DrPropertyDumper.h

@@ -0,0 +1,712 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "DrList.h"
+
+#pragma warning(push)
+#pragma warning(disable:4995)
+
+// A structure that defines a mapping between a particular bit range within a UInt32 and a readable name
+typedef struct {
+    const char *fieldName;              // The name of the bit field
+    UInt32 bitMask;                         // The bits that relate to this mask
+    const char **prgValueNames;   // An array of enum names for the field (of length 2**n-1, where n is the number of bits including
+                                                     // the least significant and most significant 1 bit in bitMask. If an entry is NULL,
+                                                     // the bitmask value will not be displayed. If prgValueNames is NULL, all values will
+                                                     // be displayed as a UInt.
+} DrBitFieldMap;
+
+// A structure that defines a mapping between a UInt32 and a set of bit fields
+typedef struct {
+    int numFields;                          // The number of fields
+    const DrBitFieldMap *prgFields; // The array of bit field mappiings
+} DrBitMaskMap;
+
+/* JC
+class DrPropertyDumper
+{
+private:
+    typedef enum {
+        // We are at the beginning of a property id
+        DrPropertyDumper_StartOfTag = 0,
+
+        // we have read the first byte of the property ID and are waiting for the second byte
+        DrPropertyDumper_ReadingPropertyId,
+
+        // We have read the property ID, but are waiting on 1-4 remaining length bytes (1 byte for a shortatom, 4 bytes for a longatom)
+        DrPropertyDumper_ReadingLength,
+
+        // We have read the property ID and the length, , but are waiting on the remaining data
+        DrPropertyDumper_ReadingData
+    } DrPropertyDumperState;
+
+public:
+
+public:
+    // The default maximum size that will be dumped for blob properties, unknown properties, and partial/malformed properties. Blob properties
+    // that exceed this length will be truncated in the dump with an "...(n bytes remaining)" annotation.
+    static const size_t k_nbDefaultMaxBlobSize = 512;
+
+    // The default maximum size that will be dumped for 
+    static const size_t k_nbDefaultMaxPayloadSize = 64;
+    static const int k_nDefaultIndentSpacesPerLevel = 2;
+
+private:
+    void Construct(size_t maxBlobSize, bool fIncludeComments, int nIndentSpacesPerLevel)
+    {
+        m_state = DrPropertyDumper_StartOfTag;
+        m_partialPropertyId = 0;
+        m_partialPropertyTotalLength = 0;
+        m_pReader = NULL;
+        m_pWriter = NULL;
+        m_fDeleteReader = false;
+        m_fDeleteWriter = false;
+        m_indent = 0;
+        m_fAtBol = true;
+        m_fOutputCrlf = true;
+        m_nbMaxBlobSize = maxBlobSize;
+        m_nbMaxPayloadSize = k_nbDefaultMaxPayloadSize;
+        if (m_nbMaxPayloadSize > m_nbMaxBlobSize) {
+            m_nbMaxPayloadSize = m_nbMaxBlobSize;
+        }
+        m_nIndentSpacesPerLevel = nIndentSpacesPerLevel;
+        m_fIncludeComments = fIncludeComments;
+    }
+
+
+public:
+    DrPropertyDumper()
+    {
+        Construct(k_nbDefaultMaxBlobSize, true, k_nDefaultIndentSpacesPerLevel);
+    }
+
+    DrPropertyDumper(bool fFullContent, bool fIncludeComments = true, int nIndentSpacesPerLevel = k_nDefaultIndentSpacesPerLevel)
+    {
+        Construct(fFullContent ? MAX_SIZE_T : k_nbDefaultMaxBlobSize, fIncludeComments, nIndentSpacesPerLevel);
+    }
+
+    DrPropertyDumper(size_t maxBlobSize, bool fIncludeComments = true, int nIndentSpacesPerLevel = k_nDefaultIndentSpacesPerLevel)
+    {
+        Construct(maxBlobSize, fIncludeComments, nIndentSpacesPerLevel);
+    }
+
+    __declspec(deprecated) void SetReader(DrMemoryReader *pReader, bool fDelete = false)
+    {
+        if (m_pReader != NULL && m_fDeleteReader) {
+            delete m_pReader;
+        }
+        m_pReader = pReader;
+        m_fDeleteReader = fDelete;
+    }
+
+    // If true, then the full content of the property set (including large blobs) will be encoded rather than summarized
+    // If false, then blobs will be limited to the first k_nbDefaultMaxBlobSize bytes.
+    void SetFullContent(bool fFullContent=true)
+    {
+        m_nbMaxBlobSize = fFullContent ? MAX_SIZE_T : k_nbDefaultMaxBlobSize;
+    }
+
+    // If parameter is true, causes bare '\n' to terminate lines, rather than "\r\n"
+    void SetSuppressOutputCr(bool fSuppressOutputCr = true)
+    {
+        m_fOutputCrlf = !fSuppressOutputCr;
+    }
+
+    // Will lower, but not raise the max payload size
+    void SetMaxBlobSize(size_t nbMaxBlobSize)
+    {
+        m_nbMaxBlobSize = nbMaxBlobSize;
+        if (m_nbMaxPayloadSize > m_nbMaxBlobSize) {
+            m_nbMaxPayloadSize = m_nbMaxBlobSize;
+        }
+    }
+
+    // Will not raise the max payload size
+    void SetUnlimitedBlobSize()
+    {
+        SetMaxBlobSize(MAX_SIZE_T);
+    }
+
+    size_t GetMaxBlobSize()
+    {
+        return m_nbMaxBlobSize;
+    }
+
+    void SetIncludeComments(bool fIncludeComments = true)
+    {
+        m_fIncludeComments = fIncludeComments;
+    }
+
+    bool ShouldIncludeComments()
+    {
+        return m_fIncludeComments;
+    }
+
+    // Does not affect the limits for non-payload blobs
+    void SetMaxPayloadSize(size_t nbMaxPayloadSize)
+    {
+        m_nbMaxPayloadSize = nbMaxPayloadSize;
+    }
+
+    // Does not affect the limits for non-payload blobs
+    void SetUnlimitedPayloadSize()
+    {
+        SetMaxPayloadSize(MAX_SIZE_T);
+    }
+
+    size_t GetMaxPayloadSize()
+    {
+        return m_nbMaxPayloadSize;
+    }
+
+    DrMemoryReader *GetReader()
+    {
+        return m_pReader;
+    }
+
+    void SetWriter(DrMemoryWriter *pWriter, bool fDelete = false)
+    {
+        if (m_pWriter != NULL && m_fDeleteWriter) {
+            delete m_pWriter;
+        }
+        m_pWriter = pWriter;
+        m_fDeleteWriter = fDelete;
+    }
+
+    DrMemoryWriter *GetWriter()
+    {
+        return m_pWriter;
+    }
+
+    void SetNumIndentSpacesPerLevel(int n)
+    {
+        m_nIndentSpacesPerLevel = n;
+    }
+
+    int GetNumIndentSpacesPerLevel()
+    {
+        return m_nIndentSpacesPerLevel;
+    }
+    
+    void SetIndent(int n)
+    {
+        m_indent = n;
+    }
+
+    int GetIndent()
+    {
+        return m_indent;
+    }
+
+    void Indent()
+    {
+        m_indent += m_nIndentSpacesPerLevel;
+    }
+
+    void Unindent()
+    {
+        m_indent -= m_nIndentSpacesPerLevel;
+        if (m_indent < 0) {
+            m_indent = 0;
+        }
+    }
+
+    DrError WriteF(const char *pszFormat, ...);
+    DrError VWriteF(const char *pszFormat, va_list args);
+
+    // Writes an XML start tag.  Does NOT affect the DrTag Nesting level
+    DrError WriteStartTag(const char *pszTagName, Size_t length, bool fIncludeLength, bool fNewlineAfter)
+    {
+        DrError err;
+        if (fIncludeLength) {
+            err = WriteF("<%s length=\"%Iu\"%s", pszTagName, length, (fNewlineAfter ? ">\n" : ">"));
+        } else {
+            err = WriteF("<%s%s", pszTagName, (fNewlineAfter ? ">\n" : ">"));
+            m_fAtBol = fNewlineAfter;
+        }
+        Indent();
+        return err;
+    }
+
+    DrError WriteSimpleStartTag(const char *pszTagName, bool fNewlineAfter)
+    {
+        DrError err;
+        err = WriteF("<%s%s", pszTagName, (fNewlineAfter ? ">\n" : ">"));
+        m_fAtBol = fNewlineAfter;
+        Indent();
+        return err;
+    }
+
+    DrError FlushFileWriter()
+    {
+    }
+
+    DrError WriteXmlFileHeader()
+    {
+        return PutStr("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+    }
+    
+    // Writes an XML start tag.  Does NOT affect the DrTag Nesting level
+    // pszAttributes is an optional XML attributes string in the form:
+    //               attrib1="value" attrib2="value"
+    //
+    DrError WriteStartTagWithAttributes(__in const char *pszTagName, __in_opt const char *pszAttributes, bool fNewlineAfter)
+    {
+        DrError err;
+        if (pszAttributes == NULL) {
+            pszAttributes = "";
+        } else {
+            while(pszAttributes[0] == ' ') {
+                pszAttributes++;
+            }
+        }
+        err = WriteF("<%s%s%s%s", pszTagName, (pszAttributes[0] == '\0') ? "" : " ", pszAttributes, (fNewlineAfter ? ">\n" : ">"));
+        m_fAtBol = fNewlineAfter;
+        Indent();
+        return err;
+    }
+
+    DrError WriteStartTagWithLengthAndAttributes(__in const char *pszTagName, Size_t length, __in_opt const char *pszAttributes, bool fNewlineAfter)
+    {
+        DrError err;
+        if (pszAttributes == NULL) {
+            pszAttributes = "";
+        } else {
+            while(pszAttributes[0] == ' ') {
+                pszAttributes++;
+            }
+        }
+        err = WriteF("<%s length=\"%Iu\"%s%s%s", pszTagName, length, (pszAttributes[0] == '\0') ? "" : " ", pszAttributes, (fNewlineAfter ? ">\n" : ">"));
+        m_fAtBol = fNewlineAfter;
+        Indent();
+        return err;
+    }
+
+    // Writes an XML end tag.  DOes NOT affect the DrTag Nesting level
+    DrError WriteEndTag(const char *pszTagName, bool fNewlineBefore)
+    {
+        Unindent();
+        DrError err = WriteF("%s/%s>\n", (fNewlineBefore ? "\n<" : "<"), pszTagName);
+        return err;
+    }
+
+    // The string generated by pszFormat must be XML-encoded or contain no XML delimiters
+    DrError VWriteSimpleTagValue(
+        const char *pszTagName,
+        UInt32 length,
+        bool fIncludeLength,
+        bool fSeperateLine,
+        const char *pszFormat,
+        va_list args)
+    {
+        DrError err;
+
+        WriteStartTag(pszTagName, length, fIncludeLength, fSeperateLine);
+        VWriteF(pszFormat, args);
+        err = WriteEndTag(pszTagName, fSeperateLine);
+        return err;
+    }
+
+    // The string generated by pszFormat must be XML-encoded or contain no XML delimiters
+    DrError WriteSimpleTagValue(
+        const char *pszTagName,
+        UInt32 length,
+        bool fIncludeLength,
+        bool fSeperateLine,
+        const char *pszFormat,
+        ...)
+    {
+        va_list args;
+        va_start(args, pszFormat);
+        return VWriteSimpleTagValue(pszTagName, length, fIncludeLength, fSeperateLine, pszFormat, args);
+    }
+
+    // The string generated by pszFormat must be XML-encoded or contain no XML delimiters
+    DrError WriteSimpleTagValue(
+        const char *pszTagName,
+        const char *pszFormat,
+        ...)
+    {
+        va_list args;
+        va_start(args, pszFormat);
+        return VWriteSimpleTagValue(pszTagName, 0, false, false, pszFormat, args);
+    }
+
+    // The string generated by pszFormat must be XML-encoded or contain no XML delimiters
+    DrError WriteSimpleTagValue(
+        const char *pszTagName,
+        UInt32 length, 
+        const char *pszFormat,
+        ...)
+    {
+        va_list args;
+        va_start(args, pszFormat);
+        return VWriteSimpleTagValue(pszTagName, length, true, false, pszFormat, args);
+    }
+
+    // Puts a string to output, indenting lines as appropriate
+    // Must obey XML conventions
+    DrError PutStr(const char *psz, int len = -1);
+
+    DrError PutBolLeader()
+    {
+        if (m_indent > 0) {
+            m_fAtBol = false;
+            for (int i = 0; i < m_indent; i++) {
+                m_pWriter->WriteChar(' ');
+            }
+        }
+        return m_pWriter->GetStatus();
+    }
+
+    DrError PutUInt64TagValue(const char *pszTagName, UInt64 val, bool fHex = false)
+    {
+        const char *pszFormat = (fHex ? "0x%016I64x" : "%I64u");
+        return WriteSimpleTagValue(pszTagName, pszFormat, val);
+    }
+
+    DrError PutUInt32TagValue(const char *pszTagName, UInt32 val, bool fHex = false)
+    {
+        const char *pszFormat = (fHex ? "0x%08x" : "%u");
+        return WriteSimpleTagValue(pszTagName, pszFormat, val);
+    }
+
+    DrError PutDoubleTagValue(const char *pszTagName, double val)
+    {
+        const char *pszFormat = "%lf";
+        return WriteSimpleTagValue(pszTagName, pszFormat, val);
+    }
+
+    DrError PutUInt32BitMaskValue(const char *pszTagName, UInt32 val, const DrBitMaskMap *pMap);
+
+    DrError PutUInt16TagValue(const char *pszTagName, UInt16 val, bool fHex = false)
+    {
+        const char *pszFormat = (fHex ? "0x%04x" : "%u");
+        return WriteSimpleTagValue(pszTagName, pszFormat, (UInt32)val);
+    }
+
+    DrError PutTagIdTagValue(const char *pszTagName, UInt16 val)
+    {
+        const char *pszTagIdName = GetTagName(val);
+        if (pszTagIdName == NULL) {
+            return WriteSimpleTagValue(pszTagName, "0x%04x", (UInt32)val);
+        } else {
+            return WriteSimpleTagValue(pszTagName, "0x%04x <!-- %s -->", (UInt32)val, pszTagIdName);
+        }
+    }
+
+    DrError PutInt64TagValue(const char *pszTagName, Int64 val, bool fHex = false)
+    {
+        const char *pszFormat = (fHex ? "0x%016I64x" : "%I64d");
+        return WriteSimpleTagValue(pszTagName, pszFormat, val);
+    }
+
+    DrError PutInt32TagValue(const char *pszTagName, Int32 val, bool fHex = false)
+    {
+        const char *pszFormat = (fHex ? "0x%08x" : "%d");
+        return WriteSimpleTagValue(pszTagName, pszFormat, val);
+    }
+
+    DrError PutInt16TagValue(const char *pszTagName, Int16 val, bool fHex = false)
+    {
+        const char *pszFormat = (fHex ? "0x%04x" : "%d");
+        return WriteSimpleTagValue(pszTagName, pszFormat, (Int32)val);
+    }
+
+    DrError PutBooleanTagValue(const char *pszTagName, bool val)
+    {
+        return WriteSimpleTagValue(pszTagName, "%s", (val ? "true" : "false"));
+    }
+
+    // safely XML-encodes the string
+    DrError PutStringTagValue(const char *pszTagName, const char *val, int len = -1)
+    {
+        if (val == NULL) {
+            return WriteSimpleTagValue(pszTagName, "%s", "<!-- null string -->");
+        } else {
+            if (len < 0) {
+                len = (int)strlen(val);
+            }
+            DrStr512 strXml;
+            strXml.AppendXmlEncodedString(val, (Size_t)len, true);
+            return WriteSimpleTagValue(pszTagName, "%s", strXml.GetString());
+        }
+    }
+
+    DrError PutGuidTagValue(const char *pszTagName, const DrGuid& val)
+    {
+        char buff[40];
+        return WriteSimpleTagValue(pszTagName, "%s", val.ToString(buff, false));
+    }
+
+    DrError PutTimeStampTagValue(const char *pszTagName, DrTimeStamp val)
+    {
+        char buff[64];
+        DrError err;
+        if (val == DrTimeStamp_Never) {
+                err = WriteSimpleTagValue(pszTagName, "never", (UInt64)val, buff);
+        } else if (val == DrTimeStamp_LongAgo) {
+                err = WriteSimpleTagValue(pszTagName, "long ago", (UInt64)val, buff);
+        } else {
+            err = DrTimeStampToString(val, buff, sizeof(buff), false);
+            if (err == DrError_OK) {
+                err = WriteSimpleTagValue(pszTagName, "%I64u<!-- %s -->", (UInt64)val, buff);
+            } else {
+                m_pWriter->SetStatus(err);
+            }
+        }
+        return err;
+    }
+
+    DrError PutNodeAddressTagValue(const char *pszTagName, const DrNodeAddress& val)
+    {
+        char buff[64];
+        DrError err = val.ToAddressPortString(buff, sizeof(buff));
+        if (err == DrError_OK) {
+            err = WriteSimpleTagValue(pszTagName, "%s", buff);
+        } else {
+            m_pWriter->SetStatus(err);
+        }
+        return err;
+    }
+
+    DrError PutTimeIntervalTagValue(const char *pszTagName, DrTimeInterval val)
+    {
+        DrError err;
+
+        char buff[k_DrTimeIntervalStringBufferSize];
+        err = DrTimeIntervalToString(val, buff, sizeof(buff));
+        LogAssert(err == DrError_OK);
+
+        err = WriteSimpleTagValue(pszTagName, "%I64d<!-- %s -->", (Int64)val, buff);
+        
+        return err;
+    }
+
+    DrError PutDrErrorTagValue(const char *pszTagName, DrError val)
+    {
+        char buff[1024];
+        const char *pszErr = DrGetErrorDescription(val, buff, sizeof(buff));
+        // Assumes error code descriptions are XML-friendly
+        DrError err = WriteSimpleTagValue(pszTagName, "0x%08x<!-- %s -->", (UInt32)val, pszErr);
+        return err;
+    }
+
+    DrError PutDrExitCodeTagValue(const char *pszTagName, DrExitCode val)
+    {
+        // Assumes exit code descriptions are XML-friendly
+        DrError err = WriteSimpleTagValue(pszTagName, "0x%08x<!-- %s -->", (UInt32)val, DREXITCODESTRING(val));
+        return err;
+    }
+
+    DrError PutBlobTagValue(const char *pszTagName, DrMemoryReader *pReader, Size_t length, Size_t maxShowLength, const char *pszExtraAttributes = NULL);
+    DrError PutEnvironmentBlockTagValue(const char *pszTagName, const DrEnvironmentStrings& envBlock);
+
+    static const char *GetPropertyName(UInt16 enumId);
+    static const char *GetTagName(UInt16 tagId);
+
+
+    // Returns the number of nested DrTag begintags that are currently in effect.  0 means we are not inside a begin tag
+    UInt32 GetCurrentTagLevel()
+    {
+        return m_beginTagStack.NumEntries();
+    }
+
+    // returns DrTag_InvalidTag if we are at level 0
+    UInt16 GetCurrentNestedTagId()
+    {
+        if (m_beginTagStack.NumEntries() == 0) {
+            return DrTag_InvalidTag;
+        } else {
+            return m_beginTagStack.TopOfStack();
+        }
+    }
+
+    // returns NULL if the current nested DrTag tag name is not known
+    const char *GetCurrentNestedTagName()
+    {
+        const char * pszTagName = GetTagName(GetCurrentNestedTagId());
+        return pszTagName;
+    }
+
+    // Returns the DrTag tag level below which we are not allowed to pop, due to a balanced block being in effect
+    UInt32 GetCurrentMinimumTagLevel()
+    {
+        if (m_balancedBlockStack.NumEntries() == 0) {
+            return 0;
+        } else {
+            return m_balancedBlockStack.TopOfStack();
+        }
+    }
+
+    // returns true if there is a partially written DrProperty waiting to be completed before it is dumped
+    bool PartialPropertyIsPending()
+    {
+        return (m_state != DrPropertyDumper_StartOfTag);
+    }
+    
+
+    // Begins a section where begin/end DrTags  must be balanced and there must not be be a partial property at the beginning or end
+    // On entry, any partially written property is written out as a specially tagged "incomplete" property
+    void BeginBalancedBlock();
+
+    // If there is a partially written property pending, it is written out as a specially tagged "incomplete" property.
+    // This resets the parser to expect a property ID next.
+    // Also, if there are outstanding open begin-DrTag beyond the currend balanced block level, writes end DrTags for them (with XML comments
+    // describing them as missing).
+    // After this call, the current DrTag level will be equal to the current balanced block level.
+    DrError SynchronizeToBalancedBlock();
+
+    // endss a section where begin/end DrTags must be balanced.
+    // If there is a partially written property pending, it is written out as a specially tagged "incomplete" property.
+    // This resets the parser to expect a property ID next.
+    // Also, if there are outstanding open begin-DrTag beyond the currend balanced block level, writes end DrTags for them (with XML comments
+    // describing them as missing).
+    // After this call, the current DrTag level will be equal to the current balanced block level.
+    DrError EndBalancedBlock()
+    {
+        LogAssert(m_balancedBlockStack.NumEntries() != 0);
+        SynchronizeToBalancedBlock();
+        LogAssert(GetCurrentTagLevel() == GetCurrentMinimumTagLevel());
+        m_balancedBlockStack.Pop();
+        return m_pWriter->GetStatus();
+    }
+
+    // writes an XML equivalent to a  DrTag "begin tag", and pushes a level
+    DrError WriteAndPushDrBeginTag(UInt16 tagId);
+
+    // writes an XML equivalent to a  DrTag "end tag", and pops a level 
+    DrError WriteAndPopDrEndTag(UInt16 tagId);
+
+    // If there is a partially written property pending, it is written out as a specially tagged "incomplete" property.
+    // This resets the parser to expect a property ID next.
+    DrError WritePartialProperty();
+
+    // Final flush of the dumper. must be called from balanced block level 0
+    DrError WriteIncompleteEndTagsAndFlushWriter()
+    {
+        LogAssert(m_balancedBlockStack.NumEntries() == 0);
+        SynchronizeToBalancedBlock();
+        return m_pWriter->FlushMemoryWriter();
+    }
+    // reads all remaining bytes from the reader and generates XML output as appropriate
+    // The content of the reader does not need to be balanced or complete, and it need not begin or end at a property boundary; you may call this multiple times to
+    // represent a coherent property stream
+    // If you want to enforce tag balancing accross a section, call BeginBalancedBlock before calling this, and EndBalancedBlock after calling this.
+    // if maxLength is specified, the function will return after reading maxLength bytes even if end of stream is not reached
+    DrError ParseAndWriteFromReader(DrMemoryReader *pReader, Size_t maxLength=MAX_SIZE_T);
+
+    DrError ParseAndWriteFromBuffer(DrMemoryBuffer *pBuffer, Size_t initialOffset=0, Size_t maxLength = MAX_SIZE_T)
+    {
+        DrMemoryBufferReader reader(pBuffer);
+        if (initialOffset != 0) {
+            reader.SetBufferOffset(initialOffset);
+        }
+        return ParseAndWriteFromReader(&reader, maxLength);
+    }
+
+    DrError ParseAndWriteFromSingleBlock(const void *pData, Size_t length)
+    {
+        DrSingleBlockReader reader(pData, length);
+        return ParseAndWriteFromReader(&reader, length);
+    }
+
+
+    // reads the next single tag&value and outputs it. Does not reads aggregates--begintag and endtag are
+    // trated as independent values; however, it does increment indentation in begintag, and decrement it on endtag.
+    __declspec(deprecated) DrError PutNextPropertyTagValue()
+    {
+        return PutNextPropertyTagValue(m_pReader);
+    }
+
+    
+    DrError PutNextPropertyTagValue(DrMemoryReader *pReader);
+
+    // Reads the remainder of the input stream (Until DrError_EndOfStream is returned) as a sequential list
+    // of 0 or more properties and/or aggregates, and outputs them accordingly. Does not return until the entire stream has been read.
+    // Will always write a balanced set even if the input is unbalanced (will add missing end tags, etc., as necessary).
+    // returns the writer status
+    __declspec(deprecated) DrError PutNestedPropertyList()
+    {
+        return PutNestedPropertyList(m_pReader);
+    }
+    DrError PutNestedPropertyList(DrMemoryReader *pReader)
+    {
+        BeginBalancedBlock();
+        ParseAndWriteFromReader(pReader);
+        return EndBalancedBlock();
+    }
+
+    bool IsAtBeginningOfLine()
+    {
+        return m_fAtBol;
+    }
+
+private:
+    DrMemoryReader *m_pReader;
+    bool m_fDeleteReader;
+    DrMemoryWriter *m_pWriter;
+    bool m_fDeleteWriter;
+    int m_indent;
+    int m_nIndentSpacesPerLevel;
+    bool m_fAtBol;  // Beginning-of-line flag
+    bool m_fOutputCrlf;  // True if '\r' should precede each '\n' on output.
+    size_t m_nbMaxBlobSize;
+    size_t m_nbMaxPayloadSize;
+    bool m_fIncludeComments;
+
+    DrPropertyDumperState m_state;
+
+    // A stack of unmatched begintag tagids. The top of stack is the end tag we are currently looking for.
+    // The number of entries in this list is the current tag "level"
+    DrValList<UInt16> m_beginTagStack;
+
+    // A stack of pushed "BeginBalancedBlock" "level" parameters.
+    // The top of the stack is the current balanced block level -- the parser will not allow end tags to pop the beginTagStack
+    // above this level.
+    DrValList<UInt32> m_balancedBlockStack;
+
+    // buffer to store an incomplete property we are parsing
+    // Includes property tag
+    DrRef<DrSimpleHeapBuffer> m_pPartialPropertyBuffer;
+
+    UInt16 m_partialPropertyId;
+    Size_t m_partialPropertyTotalLength;
+    
+};
+*/
+
+extern void DrInitPropertyTable();
+extern void DrInitTagTable();
+
+/* JC
+typedef DrError (*DrPropertyConverter)(DrPropertyDumper *pDumper, UInt16 enumId, const char *propertyName);
+
+// returns ERROR_ALREADY_ASSIGNED if the property has already been defined
+extern DrError DrAddPropertyToDumper(UInt16 prop, const char *pszDescription, DrPropertyConverter pConverter);
+
+// returns ERROR_ALREADY_ASSIGNED if the tag has already been defined
+extern DrError DrAddTagToDumper(UInt16 tag, const char *pszDescription);
+*/
+
+#pragma warning(pop)
+

DryadVertex/VertexHost/system/classlib/include/DrPropertyType.h

@@ -0,0 +1,63 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+// This file must consist only of DECLARE_DRPROPERTYTYPE statements
+
+DECLARE_DRPROPERTYTYPE(Unknown)
+DECLARE_DRPROPERTYTYPE(Boolean)
+DECLARE_DRPROPERTYTYPE(Int16)
+DECLARE_DRPROPERTYTYPE(Int32)
+DECLARE_DRPROPERTYTYPE(Int64)
+DECLARE_DRPROPERTYTYPE(UInt16)
+DECLARE_DRPROPERTYTYPE(UInt32)
+DECLARE_DRPROPERTYTYPE(UInt64)
+DECLARE_DRPROPERTYTYPE(Double)
+DECLARE_DRPROPERTYTYPE(HexUInt16)
+DECLARE_DRPROPERTYTYPE(HexUInt32)
+DECLARE_DRPROPERTYTYPE(HexUInt64)
+DECLARE_DRPROPERTYTYPE(String)
+DECLARE_DRPROPERTYTYPE(Guid)
+DECLARE_DRPROPERTYTYPE(TimeStamp)
+DECLARE_DRPROPERTYTYPE(TimeInterval)
+DECLARE_DRPROPERTYTYPE(BeginTag)
+DECLARE_DRPROPERTYTYPE(EndTag)
+DECLARE_DRPROPERTYTYPE(DrError)
+DECLARE_DRPROPERTYTYPE(DrExitCode)
+DECLARE_DRPROPERTYTYPE(Blob)
+DECLARE_DRPROPERTYTYPE(Payload)
+DECLARE_DRPROPERTYTYPE(EnvironmentBlock)
+DECLARE_DRPROPERTYTYPE(PropertyList)
+DECLARE_DRPROPERTYTYPE(TagIdValue)
+
+DECLARE_DRPROPERTYTYPE(AppendExtentOptions)
+DECLARE_DRPROPERTYTYPE(AppendBlockOptions)
+DECLARE_DRPROPERTYTYPE(SyncOptions)
+DECLARE_DRPROPERTYTYPE(SyncDirectiveOptions)
+DECLARE_DRPROPERTYTYPE(ReadExtentOptions)
+DECLARE_DRPROPERTYTYPE(AppendStreamOptions)
+DECLARE_DRPROPERTYTYPE(EnumDirectoryOptions)
+DECLARE_DRPROPERTYTYPE(EnInfoBits)
+DECLARE_DRPROPERTYTYPE(UpdateExtentMetadataOptions)
+DECLARE_DRPROPERTYTYPE(StreamInfoBits)
+DECLARE_DRPROPERTYTYPE(StreamCapabilityBits)
+DECLARE_DRPROPERTYTYPE(ExtentInfoBits)
+DECLARE_DRPROPERTYTYPE(ExtentInstanceInfoBits)
+DECLARE_DRPROPERTYTYPE(FailureInjectionOptions)
+

DryadVertex/VertexHost/system/classlib/include/DrRefCounter.h

@@ -0,0 +1,839 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#ifndef __DRYADREFCOUNTER_H__
+#define __DRYADREFCOUNTER_H__
+
+#pragma once
+
+template<class Ty> __forceinline Ty& DrRemoveConst(const Ty& x)
+{
+    return const_cast<Ty&>(x);
+}
+
+template<class Ty> __forceinline Ty *DrRemovePtrConst(const Ty *pX)
+{
+    return const_cast<Ty *>(pX);
+}
+
+UInt64 GetUniqueObjectID();
+
+// a type for automatically assigning a unique object ID for static initializers
+class DrObjID
+{
+public:
+    DrObjID() { _val = GetUniqueObjectID();}
+    UInt64 Value() const { return _val; };
+private:
+    UInt64 _val;
+};
+
+/*
+ * DrRefCounter
+ *
+ * Implements simple reference count functionality.
+ * Derive any ref counted objects from this class.
+ */
+
+class DrRefCounter;
+
+static const LONG k_lDecommissionedRefCount = (LONG)-1234;
+
+class DrRefCountMonitor
+{
+public:
+    virtual void OnRefCountChanged(const DrRefCounter *pCounter, void *pContext, LONG newCount, const char *pszReason) = 0;
+};
+
+class IDrRefCounter
+{
+public:
+    virtual LONG IncRef() = 0;
+    LONG IncRef() const
+    {
+        // IncRef/DecRef are allowed even for const instances
+        return DrRemovePtrConst(this)->IncRef();
+    }
+
+    // called by weak reference pointers (under the shared lock) to IncRef this object.
+    // returns TRUE if the IncRef could be performed, false if the object is already unreferenceable
+    // The default implementation succeeds unless the current refcount is 0, on the assumption that the implementation
+    // will clear all existing weak references after the refcount becomes 0 and before the object is deleted.
+    virtual bool IncRefFromWeakReferenceLocked() = 0;
+    bool IncRefFromWeakReferenceLocked() const
+    {
+        // IncRef/DecRef are allowed even for const instances
+        return DrRemovePtrConst(this)->IncRefFromWeakReferenceLocked();
+    }
+
+    // called by weak reference Freememory implementations (under the shared lock) to get the refcount on this object.
+    virtual LONG GetRefCountLocked() const = 0;
+
+    virtual LONG DecRef() = 0;
+    LONG DecRef() const
+    {
+        // IncRef/DecRef are allowed even for const instances
+        return DrRemovePtrConst(this)->DecRef();
+    }
+    virtual void FreeMemory() = 0;
+
+    void FreeMemory() const
+    {
+        // FreeMemory is allowed even for const instances
+        DrRemovePtrConst(this)->FreeMemory();
+    }
+    
+    virtual ~IDrRefCounter()
+    {
+    }
+    virtual UInt64 GetOID() const = 0;  // Get object-id, an (almost) unique identifier that can be used for logging (this way we can see related log lines for a given object)
+};
+
+
+class DrOneInitializedVolatileLong
+{
+public:
+   volatile LONG m_long;
+
+   DrOneInitializedVolatileLong()
+   {
+        m_long = 1;
+   }
+};
+
+class DrOneInitializedZeroDestroyedVolatileLong : public DrOneInitializedVolatileLong
+{
+public:
+    ~DrOneInitializedZeroDestroyedVolatileLong()
+    {
+        LogAssert(m_long == 0 || m_long == k_lDecommissionedRefCount);
+    }
+};
+
+class DrOneInitializedOneDestroyedVolatileLong : public DrOneInitializedVolatileLong
+{
+public:
+    ~DrOneInitializedOneDestroyedVolatileLong()
+    {
+        LogAssert(m_long == 1 || m_long == k_lDecommissionedRefCount);
+    }
+};
+
+
+#define DRREFCOUNTNOIMPL \
+    public: \
+        virtual LONG IncRef() override = 0; \
+        virtual LONG DecRef() override = 0; \
+        virtual UInt64 GetOID() const override = 0; \
+        virtual bool IncRefFromWeakReferenceLocked() override = 0; \
+        virtual LONG GetRefCountLocked() const override = 0; \
+        virtual void FreeMemory() override = 0; \
+
+
+// NOTE: the following could use DrOneInitializedZeroDestroyedVolativeLong, but some people create static instances of this...
+#define DRREFCOUNTIMPL_NOFREEMEMORY_BASE(InterfaceClass) \
+    protected: \
+        mutable DrOneInitializedVolatileLong m_iRefCount; \
+        DrObjID m_oid; \
+    public: \
+        virtual LONG IncRef() override\
+        { \
+            LONG i; \
+            i = InterlockedIncrement (&(m_iRefCount.m_long)); \
+            LogAssert (i > 1); \
+            return i; \
+        } \
+        virtual LONG DecRef() override\
+        { \
+            LONG i; \
+            i = InterlockedDecrement (&(m_iRefCount.m_long)); \
+            if (i <= 0) \
+            { \
+                LogAssert (i == 0); \
+                FreeMemory (); \
+            } \
+            return i; \
+        } \
+        virtual UInt64 GetOID() const override{ return m_oid.Value(); } \
+        virtual bool IncRefFromWeakReferenceLocked() override\
+        { \
+            if (m_iRefCount.m_long == 0) { \
+                return false; \
+            } \
+            LONG i = InterlockedIncrement (&(m_iRefCount.m_long)); \
+            LogAssert (i > 1); \
+            return true; \
+        } \
+        virtual LONG GetRefCountLocked() const override\
+        { \
+            return m_iRefCount.m_long; \
+        } \
+
+
+#define DRREFCOUNTIMPL_STATIC_BASE(InterfaceClass) \
+    private: \
+        static void * operator new( size_t) { LogAssert(false); return NULL; } \
+        static void operator delete( void *) { LogAssert(false); } \
+    protected: \
+        mutable DrOneInitializedOneDestroyedVolatileLong m_iRefCount; \
+        DrObjID m_oid; \
+    public: \
+        virtual LONG IncRef() override\
+        { \
+            LONG i; \
+            i = InterlockedIncrement (&(m_iRefCount.m_long)); \
+            LogAssert (i > 1); \
+            return i; \
+        } \
+        virtual LONG DecRef() override\
+        { \
+            LONG i; \
+            i = InterlockedDecrement (&(m_iRefCount.m_long)); \
+            LogAssert(i > 0); \
+            return i; \
+        } \
+        virtual UInt64 GetOID() const override{ return m_oid.Value(); } \
+        virtual bool IncRefFromWeakReferenceLocked() override \
+        { \
+            LONG i = InterlockedIncrement (&(m_iRefCount.m_long)); \
+            LogAssert (i > 1); \
+            return true; \
+        } \
+        virtual LONG GetRefCountLocked() const override\
+        { \
+            return m_iRefCount.m_long; \
+        } \
+
+
+#define DRREFCOUNTIMPL_NOFREEMEMORY DRREFCOUNTIMPL_NOFREEMEMORY_BASE(IDrRefCounter)
+
+#define DRREFCOUNTIMPL_BASE(InterfaceClass) \
+        DRREFCOUNTIMPL_NOFREEMEMORY_BASE(InterfaceClass) \
+    protected: \
+        virtual void FreeMemory() override\
+        { \
+            delete this; \
+        }
+
+
+#define DRREFCOUNTIMPL DRREFCOUNTIMPL_BASE(IDrRefCounter)
+#define DRREFCOUNTIMPL_STATIC DRREFCOUNTIMPL_STATIC_BASE(IDrRefCounter)
+
+class DrRefCounter : public IDrRefCounter
+{
+protected:
+    mutable volatile LONG m_iRefCount;
+    DrRefCountMonitor *m_pMonitor;
+    void *m_pMonitorContext;
+    UInt64  m_oid;
+
+    DrRefCounter()
+    {
+        m_iRefCount = 1;
+        m_pMonitor = NULL;
+        m_pMonitorContext = NULL;
+        m_oid = GetUniqueObjectID();
+    }
+
+
+    virtual ~DrRefCounter()
+    {
+        LogAssert (m_iRefCount == 0 || m_iRefCount == k_lDecommissionedRefCount);
+    }
+
+    // Called when the refcount becomes zero.
+    virtual void FreeMemory() override
+    {
+        delete this;
+    }
+
+public:
+
+    // not threadsafe
+    void SetRefCountMonitor(DrRefCountMonitor *pMonitor, void *pContext = NULL)
+    {
+        m_pMonitor = pMonitor;
+        m_pMonitorContext = pContext;
+        if (m_pMonitor != NULL) {
+            m_pMonitor->OnRefCountChanged(this, m_pMonitorContext, m_iRefCount, "SetMonitor");
+        }
+    }
+
+    virtual LONG IncRef() override
+    {
+        LONG i;
+        i = InterlockedIncrement (&m_iRefCount);
+        LogAssert (i > 1);
+        if (m_pMonitor != NULL) {
+            m_pMonitor->OnRefCountChanged(this, m_pMonitorContext, i, "IncRef");
+        }
+        return i;
+    }
+
+    // Increments without checking that the previous refcount was > 0. Used for special circumstances where we
+    // may temporarily increment from a 0 refcount
+    LONG IncRefNoCheck()
+    {
+        LONG i;
+        i = InterlockedIncrement (&m_iRefCount);
+        if (m_pMonitor != NULL) {
+            m_pMonitor->OnRefCountChanged(this, m_pMonitorContext, i, "IncRefNoCheck");
+        }
+        return i;
+    }
+
+    // called by weak reference pointers (under the shared lock) to IncRef this object.
+    // returns TRUE if the IncRef could be performed, false if the object is already unreferenceable
+    // The default implementation succeeds unless the current refcount is 0, on the assumption that the implementation
+    // will clear all existing weak references after the refcount becomes 0 and before the object is deleted.
+    virtual bool IncRefFromWeakReferenceLocked() override
+    {
+        if (m_iRefCount == 0) {
+            return false;
+        }
+        LONG i = InterlockedIncrement (&m_iRefCount);
+        LogAssert (i > 1); \
+        if (m_pMonitor != NULL) {
+            m_pMonitor->OnRefCountChanged(this, m_pMonitorContext, i, "IncRefFromWeakReferenceLocked");
+        }
+        return true;
+    }
+
+    virtual LONG GetRefCountLocked() const override
+    {
+        return m_iRefCount;
+    }
+
+    virtual LONG DecRef() override
+    {
+        LONG i;
+        if (m_pMonitor != NULL) {
+            // HACK: the refcount may be wrong, but it will work
+            m_pMonitor->OnRefCountChanged(this, m_pMonitorContext, m_iRefCount-1L, "DecRef");
+        }
+        i = InterlockedDecrement (&m_iRefCount);
+        if (i <= 0)
+        {
+            LogAssert (i == 0);
+            FreeMemory ();
+        }
+        return i;
+    }
+
+    // Decrements without special handling when the refcount becomes zero. Used for special circumstances
+    // where the refcount has been temporarily incremented from zero and another context will be
+    // calling FreeMemory().
+    LONG DecRefNoFree()
+    {
+        LONG i;
+        if (m_pMonitor != NULL) {
+            // HACK: the refcount may be wrong, but it will work
+            m_pMonitor->OnRefCountChanged(this, m_pMonitorContext, m_iRefCount-1L, "DecRefNoFree");
+        }
+        i = InterlockedDecrement (&m_iRefCount);
+        LogAssert(i >= 0);
+        return i;
+    }
+
+    virtual UInt64 GetOID() const override
+    {
+        return m_oid;
+    }
+
+
+    void ResetRefCounter()
+    {
+        LONG i;
+        i = InterlockedExchange (&m_iRefCount, 1);
+        if ( i != 0 )
+        {
+            DrLogE( "DecRef error - refCount=%ld, oid=%I64x",  i, GetOID() );
+            LogAssert (i == 0);
+        }
+        if (m_pMonitor != NULL) {
+            m_pMonitor->OnRefCountChanged(this, m_pMonitorContext, 1, "ResetRefCounter");
+        }
+    }
+
+    // Abandons the refcounter. Used when a subclass overrides the implementation. Prevents the refcounter from being used, and
+    // prevents an assertion failure when the object is destructed.
+    void DecommissionRefCounter()
+    {
+        LONG i;
+        // We set a magic number to indicate that it is decommissioned. Any call to IncRef or DecRef will assert, and destruction will succeed
+        i = InterlockedExchange (&m_iRefCount, k_lDecommissionedRefCount);
+        LogAssert(i == 1 || i == k_lDecommissionedRefCount);
+    }
+
+    // The value returned by this method is not stable unless the caller provides another mechanism for guaranteeing that
+    // noone calls IncRef or DecRef.
+    LONG GetRefCount() const
+    {
+        return m_iRefCount;
+    }
+};
+
+template <class T> class DrRef
+{
+
+public:
+    DrRef()
+    {
+        p = NULL;
+    }
+
+    DrRef(T* lp)
+    {
+        p = lp;
+
+        if (p != NULL) {
+            DrRemovePtrConst(p)->IncRef();
+        }
+    }
+
+    explicit DrRef(const DrRef<T>& ref)
+    {
+        p = ref.p;
+
+        if (p != NULL) {
+            DrRemovePtrConst(p)->IncRef();
+        }
+    }
+
+    ~DrRef()
+    {
+        if (p != NULL) {
+            DrRemovePtrConst(p)->DecRef();
+            p = NULL;   // Make sure we AV in case someone is using DrRef after DecRef
+        }
+    }
+
+    operator T*() const
+    {
+        return p;
+    }
+
+    T& operator*() const
+    {
+        return *p;
+    }
+
+    T* operator->() const
+    {
+        return p;
+    }
+
+    bool operator!() const
+    {
+        return (p == NULL);
+    }
+
+    bool operator<(T* pT) const
+    {
+        return (p < pT);
+    }
+
+    bool operator>(T* pT) const
+    {
+        return (p < pT);
+    }
+
+    bool operator<=(T* pT) const
+    {
+        return (p <= pT);
+    }
+
+    bool operator>=(T* pT) const
+    {
+        return (p < pT);
+    }
+
+    bool operator==(T* pT) const
+    {
+        return (p == pT);
+    }
+
+    bool operator!=(T* pT) const
+    {
+        return (p != pT);
+    }
+
+    DrRef& Set(T* lp)
+    {
+        if (p != lp) {
+            if (lp != NULL) {
+                DrRemovePtrConst(lp)->IncRef();
+            }
+
+            if (p != NULL) {
+                DrRemovePtrConst(p)->DecRef();
+            }
+
+            p = lp;
+        }
+
+        return *this;
+    }
+
+    DrRef& operator=(T* lp)
+    {
+        return Set(lp);
+    }
+
+    DrRef& operator=(const DrRef<T>& ref)
+    {
+        return Set(ref.p);
+    }
+
+    // Release the interface and set to NULL
+    void Release()
+    {
+        T* pTemp = p;
+        if (pTemp != NULL) {
+            p = NULL;
+            DrRemovePtrConst(pTemp)->DecRef();
+        }
+    }
+
+    //
+    // Attach to an existing interface (does not IncRef)
+    //
+    void Attach(T* p2)
+    {
+        //
+        // Remove reference to previous interface
+        //
+        if (p != NULL) 
+        {
+            DrRemovePtrConst(p)->DecRef();
+        }
+
+        //
+        // Update current interface
+        //
+        p = p2;
+    }
+
+    // Detach the interface (does not DecRef)
+    T* Detach()
+    {
+        T* pt = p;
+        p = NULL;
+        return pt;
+    }
+
+    void TransferFrom( DrRef<T>& source)
+    {
+        Attach(source.Detach());
+    }
+
+    template<class T2> void TransferFrom( DrRef<T2>& source)
+    {
+        T2 *p2 = source.Detach();
+        if (p != NULL) {
+            DrRemovePtrConst(p)->DecRef();
+        }
+        if (p2== NULL) {
+            p = NULL;
+        } else {
+            p = dynamic_cast<T *>(p2);
+            LogAssert(p != NULL);
+        }
+    }
+
+    T* Ptr() const
+    {
+        return p;
+    }
+
+private:
+    T* p;
+};
+
+// Growable vector of DrRef smart pointers to arbitrary refcounted typed items
+// Insertions and deletions can be performed both at the head and at the tail of the list, making it suitable for queues.
+template<class t> class DrRefList
+{
+public:    
+    DrRefList()
+    {
+        m_nEntries = 0;
+        m_nAllocated = 0;
+        m_prgEntries = NULL;
+    }
+
+    ~DrRefList()
+    {
+        if (m_prgEntries != NULL) {
+            delete[] m_prgEntries;
+            m_prgEntries = NULL;
+        }
+        m_nEntries = 0;
+        m_nAllocated = 0;
+    }
+
+    // forces the buffer to be reallocated with the given size, even if it
+    // is the same as the current size.
+    // the requested size must be big enough to hold the valid entries.
+    // On exit, the valid entries are always contiguous starting at offset 0
+    // if n==0, frees the buffer
+    void ForceRealloc(::UInt32 n)
+    {
+        LogAssert(n >= m_nEntries);
+        DrRef<t> *pnew = NULL;
+        if (n != 0) {
+            pnew = new DrRef<t>[n];
+            LogAssert(pnew != NULL);
+            ::UInt32 uFrom = m_uFirstEntry;
+            for (::UInt32 i = 0; i < m_nEntries; i++) {
+                pnew[i].TransferFrom(m_prgEntries[uFrom++]);
+                if (uFrom >= m_nAllocated) {
+                    uFrom = 0;
+                }
+            }
+        }
+        if (m_prgEntries != NULL) {
+            delete[] m_prgEntries;
+        }
+        m_prgEntries = pnew;
+        m_nAllocated = n;
+        m_uFirstEntry = 0;
+    }
+
+    // reallocates the buffer if there are not at least n elements allocated    
+    void GrowTo(::UInt32 n)
+    {
+        if (n > m_nAllocated) {
+            if (n < 2 * m_nAllocated) {
+                n = 2 * m_nAllocated;
+            }
+            if (n < 20) {
+                n = 20;
+            }
+            ForceRealloc(n);
+        }
+    }
+
+    // Converts a potentially wrapped list (if you have moved the head) into a
+    // contiguous list, and returns a pointer to the first item in the contiguous list
+    // If possible, nothing is moved. If the list is wrapped, a new buffer is allocated (easier than moving everything in a full list)
+    // This operation is always cheap if you never remove from or add to the head.
+    DrRef<t> *MakeContiguous()
+    {
+        if (m_uFirstEntry + m_nEntries > m_nAllocated) {
+            ForceRealloc(m_nEntries);
+        }
+
+        return m_prgEntries + m_uFirstEntry;
+    }
+
+    UInt32 NumEntries() const
+    {
+        return m_nEntries;
+    }
+
+    UInt32 NumAllocated() const
+    {
+        return m_nAllocated;
+    }
+    
+    DrRef<t>& EntryAt(UInt32 index)
+    {
+        LogAssert(index < m_nEntries);
+        return m_prgEntries[NormalizeEntryIndex(index)];
+    }
+
+    const t *ConstEntryAt(UInt32 index) const
+    {
+        LogAssert(index < m_nEntries);
+        return m_prgEntries[NormalizeEntryIndex(index)];
+    }
+
+    const t *EntryAt(UInt32 index) const
+    {
+        ConstEntryAt(index);
+    }
+
+    DrRef<t>& operator[](UInt32 index)
+    {
+        return EntryAt(index);
+    }
+
+    const t *operator[](UInt32 index) const
+    {
+        return ConstEntryAt(index);
+    }
+
+    // returns NULL if list is empty
+    t *Head()
+    {
+        if (m_nEntries == 0) {
+            return NULL;
+        }
+        return m_prgEntries[m_uFirstEntry];
+    }
+
+    // returns NULL if list is empty
+    const t *Head() const
+    {
+        if (m_nEntries == 0) {
+            return NULL;
+        }
+        return m_prgEntries[m_uFirstEntry];
+    }
+
+    // returns NULL if list is empty
+    t *Tail()
+    {
+        if (m_nEntries == 0) {
+            return NULL;
+        }
+        return EntryAt(m_nEntries-1);
+    }
+
+    // returns NULL if list is empty
+    const t *Tail() const
+    {
+        if (m_nEntries == 0) {
+            return NULL;
+        }
+        return EntryAt(m_nEntries-1);
+    }
+
+    // Invalidates all entry references and pointers previously returned
+    DrRef<t>& AddEntryToTail(t *pNewEntry = NULL)
+    {
+        GrowTo(m_nEntries+1);
+        DrRef<t>& newEntry = m_prgEntries[NormalizeEntryIndex(m_nEntries++)];
+        newEntry = pNewEntry;
+        return newEntry;
+    }
+
+    // Invalidates all entry references and pointers previously returned
+    DrRef<t>& AddEntryToHead(t *pNewEntry = NULL)
+    {
+        GrowTo(m_nEntries+1);
+        if (m_uFirstEntry == 0) {
+            m_uFirstEntry = m_nAllocated - 1;
+        } else {
+            m_uFirstEntry--;
+        }
+        m_nEntries++;
+        DrRef<t>& newEntry = m_prgEntries[m_uFirstEntry];
+        newEntry = pNewEntry;
+        return newEntry;
+    }
+
+    // Invalidates all entry references and pointers previously returned
+    DrRef<t>& AddEntry(t *pNewEntry = NULL)
+    {
+        return AddEntryToTail(pNewEntry);
+    }
+
+    void RemoveEntryFromTail(DrRef<t> *pValOut)
+    {
+        LogAssert(m_nEntries != 0);
+        pValOut->TransferFrom(m_prgEntries[NormalizeEntryIndex(--m_nEntries)]);
+    }
+
+    void RemoveEntryFromHead(DrRef<t> *pValOut)
+    {
+        LogAssert(m_nEntries != 0);
+        pValOut->TransferFrom(m_prgEntries[m_uFirstEntry++]);
+        if (m_uFirstEntry >= m_nAllocated) {
+            m_uFirstEntry = 0;
+        }
+        m_nEntries--;
+    }
+
+
+    void Clear()
+    {
+        ::UInt32 uIndex = m_uFirstEntry;
+        for (::UInt32 i = 0; i < m_nEntries; i++) {
+            m_prgEntries[uIndex++] = NULL;
+            if (uIndex >= m_nAllocated) {
+                uIndex = 0;
+            }
+        }
+        m_nEntries = 0;
+        m_uFirstEntry = 0; // might as well reset to the beginning of the array
+    }
+
+    void AddNullEntriesToTail(UInt32 numNulls)
+    {
+        if (numNulls != 0)
+        {
+            // Depends on the fact that all unused entries are NULL
+            LogAssert(m_nEntries + numNulls > m_nEntries);   // Check for overflow
+            GrowTo(m_nEntries + numNulls);
+            m_nEntries += numNulls;
+        }
+    }
+
+    typedef int (__cdecl *PDRREF_COMPARE_FUNCTION)(void *context, t *p1, t *p2);
+
+    typedef struct {
+        PDRREF_COMPARE_FUNCTION compare;
+        void *context;
+    } DrRefSortContext;
+
+    static int __cdecl InternalDrRefCompare(void *context, const void *p1, const void *p2)
+    {
+        DrRefSortContext *pSortContext = (DrRefSortContext *)context;
+        return (*pSortContext->compare)(pSortContext->context, *(DrRef<t> *)p1, *(DrRef<t> *)p2);
+    }
+    
+    // performs a quicksort on the list, with a user-provided compare function
+    // Invalidates all entry references and pointers previously returned
+    void Sort(PDRREF_COMPARE_FUNCTION compare, void * context = NULL)
+    {
+        if (m_nEntries > 1) {
+            DrRef<t> *pFirst = MakeContiguous();
+            DrRefSortContext ctx;
+            ctx.compare = compare;
+            ctx.context = context;
+            qsort_s(pFirst, (size_t)m_nEntries, sizeof(*pFirst), InternalDrRefCompare, &ctx);
+        }
+    }
+
+protected:
+    UInt32 NormalizeEntryIndex(UInt32 index) const
+    {
+        LogAssert(index < m_nAllocated);
+        if (m_uFirstEntry != 0) {
+            index += m_uFirstEntry;
+            if (index >= m_nAllocated) {
+                index -= m_nAllocated;
+            }
+        }
+        return index;
+    }
+
+protected:    
+    UInt32 m_uFirstEntry;             // Normally 0, the index of the first entry (for circular buffers)
+    
+private:
+    UInt32 m_nEntries;
+    UInt32 m_nAllocated;
+    DrRef<t> *m_prgEntries;
+};
+
+#endif  //end if not defined __DRYADREFCOUNTER_H__

DryadVertex/VertexHost/system/classlib/include/DrTags.h

@@ -0,0 +1,410 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+// Tags used for BeginTag/EndTag
+
+// The must *only* be DEFINE_DRTAG directives in this file
+
+DEFINE_DRTAG(DrTag_InvalidTag, 0, "InvalidTag")   // An invalid tag value
+
+
+// DRM Tags
+DEFINE_DRTAG(Drm_CreateStream, 1, "CreateStream")
+DEFINE_DRTAG(Drm_CreateStreamResponse, 2, "CreateStreamResponse")
+DEFINE_DRTAG(Drm_AppendStream, 3, "AppendStream")
+DEFINE_DRTAG(Drm_AppendStreamResponse, 4, "AppendStreamResponse")
+DEFINE_DRTAG(Drm_EnumDirectory, 5, "EnumDirectory")
+DEFINE_DRTAG(Drm_EnumDirectoryResponse, 6, "EnumDirectoryResponse")
+DEFINE_DRTAG(Drm_GetVolumeInfo, 7, "GetVolumeInfo")
+DEFINE_DRTAG(Drm_GetVolumeInfoResponse, 8, "GetVolumeInfoResponse")
+DEFINE_DRTAG(Drm_DeleteStream, 9, "DeleteStream")
+DEFINE_DRTAG(Drm_DeleteStreamResponse, 10, "DeleteStreamResponse")
+DEFINE_DRTAG(DrTag_AppendExtentRequest, 11, "AppendExtentRequest")
+DEFINE_DRTAG(DrTag_AppendExtentResponse, 12, "AppendExtentResponse")
+DEFINE_DRTAG(Drm_ReadStream, 13, "ReadStream")
+DEFINE_DRTAG(Drm_ReadStreamResponse, 14, "ReadStreamResponse")
+DEFINE_DRTAG(Drm_UpdateExtentMetadataEvent, 15, "UpdateExtentMetadataEvent")
+DEFINE_DRTAG(Drm_UpdateExtentInstanceMetadataEvent, 16, "UpdateExtentInstanceMetadataEvent")
+DEFINE_DRTAG(DrTag_ExtentNodeStatusEvent, 17, "ExtentNodeStatusEvent")
+DEFINE_DRTAG(DrTag_DrmGarbageCollectEvent, 19, "DrmGarbageCollect")
+DEFINE_DRTAG(DrTag_DrmSyncEnEvent, 20, "DrmSyncEN")
+DEFINE_DRTAG(Drm_RenameStream, 21, "RenameStream")
+DEFINE_DRTAG(Drm_RenameStreamResponse, 22, "RenameStreamResponse")
+DEFINE_DRTAG(Drm_NewVolumeEvent, 23, "NewVolumeEvent")
+DEFINE_DRTAG(DrTag_AppendExtentRangeRequest, 24, "AppendExtentRangeRequest")
+DEFINE_DRTAG(DrTag_AppendExtentRangeResponse, 25, "AppendExtentRangeResponse")
+DEFINE_DRTAG(DrTag_DrmSealExtentRequest, 26, "DrmSealExtentRequest")
+DEFINE_DRTAG(DrTag_DrmSealExtentResponse, 27, "DrmSealExtentResponse")
+DEFINE_DRTAG(DrTag_DrmSetStreamProperties, 28, "SetStreamProperties")
+DEFINE_DRTAG(DrTag_DrmSetStreamPropertiesResponse, 29, "SetStreamPropertiesResponse")
+// the following four tags are for synchronized version of metadata update
+DEFINE_DRTAG(Drm_UpdateExtentMetadataRequest, 30, "UpdateExtentMetadataRequest")        
+DEFINE_DRTAG(Drm_UpdateExtentMetadataResponse, 31, "UpdateExtentMetadataResponse")
+DEFINE_DRTAG(Drm_UpdateExtentInstanceMetadataRequest, 32, "UpdateExtentInstanceMetadataRequest")
+DEFINE_DRTAG(Drm_UpdateExtentInstanceMetadataResponse, 33, "UpdateExtentInstanceMetadataResponse")
+DEFINE_DRTAG(DrTag_AppendExtentRangeRequest2, 34, "AppendExtentRangeRequest2")
+DEFINE_DRTAG(DrTag_AppendExtentRangeResponse2, 35, "AppendExtentRangeResponse2")
+DEFINE_DRTAG(Drm_SubstituteStreamRequest, 36, "SubstituteStreamRequest")
+DEFINE_DRTAG(Drm_SubstituteStreamResponse, 37, "SubstituteStreamResponse")
+DEFINE_DRTAG(DrTag_ExtentRangeList, 38, "ExtentRangeList")
+DEFINE_DRTAG(DrTag_ExtentRange, 39, "ExtentRange")
+DEFINE_DRTAG(Drm_ExtentList, 51, "ExtentList") // array of CEIDs
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// REDDOG DRM extensions (shared with cosmos)
+DEFINE_DRTAG(Drm_MultiModifyStreamEntry, 61, "MultiModifyStreamEntry") // a single stream mod entry within a list
+DEFINE_DRTAG(Drm_MultiModifyStream, 62, "MultiModifyStream")
+DEFINE_DRTAG(Drm_MultiModifyStreamResponse, 63, "MultiModifyStreamResponse")
+DEFINE_DRTAG(Drm_MultiModifyExtentRange, 64, "MultiModifyExtentRange") // a single concatenate extent range within a list
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// EN Tags
+// First set are for requests from csm/client
+DEFINE_DRTAG(DrTag_EnAppendExtentRequest, 100, "EnAppendExtentRequest")
+DEFINE_DRTAG(DrTag_EnSealExtentRequest, 101, "EnSealExtentRequest")
+DEFINE_DRTAG(DrTag_EnReadExtentRequest, 102, "EnReadExtentRequest")
+DEFINE_DRTAG(DrTag_EnCreateExtentRequest, 103, "EnCreateExtentRequest")
+DEFINE_DRTAG(DrTag_EnStartSealingExtentInstanceRequest, 104, "EnStartSealingExtentInstanceRequest")
+DEFINE_DRTAG(DrTag_EnSyncRequest, 105, "EnSyncRequest")
+DEFINE_DRTAG(DrTag_EnReplicateExtentRequest, 106, "EnReplicateExtentRequest")
+DEFINE_DRTAG(DrTag_EnRecoverExtentRequest, 107, "EnRecoverExtentRequest")
+DEFINE_DRTAG(DrTag_EnCmdRequest, 108, "EnCmdRequest" )
+DEFINE_DRTAG(DrTag_EnReadaheadInfo, 109, "EnReadaheadInfo" )
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// These are the matching response tags for replies to above requests
+DEFINE_DRTAG(DrTag_EnAppendExtentResponse, 200, "EnAppendExtentResponse")
+DEFINE_DRTAG(DrTag_EnSealExtentResponse, 201, "EnSealExtentResponse")
+DEFINE_DRTAG(DrTag_EnReadExtentResponse, 202, "EnReadExtentResponse")
+DEFINE_DRTAG(DrTag_EnCreateExtentResponse, 203, "EnCreateExtentResponse")
+DEFINE_DRTAG(DrTag_EnStartSealingExtentInstanceResponse, 204, "EnStartSealingExtentInstanceResponse")
+DEFINE_DRTAG(DrTag_EnSyncResponse, 205, "EnSyncResponse")
+DEFINE_DRTAG(DrTag_EnReplicateExtentResponse, 206, "EnReplicateExtentResponse")
+DEFINE_DRTAG(DrTag_EnRecoverExtentResponse, 207, "EnRecoverExtentResponse")
+DEFINE_DRTAG(DrTag_EnCmdResponse, 208, "EnCmdResponse" )
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// Second set are for requests between ENs to handle replication
+DEFINE_DRTAG(DrTag_EnReplicateCreateExtentRequest, 300, "EnReplicateCreateExtentRequest")
+DEFINE_DRTAG(DrTag_EnReplicateAppendExtentRequest, 301, "EnReplicateAppendExtentRequest")
+//DEFINE_DRTAG(DrTag_EnReplicateExtentLengthRequest, 302, "EnReplicateExtentLengthRequest")
+DEFINE_DRTAG(DrTag_EnReplicateSealRequest, 303, "EnReplicateSealRequest")
+
+// And these are the matching responses to the above requests
+DEFINE_DRTAG(DrTag_EnReplicateCreateExtentResponse, 400, "EnReplicateCreateExtentResponse")
+DEFINE_DRTAG(DrTag_EnReplicateAppendExtentResponse, 401, "EnReplicateAppendExtentResponse")
+DEFINE_DRTAG(DrTag_EnReplicateExtentLengthResponse, 402, "EnReplicateExtentLengthResponse")
+DEFINE_DRTAG(DrTag_EnReplicateSealResponse, 403, "EnReplicateSealResponse")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// Generic messages
+DEFINE_DRTAG(DrTag_DrMalformedMessage, 500, "DrMalformedMessage")
+DEFINE_DRTAG(DrTag_DrErrorResponseMessage, 501, "DrErrorResponseMessage")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// Name Resolution tags
+DEFINE_DRTAG(DrTag_DrResolveNameRequest, 600, "DrResolveNameRequest")
+DEFINE_DRTAG(DrTag_DrResolveNameResponse, 601, "DrResolveNameResponse")
+DEFINE_DRTAG(DrTag_DrGetNameResolutionMapRequest, 602, "DrGetNameResolutionMapRequest")
+DEFINE_DRTAG(DrTag_DrGetNameResolutionMapResponse, 603, "DrGetNameResolutionMapResponse")
+DEFINE_DRTAG(DrTag_DrRegisterNamesRequest, 604, "DrRegisterNamesRequest")
+DEFINE_DRTAG(DrTag_DrRegisterNamesResponse, 605, "DrRegisterNamesResponse")
+DEFINE_DRTAG(DrTag_DrUpdateNameResolution, 606, "DrUpdateNameResolution")
+DEFINE_DRTAG(DrTag_DrUpdateNameResolutionResponse, 607, "DrUpdateNameResolutionResponse")
+DEFINE_DRTAG(DrTag_DrHostNameList, 608, "DrHostNameList")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// ProcessNode tags
+DEFINE_DRTAG(DrTag_PnCreateProcessRequest, 			700, 	"PnCreateProcessRequest")
+DEFINE_DRTAG(DrTag_PnCreateProcessResponse, 		701, 	"PnCreateProcessResponse")
+DEFINE_DRTAG(DrTag_PnGetProcessStatusRequest, 		702, 	"PnGetProcessStatusRequest")
+DEFINE_DRTAG(DrTag_PnGetProcessStatusResponse, 		703, 	"PnGetProcessStatusResponse")
+DEFINE_DRTAG(DrTag_PnTerminateProcessRequest, 		704, 	"PnTerminateProcessRequest")
+DEFINE_DRTAG(DrTag_PnTerminateProcessResponse, 		705, 	"PnTerminateProcessResponse")
+DEFINE_DRTAG(DrTag_PnSetProcessPropertyRequest, 	706, 	"PnSetProcessPropertyRequest")
+DEFINE_DRTAG(DrTag_PnSetProcessPropertyResponse, 	707, 	"PnSetProcessPropertyResponse")
+DEFINE_DRTAG(DrTag_PnGetProcessPropertyRequest, 	708, 	"PnGetProcessPropertyRequest")
+DEFINE_DRTAG(DrTag_PnGetProcessPropertyResponse, 	709, 	"PnGetProcessPropertyResponse")
+DEFINE_DRTAG(DrTag_PnEnumerateProcessesRequest, 	710, 	"PnEnumerateProcessesRequest")
+DEFINE_DRTAG(DrTag_PnEnumerateProcessesResponse, 	711, 	"PnEnumerateProcessesResponse")
+DEFINE_DRTAG(DrTag_PnUserData, 						712, 	"PnUserData")
+DEFINE_DRTAG(DrTag_PnProcessStat, 					713,	"PnProcessStat")
+DEFINE_DRTAG(DrTag_PnJobStat, 					    714,	"PnJobStat")
+DEFINE_DRTAG(DrTag_PnGetJobStatRequest,             715,    "PnGetJobStatRequest")
+DEFINE_DRTAG(DrTag_PnGetJobStatResponse,            716,    "PnGetJobStatResponse")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// Pipe protocol tags
+DEFINE_DRTAG(DrTag_DrRendezvousConnect, 800, "DrRendezvousConnect")
+DEFINE_DRTAG(DrTag_DrRendezvousConnected, 801, "DrRendezvousConnected")
+DEFINE_DRTAG(DrTag_DrRendezvousConnectedAck, 802, "DrRendezvousConnectedAck")
+DEFINE_DRTAG(DrTag_DrRendezvousConnectFailed, 803, "DrRendezvousConnectFailed")
+DEFINE_DRTAG(DrTag_DrRendezvousRedirect, 804, "DrRendezvousRedirect")
+DEFINE_DRTAG(DrTag_DrRendezvousWait, 805, "DrRendezvousWait")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// Execution monitoring tags
+DEFINE_DRTAG(DrTag_JmJobExecutionStatistics, 900, "JmJobExecutionStatistics")
+DEFINE_DRTAG(DrTag_JmVertexClassExecutionStatistics, 901, "JmVertexClassExecutionStatistics")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// Cache Manager request tags
+DEFINE_DRTAG(DrTag_CmSeedRequest,          1000,     "CmSeedRequest")
+DEFINE_DRTAG(DrTag_CmQueryFileRequest,     1001,     "CmQueryFileRequest")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// Cache Manager response tags
+DEFINE_DRTAG(DrTag_CmSeedResponse,         1010,     "CmSeedResponse")
+DEFINE_DRTAG(DrTag_CmQueryFileResponse,    1011,     "CmQueryFileResponse")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// Cache Manager event tags
+DEFINE_DRTAG(DrTag_CmGossipEvent,          1020,     "CmGossipEvent")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// Dryad Task Scheduler tags
+DEFINE_DRTAG(DrTag_ScheduleTaskRequest,    1100,     "ScheduleTaskRequest")
+DEFINE_DRTAG(DrTag_ScheduleTaskResponse,   1101,     "ScheduleTaskResponse")
+DEFINE_DRTAG(DrTag_TaskJournalStart,       1102,     "TaskJournalStart")
+DEFINE_DRTAG(DrTag_TaskJournalEnd,         1103,     "TaskJournalEnd")
+DEFINE_DRTAG(DrTag_GetTaskInfoRequest,     1104,     "GetTaskInfoRequest")
+DEFINE_DRTAG(DrTag_GetTaskInfoResponse,    1105,     "GetTaskInfoResponse")
+DEFINE_DRTAG(DrTag_GetTaskJournalRequest,  1106,     "GetTaskJournalRequest")
+DEFINE_DRTAG(DrTag_GetTaskJournalResponse, 1107,     "GetTaskJournalResponse")
+DEFINE_DRTAG(DrTag_DeleteTaskRequest,      1108,     "DeleteTaskRequest")
+DEFINE_DRTAG(DrTag_DeleteTaskResponse,     1109,     "DeleteTaskResponse")
+DEFINE_DRTAG(DrTag_EnumerateTasksRequest,  1110,     "EnumerateTasksRequest")
+DEFINE_DRTAG(DrTag_EnumerateTasksResponse, 1111,     "EnumerateTasksResponse")
+DEFINE_DRTAG(DrTag_EnumeratedTask,         1112,     "EnumeratedTask")
+DEFINE_DRTAG(DrTag_TaskJournalEntry,       1113,     "TaskJournalEntry")
+DEFINE_DRTAG(DrTag_EnumerateSchedulerLogRequest, 1114, "EnumerateSchedulerLogRequest")
+DEFINE_DRTAG(DrTag_EnumerateSchedulerLogResponse,1115, "EnumerateSchedulerLogResponse")
+DEFINE_DRTAG(DrTag_SetSchedulerPropertiesRequest,1116, "SetSchedulerPropertiesRequest")
+DEFINE_DRTAG(DrTag_SetSchedulerPropertiesResponse,1117, "SetSchedulerPropertiesResponse")
+DEFINE_DRTAG(DrTag_SetTaskPropertiesRequest,1118,    "SetTaskPropertiesRequest")
+DEFINE_DRTAG(DrTag_SetTaskPropertiesResponse,1119,   "SetTaskPropertiesResponse")
+DEFINE_DRTAG(DrTag_AppendJobDataRequest,1120,        "AppendJobDataRequest")
+DEFINE_DRTAG(DrTag_AppendJobDataResponse,1121,       "AppendJobDataResponse")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// jforbes: Tags between 1122-1139 reserved for task scheduler
+
+// Dryad Task scheduler log
+DEFINE_DRTAG(DrTag_TaskLogTaskSubmitted,   1140,     "TaskSubmitted")
+DEFINE_DRTAG(DrTag_TaskLogTaskDeleted,     1141,     "TaskDeleted")
+DEFINE_DRTAG(DrTag_TaskLogTaskCompleted,   1142,     "TaskCompleted")
+DEFINE_DRTAG(DrTag_TaskLogJobStarted,      1143,     "JobStarted")
+DEFINE_DRTAG(DrTag_TaskLogJobEnded,        1144,     "JobEnded")
+DEFINE_DRTAG(DrTag_TaskLogGainedMasterStatus, 1145,  "GainedMasterStatus")
+DEFINE_DRTAG(DrTag_TaskLogSetSchedulerProperties, 1146,  "SetSchedulerProperties")
+DEFINE_DRTAG(DrTag_TaskLogSetTaskProperties, 1147,   "SetTaskProperties")
+DEFINE_DRTAG(DrTag_TaskLogJobData,         1148,   "JobData")
+
+// This is not actually a log entry, but the custom property bag part of a TaskLogJobData entry
+DEFINE_DRTAG(DrTag_TaskLogJobDataContents, 1149,   "JobDataContents")
+
+// This is a tag that will be used within a DrTag_TaskLogJobDataContents
+// It describes data that has been uploaded Dryad URL, typically by Dryad, that should be
+// preserved.  Sub-properties will include a Dryad URI and a flag indicating whether it is
+// temporary output (e.g. stdout.txt for the job) that should be cleaned up by the scheduler
+// at some later point.
+DEFINE_DRTAG(DrTag_JobOutput, 1150, "JobOutput")
+
+// This entry indicates the task scheduler appended its state to the permanent storage stream
+// at a particular sequence number.
+DEFINE_DRTAG(DrTag_TaskLogArchive, 1151,   "Archive")
+
+// This entry indicates an error or warning related to the scheduler
+// This may include being unable to start a job
+DEFINE_DRTAG(DrTag_TaskLogError, 1152,   "SchedulerError")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// jforbes: Tags between 1153-1199 reserved for task scheduler
+
+// Dryad Async RPC Server Protocol tags
+DEFINE_DRTAG(DrTag_RpcTunnelTransportRequest, 1200, "RpcTunnelTransportRequest")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportRequestHeader, 1201, "RpcTunnelTransportRequestHeader")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportRequestBody, 1202, "RpcTunnelTransportRequestBody")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportResponse, 1203, "RpcTunnelTransportResponse")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportResponseHeader, 1204, "RpcTunnelTransportResponseHeader")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportResponseBody, 1205, "RpcTunnelTransportResponseBody")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportSessionOpenRequest, 1206, "RpcTunnelTransportSessionOpenRequest")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportSessionOpenResponse, 1207, "RpcTunnelTransportSessionOpenResponse")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportSessionCloseRequest, 1208, "RpcTunnelTransportSessionCloseRequest")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportSessionCloseResponse, 1209, "RpcTunnelTransportSessionCloseResponse")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportSessionEnqueueRequest, 1210, "RpcTunnelTransportSessionEnqueueRequest")
+DEFINE_DRTAG(DrTag_RpcRequest, 1211, "RpcRequest")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportSessionEnqueueResponse, 1212, "RpcTunnelTransportSessionEnqueueResponse")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportSessionPollRequest, 1213, "RpcTunnelTransportSessionPollRequest")
+DEFINE_DRTAG(DrTag_RpcTunnelTransportSessionPollResponse, 1214, "RpcTunnelTransportSessionPollResponse")
+DEFINE_DRTAG(DrTag_RpcResponse, 1215, "RpcResponse")
+DEFINE_DRTAG(DrTag_RpcRequestHeader, 1216, "RpcRequestHeader")
+DEFINE_DRTAG(DrTag_RpcGenerateEmptyResultRequest, 1217, "RpcGenerateEmptyResultRequest")
+DEFINE_DRTAG(DrTag_RpcResponseHeader, 1218, "RpcResponseHeader")
+DEFINE_DRTAG(DrTag_RpcSessionKey, 1219, "RpcSessionKey")
+DEFINE_DRTAG(DrTag_RpcProxySendMessageRequest, 1220, "RpcProxySendMessageRequest")
+DEFINE_DRTAG(DrTag_RpcProxySendMessageResponse, 1221, "RpcProxySendMessageResponse")
+DEFINE_DRTAG(DrTag_DrWrappedProtocolMessage, 1222, "DrWrappedProtocolMessage")
+DEFINE_DRTAG(DrTag_RpcRequestPacketHeader, 1223, "RpcRequestPacketHeader")
+DEFINE_DRTAG(DrTag_RpcResponsePacketHeader, 1224, "RpcResponsePacketHeader")
+DEFINE_DRTAG(DrTag_CompoundOpaqueKey, 1225, "CompundOpaqueKey")
+DEFINE_DRTAG(DrTag_RpcRequestPacket, 1226, "RpcRequestPacket")
+DEFINE_DRTAG(DrTag_RpcResponsePacket, 1227, "RpcResponsePacket")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+//
+// cosmos resource selector Tags
+//
+DEFINE_DRTAG(DrTag_GetResourceListRequest, 1300, "DrTag_GetResourceListRequest")
+DEFINE_DRTAG(DrTag_GetResourceListResponse, 1301, "DrTag_GetResourceListResponse")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// base protocol tags
+DEFINE_DRTAG(DrTag_DrMessage, 9000, "DrMessage")
+DEFINE_DRTAG(DrTag_DrClientMessageHeader, 9001, "DrClientMessageHeader")
+DEFINE_DRTAG(DrTag_DrServerMessageHeader, 9002, "DrServerMessageHeader")
+DEFINE_DRTAG(DrTag_DrServiceDescriptor, 9003, "DrServiceDescriptor")
+DEFINE_DRTAG(DrTag_DrExtentNodeInfo, 9004, "DrExtentNodeInfo")
+DEFINE_DRTAG(DrTag_DrStreamInfo, 9005, "DrStreamInfo")
+DEFINE_DRTAG(DrTag_DrExtentInfo, 9006, "DrExtentInfo")
+DEFINE_DRTAG(DrTag_DrExtentInstanceInfo, 9007, "DrExtentInstanceInfo")
+DEFINE_DRTAG(DrTag_DrExtentInstanceMetadata, 9008, "DrExtentInstanceMetadata")
+DEFINE_DRTAG(DrTag_DrNameResolutionMapEntry, 9009, "DrNameResolutionMapEntry")
+DEFINE_DRTAG(DrTag_DrEnSyncDirective, 9010, "DrEnSyncDirective")
+DEFINE_DRTAG(DrTag_DrHostAndPort, 9011, "DrHostAndPort")
+DEFINE_DRTAG(DrTag_DrSimpleProcessFile, 9012, "DrSimpleProcessFile")
+DEFINE_DRTAG(DrTag_DrPingRequest, 9013, "DrPingRequest")
+DEFINE_DRTAG(DrTag_DrPingResponse, 9014, "DrPingResponse")
+DEFINE_DRTAG(DrTag_DrProcessConstraints, 9015, "DrProcessConstraints")
+DEFINE_DRTAG(DrTag_DrNodeErrorInfo, 9016, "DrNodeErrorInfo")
+DEFINE_DRTAG(DrTag_DrProcessInfo, 9017, "DrProcessInfo")
+DEFINE_DRTAG(DrTag_DrProcessPropertyInfo, 9018, "DrProcessPropertyInfo")
+DEFINE_DRTAG(DrTag_DrUserDescriptor, 9019, "DrUserDescriptor")
+DEFINE_DRTAG(DrTag_DrUserTicket, 9020, "DrUserTicket")
+DEFINE_DRTAG(DrTag_DrJobDescriptor, 9021, "DrJobDescriptor")
+DEFINE_DRTAG(DrTag_DrJobTicket, 9022, "DrJobTicket")
+DEFINE_DRTAG(DrTag_DrProcessDescriptor, 9023, "DrProcessDescriptor")
+DEFINE_DRTAG(DrTag_DrProcessTicket, 9024, "DrProcessTicket")
+DEFINE_DRTAG(DrTag_DrParentProcess, 9025, "DrParentProcess")
+DEFINE_DRTAG(DrTag_DrRootProcess, 9026, "DrRootProcess")
+DEFINE_DRTAG(DrTag_DrExtentInstanceMetadataSet, 9027, "DrExtentInstanceMetadataSet")
+DEFINE_DRTAG(DrTag_DrDetailedError, 9028, "DrDetailedError")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// BUGBUG: These were added by XStore and should not be here, they will collide with Dryad integrations.
+// TODO: consider breaking XStream interop to move them to the right place!
+
+DEFINE_DRTAG(DrTag_ParamList, 9029, "ParamList" )
+DEFINE_DRTAG(DrTag_ValueList, 9030, "ValueList" )
+DEFINE_DRTAG(DrTag_ParamValueList, 9031, "ParamValueList")
+DEFINE_DRTAG(DrTag_DrStreamPolicyInfo, 9032, "StreamPolicyInfo")
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+// NOTE NOTE NOTE
+//
+// 10000-10999 are reserved for Dryad
+
+// WARNING: DO NOT INSERT REDDOG TAGS HERE!
+
+
+// NOTE NOTE NOTE
+//
+// 11000-11999 are reserved for xstore
+//
+//
+// XStream Tags
+//
+// WARNING: DO NOT INSERT REDDOG TAGS BEFORE THIS POINT
+// DO NOT REORDER, DELETE, OR CHANGE THE VALUE OF ANY ENTRY IN THIS FILE
+DEFINE_DRTAG(XsTag_ClientMessageHeader, 11000, "ClientMessageHeader")
+DEFINE_DRTAG(XsTag_ServerMessageHeader, 11001, "ServerMessageHeader")
+DEFINE_DRTAG(XsTag_XStoreMessage, 11002, "XStoreMessage")
+DEFINE_DRTAG(XsTag_XStoreMessageBody, 11003, "XStoreMessageBody")
+DEFINE_DRTAG(DrTag_DrFailureInjectionRequest, 11004, "DrFailureInjectionRequest")
+DEFINE_DRTAG(DrTag_DrFailureInjectionResponse, 11005, "DrFailureInjectionResponse")
+DEFINE_DRTAG(DrTag_DrEnumFileDirRequest, 11006, "DrEnumFileDirRequest")
+DEFINE_DRTAG(DrTag_DrEnumFileDirResponse, 11007, "DrEnumFileDirResponse")
+DEFINE_DRTAG(DrTag_DrReadFileRequest, 11008, "DrReadFileRequest")
+DEFINE_DRTAG(DrTag_DrReadFileResponse, 11009, "DrReadFileResponse")
+DEFINE_DRTAG(DrTag_DrWriteFileRequest, 11010, "DrWriteFileRequest")
+DEFINE_DRTAG(DrTag_DrWriteFileResponse, 11011, "DrWriteFileResponse")
+DEFINE_DRTAG(DrTag_DrDirectoryEntry, 11012, "DrDirectoryEntry")
+DEFINE_DRTAG(DrTag_EnAppendBlockInfo, 11013, "EnAppendBlockInfo" )
+DEFINE_DRTAG(Drm_SetStreamPolicyRequest, 11014, "SetStreamPolicyRequest")
+DEFINE_DRTAG(Drm_SetStreamPolicyResponse, 11015, "SetStreamPolicyResponse")
+
+// SAMMCK: renamed from QueryStreamPolicyRequest to match the API
+DEFINE_DRTAG(Drm_GetStreamPoliciesRequest, 11016, "GetStreamPolociesRequest")
+
+// SAMMCK: renamed from QueryStreamPolicyResponse to match the API
+DEFINE_DRTAG(Drm_GetStreamPoliciesResponse, 11017, "QueryStreamPolicyResponse")
+DEFINE_DRTAG(Drm_PolicyList, 11018, "PolicyList") 
+DEFINE_DRTAG(Drm_ENConfigChangeEvent, 11019, "ENConfigChangeEvent")
+DEFINE_DRTAG(DrTag_DrCommandLineParams, 11020, "DrCommandLineParams" )
+DEFINE_DRTAG(DrTag_DrCommandLineResults, 11021, "DrCommandLineResults" )
+DEFINE_DRTAG(DrTag_XcPsScheduleProcessRequest, 11022, "XcPsScheduleProcessRequest")
+DEFINE_DRTAG(DrTag_XcPsScheduleProcessResponse, 11023, "XcPsScheduleProcessResponse")
+DEFINE_DRTAG(DrTag_DrExecuteCommandLineRequest, 11024, "DrExecuteCommandLineRequest" )
+DEFINE_DRTAG(DrTag_DrExecuteCommandLineResponse, 11025, "DrExecuteCommandLineResponse" )
+DEFINE_DRTAG(DrTag_DrReferencingStreamInfo, 11033, "ReferencingStreamInfo")
+DEFINE_DRTAG(Drm_GetExtentInfoRequest, 11034, "GetExtentInfoRequest")
+DEFINE_DRTAG(Drm_GetExtentInfoResponse, 11035, "GetExtentInfoResponse")
+DEFINE_DRTAG(DrTag_EmbeddedDrmCommand, 11036, "EmbeddedDrmCommand")
+DEFINE_DRTAG(DrTag_EmbeddedRslRequest, 11037, "EmbeddedRslRequest")
+DEFINE_DRTAG(Drm_GetLazyReplicationInfo, 11038, "GetLazyReplicationInfo")
+DEFINE_DRTAG(Drm_GetLazyReplicationInfoResponse, 11039, "GetLazyReplicationInfoResponse")
+DEFINE_DRTAG(DrEnTag_RequestLatenciesDeprecated, 11040, "RequestLatenciesDeprecated")
+
+// The following two tags are used for extent 
+// recovery after DRM emergency rollback.
+DEFINE_DRTAG(DrTag_RecoverAndAppendExtentsRequest, 11041, "RecoverAndAppendExtentsRequest")
+DEFINE_DRTAG(DrTag_RecoverAndAppendExtentsResponse, 11042, "RecoverAndAppendExtentsResponse")
+
+// The following two tags are used to enumerate unsealable
+// extents following a rollback/recovery operation.
+DEFINE_DRTAG(DrTag_EnumerateUnsealableExtentsRequest, 11043, "EnumerateUnsealableExtentsRequest")
+DEFINE_DRTAG(DrTag_EnumerateUnsealableExtentsResponse, 11044, "EnumerateUnsealableExtentsResponse")
+
+// The following two tags are deprecated and may be reused 
+// once all known nodes have been upgraded.
+DEFINE_DRTAG(DrTag_ProtocolMessageTimestampsDeprecated, 11045, "ProtocolMessageTimestampsDeprecated")
+DEFINE_DRTAG(DrEnTag_RequestLatencies, 11046, "RequestLatencies")
+
+// ADD ALL XSTREAM/XSTORE/REDDOG TAGS ABOVE THIS POINT!
+

DryadVertex/VertexHost/system/classlib/include/DrTagsDef.h

@@ -0,0 +1,33 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#ifdef DEFINE_DRTAG
+#undef DEFINE_DRTAG
+#endif
+
+#define DEFINE_DRTAG(var, value, tagName) \
+    static const UInt16 var = value;
+
+#include "DrTags.h"
+
+#undef DEFINE_DRTAG
+

DryadVertex/VertexHost/system/classlib/include/DrTypes.h

@@ -0,0 +1,118 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "basic_types.h"
+
+typedef HRESULT DrError;
+
+typedef size_t Size_t;
+static const Size_t Max_Size_t = (Size_t)((Size_t)0 - (Size_t)1);
+
+//JCnamespace apsdk
+//JC{
+
+__forceinline DrError DrGetLastError()
+{
+    return HRESULT_FROM_WIN32( GetLastError() );
+}
+
+__forceinline DrError DrGetLastErrorEnforceFail()
+{
+    DrError hr = HRESULT_FROM_WIN32(GetLastError());
+    return FAILED(hr) ? hr : E_UNEXPECTED;
+}
+
+__forceinline DrError DrErrorFromWin32(DWORD err)
+{
+    return HRESULT_FROM_WIN32(err);
+}
+
+///////////////////////////////////
+#pragma warning(push)
+#pragma warning (disable: 4201) // nonstandard extension used : nameless struct/union
+
+/* 
+   A cosmos timestamp is defined as the number of 100-nanosecond intervals that have elapsed
+   since 12:00 A.M. January 1, 1601 (UTC). It is the representation of choice whenever an
+   absolute date/time must be used.
+*/
+typedef unsigned __int64 DrTimeStamp;
+
+static const DrTimeStamp DrTimeStamp_Never = (DrTimeStamp)_UI64_MAX;
+static const DrTimeStamp DrTimeStamp_LongAgo = (DrTimeStamp)0;
+
+/*
+   A cosmos elapsed time is defined as the number of 100-nanosecond intervals between two
+   points in time. It may be negative. It is what you get when you subtract two DrTimestamp values, and is
+   the representation of choice whenever a time interval needs to be represented persistently
+   or in a network protocol.
+*/
+typedef __int64 DrTimeInterval;
+
+static const DrTimeInterval DrTimeInterval_Infinite = (DrTimeInterval)_I64_MAX;
+static const DrTimeInterval DrTimeInterval_NegativeInfinite = (DrTimeInterval)_I64_MIN;
+static const DrTimeInterval DrTimeInterval_Zero = (DrTimeInterval)0;
+static const DrTimeInterval DrTimeInterval_Quantum = (DrTimeInterval)1;
+static const DrTimeInterval DrTimeInterval_100ns = DrTimeInterval_Quantum;
+static const DrTimeInterval DrTimeInterval_Microsecond = DrTimeInterval_100ns * 10;
+static const DrTimeInterval DrTimeInterval_Millisecond = DrTimeInterval_Microsecond * 1000;
+static const DrTimeInterval DrTimeInterval_Second = DrTimeInterval_Millisecond * 1000;
+static const DrTimeInterval DrTimeInterval_Minute = DrTimeInterval_Second * 60;
+static const DrTimeInterval DrTimeInterval_Hour = DrTimeInterval_Minute * 60;
+static const DrTimeInterval DrTimeInterval_Day = DrTimeInterval_Hour * 24;
+static const DrTimeInterval DrTimeInterval_Week = DrTimeInterval_Day * 7;
+
+// A DrTimeInterval_Year is defined as 52 weeks. It is for convenience, not for computing exact years.
+static const DrTimeInterval DrTimeInterval_Year = DrTimeInterval_Week * 52;
+
+/* An IPV4 IP address in host byte order */
+typedef UInt32 DrIpAddress;
+
+static const DrIpAddress DrAnyIpAddress = 0;
+static const DrIpAddress DrInvalidIpAddress = 0;
+static const DrIpAddress DrLocalIpAddress = 0x7F000001;  // 127.0.0.1
+
+
+/* An IP port number in host byte order */
+typedef UInt16 DrPortNumber;
+
+static const DrPortNumber DrInvalidPortNumber = 0xFFFF;
+static const DrPortNumber DrAnyPortNumber = 0;
+
+#pragma warning(pop)
+
+//JC} // namespace apsdk
+#ifdef USING_APSDK_NAMESPACE
+using namespace apsdk;
+#endif
+
+//
+// disabling a couple of warnings that show up with /Wall and are pretty much useless -- they come mostly from macros
+//
+#pragma warning (disable: 4514) // unreferenced inline function has been removed
+#pragma warning (disable: 4820) // 'N' bytes padding added after data member 'XXX'
+#pragma warning (disable: 4265) // class has virtual functions, but destructor is not virtual
+#pragma warning (disable: 4668) // XXX is not defined as preprocessor macro, replacing with 0
+#pragma warning (disable: 4711) // function XXX selected for automatic inline expansion
+#pragma warning (disable: 4548) // malloc.h(245) & STL: expression before comma has no effect; expected expression with side-effect
+#pragma warning (disable: 4127) // conditional expression is constant
+

DryadVertex/VertexHost/system/classlib/include/DryadTags.h

@@ -0,0 +1,57 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+// Tags used for BeginTag/EndTag
+
+// There must *only* be DEFINE_DRYADTAG directives in this file
+
+DEFINE_DRYADTAG(DryadTag_InvalidTag, 0, "InvalidTag", MetaData)   // An invalid tag value
+
+DEFINE_DRYADTAG(DryadTag_HostNameList, 608, "HostNameList", MetaData)
+DEFINE_DRYADTAG(DryadTag_HostAndPort, 9011, "HostAndPort", MetaData)
+
+DEFINE_DRYADTAG(DryadTag_InputChannelDescription, 10000, "InputChannelDescription", InputChannelDescription)
+DEFINE_DRYADTAG(DryadTag_OutputChannelDescription, 10001, "OutputChannelDescription", OutputChannelDescription)
+DEFINE_DRYADTAG(DryadTag_VertexProcessStatus, 10002, "VertexProcessStatusBlock", VertexProcessStatus)
+DEFINE_DRYADTAG(DryadTag_VertexStatus, 10003, "VertexStatusBlock", VertexStatus)
+DEFINE_DRYADTAG(DryadTag_VertexCommand, 10004, "VertexCommandBlock", VertexCommandBlock)
+DEFINE_DRYADTAG(DryadTag_ItemStart, 10005, "ItemStartBlock", MetaData)
+DEFINE_DRYADTAG(DryadTag_ItemEnd, 10006, "ItemEndBlock", MetaData)
+DEFINE_DRYADTAG(DryadTag_ChannelMetaData, 10007, "ChannelMetaData", MetaData)
+DEFINE_DRYADTAG(DryadTag_VertexMetaData, 10008, "VertexMetaData", MetaData)
+DEFINE_DRYADTAG(DryadTag_ArgumentArray, 10009, "ArgumentArray", MetaData)
+DEFINE_DRYADTAG(DryadTag_VertexArray, 10010, "VertexArray", MetaData)
+DEFINE_DRYADTAG(DryadTag_VertexInfo, 10011, "VertexInfo", MetaData)
+DEFINE_DRYADTAG(DryadTag_EdgeArray, 10012, "EdgeArray", MetaData)
+DEFINE_DRYADTAG(DryadTag_EdgeInfo, 10013, "EdgeInfo", MetaData)
+DEFINE_DRYADTAG(DryadTag_GraphDescription, 10014, "GraphDescription", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSCAReturnMachine, 10015, "ReturnMachine", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSCAEnqueueProcess, 10016, "EnqueueProcess", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSCAReportFailedMachine, 10017, "ReportFailedMachine", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSCADiscardProcess, 10018, "DiscardProcess", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSClientResponse, 10019, "RSClientResponse", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSClientRootProcessRequest, 10020, "RSClientRootProcessRequest", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSClientRootProcessResponse, 10021, "RSClientRootProcessResponse", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSClientInitializeRequest, 10022, "RSClientInitializeRequest", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSClientActionRequest, 10023, "RSClientActionRequest", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSClientStatusRequest, 10024, "RSClientStatusRequest", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSClientMatch, 10025, "RSClientMatch", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSRevocation, 10026, "RSRevocation", MetaData)
+DEFINE_DRYADTAG(DryadTag_RSClientCommand, 10027, "RSClientCommand", MetaData)

DryadVertex/VertexHost/system/classlib/include/DryadTagsDef.h

@@ -0,0 +1,33 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#ifdef DEFINE_DRYADTAG
+#undef DEFINE_DRYADTAG
+#endif
+
+#define DEFINE_DRYADTAG(var, value, tagName, tagType) \
+    static const UInt16 var = value;
+
+#include "dryadtags.h"
+
+#undef DEFINE_DRYADTAG
+

DryadVertex/VertexHost/system/classlib/include/Interlocked.h

@@ -0,0 +1,150 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+#include <windows.h>
+
+// The Interlocked64 APIs are only available on Windows Server 2003
+// and later.  Even though the Interlocked64 operation are declared in
+// winbase.h, they are not in kernel32, so the application breaks at
+// runtime.
+
+// Also, AMD64 machines do not have InterlockedCompareExchange64() in
+// kernel32, since the function is an intrinsic, so it must be called
+// directly.
+//
+// This file defines Interlocked64 operations on 32 bit platforms.
+// On 64 bit platforms (AMD or IA64), the inlined functions
+// call the intrinsic functions.
+
+namespace Interlocked
+{
+#if defined(_M_AMD64) || defined(_M_IA64)
+
+    LONGLONG inline CompareExchange64(LONGLONG volatile* destination,
+                                      LONGLONG comparand,
+                                      LONGLONG exchange)
+    {
+        return ::InterlockedCompareExchange64(destination, comparand, exchange);
+    }
+
+    LONGLONG inline Increment64(LONGLONG volatile *Addend)
+    {
+        return ::InterlockedIncrement64(Addend);
+    }
+    
+    LONGLONG inline Decrement64(LONGLONG volatile *Addend)
+    {
+        return InterlockedDecrement64(Addend);
+    }
+
+    LONGLONG inline Exchange64(LONGLONG volatile *Target, LONGLONG Value)
+    {
+        return ::InterlockedExchange64(Target, Value);
+    }
+
+    LONGLONG inline ExchangeAdd64(LONGLONG volatile *Addend, LONGLONG Value)
+    {
+        return ::InterlockedExchangeAdd64(Addend, Value);
+    }
+    
+    LONGLONG inline Read64(LONGLONG volatile *target)
+    {
+        return *target;
+    }
+    
+#else
+    
+    LONGLONG inline __cdecl CompareExchange64(LONGLONG volatile* destination,
+                                              LONGLONG comparand,
+                                              LONGLONG exchange)
+    {
+        __asm
+            {
+                mov   esi, [destination]
+                mov   ebx, dword ptr [comparand]
+                mov   ecx, dword ptr [comparand + 4]
+                mov   eax, dword ptr [exchange]
+                mov   edx, dword ptr [exchange + 4]
+                lock  cmpxchg8b [esi]
+            };
+    }
+
+    // Copied from winbase.h
+    LONGLONG inline Increment64(LONGLONG volatile *Addend)
+    {
+        LONGLONG Old;
+
+        do {
+            Old = *Addend;
+        } while (CompareExchange64(Addend, Old + 1, Old) != Old);
+        return Old + 1;
+    }
+
+    // Copied from winbase.h
+    LONGLONG inline Decrement64(LONGLONG volatile *Addend)
+    {
+        LONGLONG Old;
+
+        do {
+            Old = *Addend;
+        } while (CompareExchange64(Addend, Old - 1, Old) != Old);
+        return Old - 1;
+    }
+
+    // Copied from winbase.h
+    LONGLONG inline Exchange64(LONGLONG volatile *Target, LONGLONG Value)
+    {
+        LONGLONG Old;
+
+        do {
+            Old = *Target;
+        } while (CompareExchange64(Target, Value, Old) != Old);
+
+        return Old;
+    }
+
+
+    // Copied from winbase.h
+    LONGLONG inline ExchangeAdd64(LONGLONG volatile *Addend,
+                                  LONGLONG Value)
+    {
+        LONGLONG Old;
+
+        do {
+            Old = *Addend;
+        } while (CompareExchange64(Addend, Old + Value, Old) != Old);
+
+        return Old;
+    }
+
+    LONGLONG inline Read64(LONGLONG volatile *target)
+    {
+        // As far as I know, this is the only way to atomically read a
+        // 64 bit value on a 32 bit platform.
+        // InterlockedCompareExchange64 reads the target value
+        // atomically.  If the value is 0, it sets the value to 0 and
+        // returns 0 (no-op) If the value is non-zero, it does not
+        // change the value and returns old value.
+        return CompareExchange64(target, 0, 0);
+    }
+    
+#endif
+};

DryadVertex/VertexHost/system/classlib/include/LogIds.h

@@ -0,0 +1,846 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+//JC Check this file for unnecessary content.
+
+//JCnamespace apsdk
+//JC{
+
+// Add your log entries (areas) here
+// This list must be kept in sync with g_LogIDNames[] below
+// The list must start at 0 and increase sequentially
+typedef enum
+{
+    LogID_Logging, // The logging system can log stuff too!
+    LogID_MsnbotFileGen, // shared module used by different process like chunkbuilder, fex, RTDiscovery
+    LogID_Crawler,
+    LogID_CrawlerReceiver,
+    LogID_CrawlerFetcher,
+    LogID_CrawlerScheduler,
+    LogID_CrawlerDownloadLog,
+    LogID_Counters,
+    LogID_Parser,
+    LogID_IndexBuild,
+    LogID_IndexGeneration,
+    LogID_QueryProcessor,
+    LogID_IndexMerge,
+    LogID_IndexGeneral,
+    LogID_IndexQueryLog,
+    LogID_IndexServer,
+    LogID_MCPClient,
+    LogID_FileSync,
+    LogID_GetAndSyncChunkManifest,
+    LogID_ServiceManager,
+    LogID_DynamicRank,
+    LogID_Caption,
+    LogID_Federation,
+    LogID_APRunService,
+    LogID_GetAndRunServices,
+    LogID_Categorization,
+    LogID_StaticRanker,
+    LogID_StaticRankerBackup,
+    LogID_DRClient,
+    LogID_DRServer,
+    LogID_DRLib,
+    LogID_Common,
+    LogID_Aggregator,
+    LogID_Netlib,
+    LogID_CatCFSpecific,
+    LogID_SpamDetection,
+    LogID_CrawlerUrlTracker,
+    LogID_StaticRankerDetail,
+    LogID_FixRegKeys,
+    LogID_CreateCounter,
+    LogID_CRCHashComputation,
+    LogID_FEX,
+    LogID_Moonshot,
+    LogID_Fcslite,
+    LogID_FcsliteCache,
+    LogID_QualityOfPage,
+    LogID_UST,
+    LogID_SEARCH,
+    LogID_FSServer,
+    LogID_FSClient,
+    LogID_DPClient,
+    LogID_ENCARTA,
+    LogID_MUSIC,
+    LogID_LeaseVote,
+    LogID_Test,
+    LogID_CreateSocEvent,
+    LogID_SyncFiles,
+    LogID_Doodad,
+    LogID_DoodadFCtoLC,
+    LogID_DoodadLCtoFC,
+    LogID_DoodadWatchdog,
+    LogID_DOSDog,
+    LogID_StaticRankerTester,
+    LogID_LogCollector,
+    LogID_LogConvertor,
+    LogID_LogUploader,
+    LogID_LocalWD,
+    LogID_MachineWatchdog,
+    LogID_IPBlock,
+    LogID_DisableServerHeader,
+    LogID_IAdmin,
+    LogID_GENERICAP,
+    LogID_PSGateway,
+    LogID_HttpProxy,
+    LogID_CacheInfo,
+    LogID_FeedBuilder,
+    LogID_FeedBuilderData,
+    LogID_SysWD,
+    LogID_AlertService,
+    LogID_FexRLog,
+    LogID_FexHLog,
+    LogID_FexPreLog,
+    LogID_FexPostDataLog,
+    LogID_PsTFTP,
+    LogID_PsGatewayWatchdog,
+    LogID_CacheManager,
+    LogID_RSLLIB,
+    LogID_BurninTest,
+    LogID_QpsWatchdog,
+    LogID_ClusterBuilder,
+    LogID_MSRATermExtractor,
+    LogID_RealTimeCrawl,
+    LogID_RealTimeBuild,
+    LogID_DPS,
+    LogID_PerfCollector,
+    LogID_PerfAggregator,
+    LogID_ReplicatedDataSet,
+    LogID_WdWatcher,
+    LogID_DigiWatchdog,
+    LogID_OutBandMappingWatchdog,
+    LogID_PhantomWatchdog,
+    LogID_DigiProvisioning,
+    LogID_MiniSwitchProvisioning,
+    LogID_TLALoadBalancer,
+    LogID_Shopping,
+    LogID_CacheLogsProcessor,
+    LogID_DsSync,
+    LogID_DM,
+    LogID_DMWD,
+    LogID_DMAudit,
+    LogID_DMSQL,
+    LogID_PhraseTokenExtractor,
+    LogID_CQAClient,
+    LogID_CQABuilder,
+    LogID_CQAOQI,
+    LogID_CQAUserStore,
+    LogID_CQATagStore,
+    LogID_CQAQAStore,
+    LogID_TermExtractor,
+    LogID_EMS,
+    LogID_EMSStat,
+    LogID_Macro,
+    LogID_DRM,
+    LogID_EN,
+    LogID_EnWd,
+    LogID_ENTest,
+    LogID_Cosmos,
+    LogID_DrClient,
+    LogID_DrNameServer,
+    LogID_PN,
+    LogID_Dryad,
+    LogID_DryadTest,
+    LogID_DryadTestJournal,
+    LogID_DryadClient,
+    LogID_CockpitServer,
+    LogID_APWebServer,
+    LogID_QueryService,
+    LogID_MultiEnv,
+    LogID_Election,
+    LogID_APTextProtocol,
+    LogID_APMPClient,
+    LogID_APManagementProxy,
+    LogID_APProxyCommand,
+    LogID_FexLiveLog,
+    LogID_Wssync,
+    LogID_WmiLib,
+    LogID_PhantomLib,
+    LogID_AssetTool,
+    LogID_DeviceScanner,
+    LogID_DeadMachineMacScanner,
+    LogID_ServerTools,
+    LogID_BOOTP,
+    LogID_DHCP,
+    LogID_PXE,
+    LogID_BOOTPServer,
+    LogID_Duff,
+    LogID_IdCollect,
+    LogID_DryadWebServer,
+    LogID_DryadLogLoader,
+    LogID_DrmReplication,
+    LogID_CQACloseQuestions,
+    LogID_CQAPresser,
+    LogID_CQAPoller,
+    LogID_CQAStaticRanker,
+    LogID_CQATagTextProcessor,
+    LogID_CQAAlert,
+    LogID_CQAAlertStore,
+    LogID_FEX_CQA,
+    LogID_CQAWatchdogs,
+    LogID_MultiMedia,
+    LogID_ImageFetcher,
+    LogID_ImageFetcherCrawler,
+    LogID_ImageFetcherThumbnailer,
+    LogID_LiveSearchPane,
+    LogID_PsServer,
+    LogID_PsClient,
+    LogID_PsAgent,
+    LogID_RMAUtils,
+    LogID_LCDService,
+    LogID_DRV,
+    LogID_DRVParser,
+    LogID_CreateImage,
+    LogID_CreateImageFiles,
+    LogID_PhantomPowerStateWatchdog,
+    LogID_WebFeedDiscovery,
+    LogID_Answers,
+    LogID_Newsgroup,
+    LogID_TLARemoting,
+    LogID_HttpClient,
+    LogID_DiskTest,
+    LogID_DiskTestSequential,
+    LogID_DiskTestRandom,
+    LogID_MemoryTest,
+    LogID_WebFeedDiscoveryReader,
+    LogID_WebFeedDiscoveryProcessor,
+    LogID_WebFeedDiscoveryCrawlfileGenerator,
+    LogID_WebFeedDiscoveryRSSFetcher,
+    LogID_WebFeedDiscoveryProvider,
+    LogID_FileSyncIgnoreCRC,
+    LogID_FileSyncWrongCRC,
+    LogID_DSSlaveSync,
+    LogID_OSUpgrade,
+    LogID_DeviceUpgrade,
+    LogID_SyncOSImage,
+    LogID_ReliableRebootService,
+    LogID_GenericRepair,
+    LogID_NetlibCorruptPacket,
+    LogID_QuerySuggestion,
+    LogID_ISNManager,
+    LogID_ChunkLocator,
+    LogID_ISAgent,
+    LogID_ConfigViews,
+    LogID_CDGDominantImage,
+    LogID_DrJobManager,
+    LogID_DrJobManagerWd,
+    LogID_CachePropagator,
+    LogID_CachePropagatorClient,
+    LogID_MlToHosts,
+    LogID_TidyFS,
+    LogID_UrlTracker,
+    LogID_DNSWatchdog,
+    LogID_PsWatchdog,
+    LogID_WebFeedDiscoveryPinger,
+    LogID_MacroSuggestion,
+    LogID_AppAlertService,
+    LogID_TLARowStatus,
+    LogID_Sputnik,
+    LogID_GeneralClient,
+    LogID_QueryAugmenter,
+    LogID_Dictionary,
+    LogID_QueryStatistics,
+    LogID_QueryISNStatistics,
+    LogID_AnswersRLog,
+    LogID_AnswersALog,
+    LogID_AnswersPreLog,
+    LogID_IPTable,
+    LogID_DNSService,
+    LogID_FexMissingSnippetLog,
+    LogID_XifBuilder,
+    LogID_SharedModules,
+    LogID_DryadProfiler,
+    LogID_DryadProxy,
+    LogID_RTIndexCoverage,
+    LogID_Hardware,
+    LogID_PowerstripProvisioning,
+    LogID_PowerstripWatchdog,
+    LogID_Watchdog,
+    LogID_ApCommClientServer,
+    LogID_AdminGroup,
+    LogID_DMClient,
+    LogID_SvcMgrClient,
+    LogID_Speller,
+    LogID_FDR,
+    LogID_MT_HttpServer,
+    LogID_MT_Distributor,
+    LogID_MT_Cache,
+    LogID_MT_Translator,
+    LogID_MT_ResearchSdk,
+    LogID_MT_ModelServer,
+    LogID_MT_DB,
+    LogID_Localization,
+    LogID_MachineStatusClient,
+    LogID_ExpensiveQueryMonitor,
+    LogID_FeedChunkCleaner,
+    LogID_Hello,
+    LogID_AnswersPerformanceMonitor,
+    LogID_APMResult,
+    LogID_BackEndMachines,
+    LogID_LogService,
+    LogID_UserEvent,
+    LogID_FexImpressionLog,
+    LogID_RMAService,
+    LogID_RMAProtocol,
+    LogID_CockpitWatchdog,
+    LogID_DeviceCounterCollector,
+    LogID_DhcpMonitor,
+    LogID_DeviceCounter,
+    LogID_InstallService,
+    LogID_MinidumpSummary,
+    LogID_DeviceValidater,
+    LogID_ExpressRanker,
+    LogID_CrawlerUseChunk,
+    LogID_CrawlerDropLog,
+    LogID_CrawlerToBuilderLog,
+    LogID_ChunkBuilderArchive,
+    LogID_DocConverter,
+    LogID_ChunkBuilderCanary,
+    LogID_DynamicCrawler,
+    LogID_CacheClient,
+    LogID_CacheCommon,
+    LogID_QueryAlteration,
+    LogID_ResultAlteration,
+    LogID_ChunkSyncManager,
+    LogID_ChunkSyncManagerVerbose,
+    LogID_Extractor,
+    LogID_StaticRankManager,
+    LogID_ISNMonitorWatchdog,
+    LogID_VoxPopuliRatingLog,
+    LogID_KickServices,
+    LogID_DnsServer,
+    LogID_DnsServerRequest,
+    LogID_UpdateSecurityGroups,
+    LogID_SearchRepository,
+    LogID_SearchRepositoryCommon,
+    LogID_SearchRepositoryFELib,
+    LogID_SearchRepositoryProtocol,
+    LogID_SearchRepositoryLocator,
+    LogID_SearchRepositoryReadNode,
+    LogID_SearchRepositoryTest,
+    LogID_SearchRepositoryLog,
+    LogID_SearchRepositoryHttpServer,
+    LogID_SearchRepositoryWatchdog,
+    LogID_SearchRepositoryBackDoor,
+    LogID_SearchRepositoryMergeMgr,
+    LogID_SearchRepositoryMerger,
+    LogID_SearchRepositoryClient,
+    LogID_BackendQueryResult,
+    LogID_TLAPreLog,
+    LogID_APM_AlertLog,
+    LogID_WebAnswer,
+    LogID_QueryLog,
+    LogID_WatchDogClient,
+    LogID_WatchDogServer,
+    LogID_FcsXml,
+    LogID_FcsPostLog,
+    LogID_FcsErrorQueriesLog,
+    LogID_FcsLostQueriesLog,
+    LogID_DUI,
+    LogID_StufSync,
+    LogID_ISMerge,
+    LogID_WebMerge,
+    LogID_UserData,
+    LogID_Commerce,
+    LogID_XRank,
+    LogID_DocFetcher,
+    LogID_VoxPopuliGeneralLog,
+    LogID_VoxPopuliDecisionLog,
+    LogID_ChunkPublisher,
+    LogID_MediaProcessor,
+    LogID_KeywordExtractor,
+    LogID_IndexTracker,
+    LogID_Syslog,
+    LogID_Environment,
+    LogID_GenericAudit,
+    LogID_FcsAnswerQueryLog,
+    LogID_FeedsImportClient,
+    LogID_FeedsImportServer,
+    LogID_FeedsCapacityManager,
+    LogID_RealtimeVideo,
+    LogID_ThreadPoolLib,
+    LogID_TSFDR,
+    LogID_TSClient,
+    LogID_TSServer,
+    LogID_TSPipeline,
+    LogID_API,
+    LogID_API_Request,
+    LogID_API_Response,
+    LogID_ThrottleManagement,
+    LogID_SRCache,
+    LogID_DrmMirroring,
+    LogID_FrontDoor,
+    LogID_News,
+    LogID_FexCrash,
+    LogID_Webmaster,
+    LogID_WMSitemapSubmit,
+    LogID_WMLinkDataAggregator,
+    LogID_SqlRepository,
+    LogID_SqlRepositoryWatchDog,
+    LogID_LinkDataAggregator,
+    LogID_LinkDataAggregatorPreLog,
+    LogID_LinkDataAggregatorPostLog,
+    LogID_LinkDataAggregatorErrorLog,
+    LogID_WebmasterSitemapService,
+    LogID_WebmasterClog,
+    LogID_PhonebookResult,
+    LogID_PhonebookClassifier,
+    LogID_QueryDiagnostic,
+    LogID_DMExhibitCounter,
+    LogID_SyncAutopilotData,
+    LogID_AuthProxyServer,
+    LogID_AnswersXifBuilder,
+    LogID_FeedsDataProvider,
+    LogID_Scrounger,
+    LogID_Kif,
+    LogID_VariantConfigParser,
+
+    // This will include all the logids added
+    // by partners that are using the logging API
+#ifdef APSDK_CUSTOMIZED_LOGIDS_STRING
+#error APSDK_CUSTOMIZED_LOGIDS_STRING has already be defined.
+#endif
+#define APSDK_CUSTOMIZED_LOGIDS_STRING(x) LogIDEx_##x
+#include "LogIdsCustomized.h"
+#undef APSDK_CUSTOMIZED_LOGIDS_STRING
+
+    // This must be the last entry
+    LogID_Count
+} LogID;
+
+extern char *g_LogIDNames[];
+
+#ifdef DECLARE_DATA
+
+// This is an array of possible log entries (areas)
+// This must be kept in sync with the LogID enumeration above
+char *g_LogIDNames[] =
+{
+    "Logging",
+    "MsnbotFileGen",
+    "Crawler",
+    "CrawlerReceiver",
+    "CrawlerFetcher",
+    "CrawlerScheduler",
+    "CrawlerDownloadLog",
+    "Counters",
+    "Parser",
+    "IndexBuild",
+    "IndexGeneration",
+    "QueryProcessor",
+    "IndexMerge",
+    "IndexGeneral",
+    "IndexQueryLog",
+    "IndexServer",
+    "MCPClient",
+    "FileSync",
+    "GetAndSyncChunkManifest",
+    "ServiceManager",
+    "DynamicRank",
+    "Caption",
+    "Federation",
+    "APRunService",
+    "GetAndRunServices",
+    "Categorization",
+    "StaticRanker",
+    "StaticRankerBackup",
+    "DRClient",
+    "DRServer",
+    "DRLib",
+    "Common",
+    "Aggregator",
+    "Netlib",
+    "CatCFSpecific",
+    "SpamDetection",
+    "CrawlerUrlTracker",
+    "StaticRankerDetail",
+    "FixRegKeys",
+    "CreateCounter",
+    "CRCHashComputation",
+    "FEX",
+    "Moonshot",
+    "Fcslite",
+    "FcsliteCache",
+    "QualityOfPage",
+    "UrlSubmissionTool",
+    "Search",
+    "FSServer",
+    "FSClient",
+    "DPClient",
+    "Encarta",
+    "Music",
+    "LeaseVote",
+    "Test",
+    "CreateSocEvent",
+    "SyncFiles",
+    "Doodad",
+    "DoodadFCtoLC",
+    "DoodadLCtoFC",
+    "DoodadWatchdog",
+    "DOSDog",
+    "StaticRankerTester",
+    "Collector",
+    "Convertor",
+    "Uploader",
+    "LocalWD",
+    "MachineWatchdog",
+    "IPBlock",
+    "DisableServerHeader",
+    "IAdmin",
+    "GenericAP",
+    "PSGateway",
+    "HttpProxy",
+    "CacheInfo",
+    "FeedBuilder",
+    "FeedBuilderData",
+    "SysWatchdog",
+    "AlertService",
+    "FexRLog",
+    "FexHLog",
+    "FexPreLog",
+    "FexPostDataLog",
+    "PsTFTP",
+    "PsGatewayWatchdog",
+    "CacheManager",
+    "RSL",
+    "BurninTest",
+    "QpsWatchdog",
+    "ClusterBuilder",
+    "MSRATermExtractor",
+    "RealTimeCrawl",
+    "RealTimeBuild",
+    "ProfileStore",
+    "PerfCollector",
+    "PerfAggregator",
+    "ReplicatedDataSet",
+    "WatchdogsWatcher",
+    "DigiWatchdog",
+    "OutBandMappingWatchdog",
+    "PhantomWatchdog",
+    "DigiProvisioning",
+    "MiniSwitchProvisioning",
+    "TLALoadBalancer",
+    "Shopping",
+    "CacheLogsProcessor",
+    "DatasetSync",
+    "DeviceManager",
+    "DeviceManagerIncomingWatchdog",
+    "DMAudit",
+    "DeviceManagerSQL",
+    "PhraseTokenExtractor",
+    "CQAClient",
+    "CQABuilder",
+    "CQAOQI",
+    "CQAUserStore",
+    "CQATagStore",
+    "CQAQAStore",
+    "TermExtractor",
+    "EMS",
+    "EMSStat",
+    "Macro",
+    "DRM",
+    "EN",
+    "EnWd",
+    "ENTest",
+    "Dryad",
+    "DrClient",
+    "DrNameServer",
+    "PN",
+    "Dryad",
+    "DryadTest",
+    "DryadTestJournal",
+    "DryadClient",
+    "CockpitServer",
+    "APWebServer",
+    "QueryService",
+    "MultiEnv",
+    "Election",
+    "APTextProtocol",
+    "APMPClient",
+    "APProxy",
+    "APProxyCommand",
+    "FexLiveLog",
+    "wssynclog",
+    "WmiLib",
+    "PhantomLib",
+    "AssetTool",
+    "DeviceScanner",
+    "DeadMachineMacScanner",
+    "ServerTools",
+    "BOOTP",
+    "DHCP",
+    "PXE",
+    "BOOTPServer",
+    "Duff",
+    "IdCollect",
+    "DryadWebServer",
+    "DryadLogLoader",
+    "DrmReplication",
+    "CQACloseQuestions",
+    "CQAPresser",
+    "CQAPoller",
+    "CQAStaticRanker",
+    "CQATagTextProcessor",
+    "CQAAlert",
+    "CQAAlertStore",
+    "FexCQA",
+    "CQAWatchdogs",
+    "MultiMedia",
+    "ImageFetcher",
+    "ImageFetcherCrawler",
+    "ImageFetcherThumbnailer",
+    "LiveSearchPane",
+    "PsServer",
+    "PsClient",
+    "PsAgent",
+    "RMAUtils",
+    "LCDService",
+    "DRV",
+    "DRVParser",
+    "CreateImage",
+    "CreateImageFiles",
+    "PhantomPowerStateWatchdog",
+    "WebFeedDiscovery",
+    "Answers",
+    "Newsgroup",
+    "TLARemoting",
+    "HttpClient",
+    "DiskTest",
+    "DiskTestSequential",
+    "DiskTestRandom",
+    "MemoryTest",
+    "WebFeedDiscoveryReader",
+    "WebFeedDiscoveryProcessor",
+    "WebFeedDiscoveryCrawlfileGenerator",
+    "WebFeedDiscoveryRSSFetcher",
+    "WebFeedDiscoveryProvider",
+    "FileSyncIgnoreCRC",
+    "FileSyncWrongCRC",
+    "DSSlaveSync",
+    "OSUpgrade",
+    "DeviceUpgrade",
+    "SyncOSImage",
+    "ReliableRebootService",
+    "GenericRepair",
+    "NetlibCorruptPacket",
+    "QuerySuggestion",
+    "ISNManager",
+    "ChunkLocator",
+    "ISAgent",
+    "ConfigViews",
+    "CDGDominantImage",
+    "DryadJobManager",
+    "DryadJobManagerWd",
+    "CachePropagator",
+    "CachePropagatorClient",
+    "MlToHosts",
+    "TidyFS",
+    "UrlTracker",
+    "DNSWatchdog",
+    "PsWatchdog",
+    "WebFeedDiscoveryPinger",
+    "MacroSuggestion",
+    "AppAlertService",
+    "RowStatus",
+    "Sputnik",
+    "GeneralClient",
+    "QueryAugmenter",
+    "Dictionary",
+    "QueryStatistics",
+    "QueryISNStatistics",
+    "AnswersRLog",
+    "AnswersALog",
+    "AnswersPreLog",
+    "IPTable",
+    "DNSService",
+    "FexMissingSnippetLog",
+    "XifBuilder",
+    "SharedModules",
+    "DryadProfiler",
+    "DryadProxy",
+    "RTIndexCoverage",
+    "Hardware",
+    "PowerstripProvision",
+    "PowerstripWatchdog",
+    "Watchdog",
+    "ApCommClientServer",
+    "LocalAdminGroup",
+    "DMClient",
+    "SvcMgrClient",
+    "Speller",
+    "FDR",
+    "MT_HttpServer",
+    "MT_Distributor",
+    "MT_Cache",
+    "MT_Translator",
+    "MT_ResearchSdk",
+    "MT_ModelServer",
+    "MT_DB",
+    "Localization",
+    "MachineStatusClient",
+    "ExpensiveQueryMonitor",
+    "FeedChunkCleaner",
+    "Hello",
+    "AnswersPerformanceMonitor",
+    "APMResult",
+    "BackEndMachines",
+    "LogService",
+    "UserEvent",
+    "FexImpressionLog",
+    "RMAService",
+    "RMAProtocol",
+    "CockpitWatchdog",
+    "DeviceCounterCollector",
+    "DhcpMonitor",
+    "DeviceCounter",
+    "InstallService",
+    "MinidumpSummary",
+    "DeviceValidater",
+    "ExpressRanker",
+    "CrawlerUseChunk",
+    "CrawlerDropLog",
+    "CrawlerToBuilderLog",
+    "ChunkBuilderArchive",
+    "DocumentConverter",
+    "ChunkBuilderCanary",
+    "DynamicCrawler",
+    "CacheClient",
+    "CacheCommon",
+    "QueryAlteration",
+    "ResultAlteration",
+    "ChunkSyncManager",
+    "ChunkSyncManagerVerbose",
+    "Extractor",
+    "StaticRankManager",
+    "ISNMonitorWatchdog",
+    "VoxPopuliRatingLog",
+    "KickServices",
+    "DnsServer",
+    "DnsServerRequest",
+    "UpdateSecurityGroups",
+    "SearchRepository",
+    "SearchRepositoryCommon",
+    "SearchRepositoryFELib",
+    "SearchRepositoryProtocol",
+    "SearchRepositoryLocator",
+    "SearchRepositoryReadNode",
+    "SearchRepositoryTest",
+    "SearchRepositoryLog",
+    "SearchRepositoryHttpServer",
+    "SearchRepositoryWatchdog",
+    "SearchRepositoryBackDoor",
+    "SearchRepositoryMergeMgr",
+    "SearchRepositoryMerger",
+    "SearchRepositoryClient",
+    "BackendQueryResult",
+    "TLAPreLog",
+    "APMAlertLog",
+    "WebAnswer",
+    "QueryLog",
+    "WatchDogClient",
+    "WatchDogServer",
+    "FcsXml",
+    "FcsPostLog",
+    "FcsErrorQueriesLog",
+    "FcsLostQueriesLog",
+    "DUIProcess",
+    "StufSync",
+    "ISMerge",
+    "WebMerge",
+    "UserData",
+    "Commerce",
+    "XRank",
+    "DocumentFetcherService",
+    "VoxPopuliGeneralLog",
+    "VoxPopuliDecisionLog",
+    "ChunkPublisher",
+    "MediaProcessor",
+    "KeywordExtractor",
+    "IndexTracker",
+    "Syslog",
+    "Environment",
+    "GenericAudit",
+    "FcsAnswerQueryLog",
+    "FeedsImportClient",
+    "FeedsImportServer",
+    "FeedsCapacityManager",
+    "RTVideo",
+    "ThreadPoolLib",
+    "TSFDR",
+    "TSClient",
+    "TSServer",
+    "TSPipeline",
+    "API",
+    "APIRequest",
+    "APIResponse",
+    "ThrottleManagement",
+    "SearchRepositoryCache",
+    "DrmMirroring",
+    "FrontDoor",
+    "News",
+    "FexCrash",
+    "Webmaster",
+    "WMSitemapSubmit",
+    "WMLinkDataAggregator",
+    "SqlRepository",
+    "SqlRepositoryWatchDog",
+    "LinkDataAggregator",
+    "LinkDataAggregatorPreLog",
+    "LinkDataAggregatorPostLog",
+    "LinkDataAggregatorErrorLog",
+    "WebmasterSitemapService",
+    "WebmasterClog",
+    "PhonebookResult",
+    "PhonebookClassifier",
+    "QueryDiag",
+    "DMExhibitCounter",
+    "SyncAutopilotData",
+    "AuthProxyServer",
+    "AnswersXifBuilder",
+    "FeedsDataProvider",
+    "Scrounger",
+    "Kif",
+    "VariantConfigParser",
+	
+    // This will include all the logids added
+    // by partners that are using the logging API
+#ifdef APSDK_CUSTOMIZED_LOGIDS_STRING
+#error APSDK_CUSTOMIZED_LOGIDS_STRING has already be defined.
+#endif
+#define APSDK_CUSTOMIZED_LOGIDS_STRING(x) #x
+#include "LogIdsCustomized.h"
+#undef APSDK_CUSTOMIZED_LOGIDS_STRING
+
+    // Last entry must be a NULL (for sanity checking)
+    NULL
+};
+
+#endif
+
+//JC} // namespace apsdk
+//JC
+
+#ifdef USING_APSDK_NAMESPACE
+using namespace apsdk;
+#endif

DryadVertex/VertexHost/system/classlib/include/LogIdsCustomized.h

@@ -0,0 +1,91 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//JC Check this file for unnecessary content.
+
+//
+// This header file is designed to be included twice by the
+// old autopilot logging code, once by new logging.
+//
+// This file meant to be used by
+// non-autopilot code to define logids without having to
+// modify any the autopilot SDK files (not really necessary
+// with new logging).
+//
+
+#ifndef USE_DRTRACE
+
+#if !defined(APSDK_CUSTOMIZED_LOGIDS_STRING)
+#error This file is not meant to be included directly. APSDK_CUSTOMIZED_LOGIDS_STRING is not defined.
+#endif
+
+// To add a new log id, append a line below in the form of:
+//      APSDK_CUSTOMIZED_LOGIDS_STRING(Foo),
+// This will end up creating two things, an enum LogIDEx_Foo of enum type LogID and a string "Foo"
+// as the value for g_LogIDNames[LogIDEx_Foo].
+
+    APSDK_CUSTOMIZED_LOGIDS_STRING(SDKBasicSampleWatchdog),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(SDKSample),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(AnswersMatchLog),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(AnswersRemoteTLAPreLog),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(VoxPopuliRatingLog), 
+    APSDK_CUSTOMIZED_LOGIDS_STRING(DryadAudit),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(DryadRS),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(DryadCache),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(DryadSimulator),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(QUERY_PROCESSING),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(QUERY_PARSING),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(QUERY_REWRITING),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(QUERY_CLASSIFICATION),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(WatchDogClient),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(WatchDogServer),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(CacheSync),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(AdServiceHttpRequestLog),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(AdServiceHttpResponseLog),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(AdServiceXMLResponseLog),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(AnswersFrameworkDebug),
+    APSDK_CUSTOMIZED_LOGIDS_STRING(NoCodeUser),
+#else
+
+//
+// With new logging, just define the value (or, better yet, do it somewhere in your project)
+//
+
+#define LogIDEx_SDKBasicSampleWatchdog          "SDKBasicSampleWatchdog"
+#define LogIDEx_SDKSample                       "SDKSample"
+#define LogIDEx_AnswersMatchLog                 "AnswersMatchLog"
+#define LogIDEx_AnswersRemoteTLAPreLog          "AnswersRemoteTLAPreLog"
+#define LogIDEx_VoxPopuliRatingLog              "VoxPopuliRatingLog" 
+#define LogIDEx_DryadAudit                     "DryadAudit"
+#define LogIDEx_DryadRS                        "DryadRS"
+#define LogIDEx_DryadCache                     "DryadCache"
+#define LogIDEx_DryadSimulator                  "DryadSimulator"
+#define LogIDEx_QUERY_PROCESSING                "QUERY_PROCESSING"
+#define LogIDEx_QUERY_PARSING                   "QUERY_PARSING"
+#define LogIDEx_QUERY_REWRITING                 "QUERY_REWRITING"
+#define LogIDEx_QUERY_CLASSIFICATION            "QUERY_CLASSIFICATION"
+#define LogIDEx_AdServiceHttpRequestLog         "AdServiceHttpRequestLog"
+#define LogIDEx_AdServiceHttpResponseLog        "AdServiceHttpResponseLog"
+#define LogIDEx_AdServiceXMLResponseLog         "AdserviceXMLResponseLog"
+#define LogIDEx_AnswersFrameworkDebug           "AnswersFrameworkDebug"
+#define LogIDEx_NoCodeUser                      "NoCodeUser"
+
+#endif
+

DryadVertex/VertexHost/system/classlib/include/LogTagIds.h

@@ -0,0 +1,612 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+//JC Check this file for unnecessary content.
+
+//
+// This file lists all the known tag values for logging.
+//
+// This file is included twice, with DeclareTag() defined to do different things, to initialize
+// different data structures with the logging code.
+//
+// Add any new named log tags to this file
+// The IDs MUST be sequential
+// Update LogTag_End if you add a new one
+//
+// The value in quotes is the name under which it will appear in the log file;
+// e.g. Filename="Foo"
+//
+
+// Use these as generic log tags parameters if you don't feel it makes sense to add a new
+// log tag entry.  There really is no real overhead for adding more, though, so if you're
+// going to log the same data item in more than a few places, you probably want to consider
+// making a log tag for it, in case we want WatchDog to be able to key off it.
+DeclareTag(LogTag_Int1,          "Int1",             LogTagType_Int32),
+DeclareTag(LogTag_Int2,          "Int2",             LogTagType_Int32),
+DeclareTag(LogTag_Int64_1,       "Int64_1",          LogTagType_Int64),
+DeclareTag(LogTag_Int64_2,       "Int64_2",          LogTagType_Int64),
+DeclareTag(LogTag_String1,       "String1",          LogTagType_String),
+DeclareTag(LogTag_String2,       "String2",          LogTagType_String),
+DeclareTag(LogTag_WString1,      "WString1",         LogTagType_WideString),
+DeclareTag(LogTag_WString2,      "WString2",         LogTagType_WideString),
+DeclareTag(LogTag_UInt1,         "UInt1",            LogTagType_UInt32),
+DeclareTag(LogTag_UInt2,         "UInt2",            LogTagType_UInt32),
+DeclareTag(LogTag_U32X1,         "Hex1",            LogTagType_Hex32),
+DeclareTag(LogTag_U32X2,         "Hex2",            LogTagType_Hex32),
+DeclareTag(LogTag_U64X1,         "Hex64_1",         LogTagType_Hex64),
+DeclareTag(LogTag_U64X2,         "Hex64_2",         LogTagType_Hex64),
+DeclareTag(LogTag_Float1,        "Float1",          LogTagType_Float),
+DeclareTag(LogTag_Float2,        "Float2",          LogTagType_Float),
+
+DeclareTag(LogTag_Filename,     "Filename",          LogTagType_String),
+DeclareTag(LogTag_TargetServer, "TargetServer",      LogTagType_String),
+DeclareTag(LogTag_QueryString,  "Q",                 LogTagType_String),
+DeclareTag(LogTag_TraceID,      "TraceID",           LogTagType_String),
+DeclareTag(LogTag_URL,          "URL",              LogTagType_String),
+DeclareTag(LogTag_ErrorCode,    "ErrorCode",        LogTagType_UInt32),
+DeclareTag(LogTag_ThreadID,    "ThreadID",          LogTagType_Int32),
+DeclareTag(LogTag_Description, "Desc",              LogTagType_String),
+
+// ids for ini error
+DeclareTag(LogTag_Section,     "Section",           LogTagType_String),
+DeclareTag(LogTag_Param1,      "Param1",            LogTagType_String),
+DeclareTag(LogTag_Param2,      "Param2",            LogTagType_String),
+DeclareTag(LogTag_Param3,      "Param3",            LogTagType_String),
+DeclareTag(LogTag_Param4,      "Param4",            LogTagType_String),
+
+// ids for SQL ODBC error
+DeclareTag(LogTag_SQLMessage,          "SQLMessage",        LogTagType_String),
+DeclareTag(LogTag_SQLErrorCode,        "SQLErrorCode",      LogTagType_Int32),
+DeclareTag(LogTag_SQLState,            "SQLState",          LogTagType_String),
+
+// ids for SQL BCP error
+DeclareTag(LogTag_SQLBCPFunction,      "SQLBCPFunction",    LogTagType_String),
+DeclareTag(LogTag_SQLBCPToTable,       "SQLBCPToTable",     LogTagType_String),
+DeclareTag(LogTag_SQLBCPColumn,        "SQLBCPColumn",      LogTagType_Int32),
+DeclareTag(LogTag_SQLBCPRowCountSum,   "SQLBCPRowCountSum", LogTagType_Int32),
+DeclareTag(LogTag_SQLBCPRowCountInc,   "SQLBCPRowCountInc", LogTagType_Int32),
+DeclareTag(LogTag_SQLBCPRowCountCur,   "SQLBCPRowCountCur", LogTagType_Int32),
+
+// ids for aggregator machine status notifications
+DeclareTag(LogTag_ISNName,        "ISNName", LogTagType_String),
+DeclareTag(LogTag_ISNService,     "ISNService", LogTagType_String),
+DeclareTag(LogTag_ISNFailureType, "ISNFailureType", LogTagType_String),
+
+// ids for Caption Generator
+DeclareTag(LogTag_DocId,            "DocId",            LogTagType_Hex64),
+DeclareTag(LogTag_ContentChunkId,   "ContentChunkId",   LogTagType_Int32),
+
+// ids for Netlib
+DeclareTag(LogTag_Port,         "Port",             LogTagType_UInt32),
+DeclareTag(LogTag_IP,           "IP",               LogTagType_String),
+DeclareTag(LogTag_NumericIP,    "NumericIP",        LogTagType_Hex32),
+DeclareTag(LogTag_LastError,    "LastError",        LogTagType_UInt32),
+
+// ids for Fcslite
+DeclareTag(LogTag_PacketId,             "PacketId",             LogTagType_UInt32),
+DeclareTag(LogTag_ConfigName,           "ConfigName",           LogTagType_String),
+DeclareTag(LogTag_ResultBase,           "ResultBase",           LogTagType_UInt32),
+DeclareTag(LogTag_ResultsCount,         "ResultsCount",         LogTagType_UInt32),
+DeclareTag(LogTag_MaxResultsPerHost,    "MaxResultsPerHost",    LogTagType_UInt32),
+DeclareTag(LogTag_FcsOptions,           "FcsOptions",           LogTagType_Hex64),
+DeclareTag(LogTag_AggregatorOptions,    "AggregatorOptions",    LogTagType_Hex64),
+DeclareTag(LogTag_QueryOptions,         "QueryOptions",         LogTagType_Hex64),
+DeclareTag(LogTag_CaptionOptions,       "CaptionOptions",       LogTagType_Hex64),
+DeclareTag(LogTag_NextTierToQuery,      "NextTierToQuery",      LogTagType_UInt32),
+DeclareTag(LogTag_MaxTiersInIndex,      "MaxTiersInIndex",      LogTagType_UInt32),
+DeclareTag(LogTag_RowID,                "RowID",                LogTagType_UInt32),
+
+// These should have been in the generic section but are here to be in the proper range
+DeclareTag(LogTag_UInt641,       "UInt641",          LogTagType_UInt64),
+DeclareTag(LogTag_UInt642,       "UInt642",          LogTagType_UInt64),
+
+#if defined(_M_AMD64)
+DeclareTag(LogTag_Sizet1,       "Size_t1",      LogTagType_UInt64),
+DeclareTag(LogTag_Sizet2,       "Size_t2",      LogTagType_UInt64),
+#else
+DeclareTag(LogTag_Sizet1,       "Size_t1",      LogTagType_UInt32),
+DeclareTag(LogTag_Sizet2,       "Size_t2",      LogTagType_UInt32),
+#endif
+
+DeclareTag(LogTag_ActiveMCP,    "Active",           LogTagType_Int32),
+DeclareTag(LogTag_VoteCount,    "VoteCount",        LogTagType_Int32),
+
+#if defined(_M_AMD64)
+DeclareTag(LogTag_Ptr1,       "Ptr1",       LogTagType_Hex64),
+DeclareTag(LogTag_Ptr2,       "Ptr2",       LogTagType_Hex64),
+#else
+DeclareTag(LogTag_Ptr1,       "Ptr1",       LogTagType_Hex32),
+DeclareTag(LogTag_Ptr2,       "Ptr2",       LogTagType_Hex32),
+#endif
+
+DeclareTag(LogTag_Latitude, "Latitude",     LogTagType_Float),
+DeclareTag(LogTag_Longitude, "Longitude",    LogTagType_Float),
+
+DeclareTag(LogTag_VarString1,       "VarString1",          LogTagType_VarString),
+DeclareTag(LogTag_VarString2,       "VarString2",          LogTagType_VarString),
+DeclareTag(LogTag_VarWString1,      "VarWString1",         LogTagType_VarWideString),
+DeclareTag(LogTag_VarWString2,      "VarWString2",         LogTagType_VarWideString),
+
+// ids for service manager
+DeclareTag(LogTag_PID,  "ProcessID",  LogTagType_UInt32),
+DeclareTag(LogTag_PPID, "ParentProcessID", LogTagType_UInt32),
+
+// ids for fex
+DeclareTag(LogTag_FEXLatency,  "FEXLatency",  LogTagType_Float),
+DeclareTag(LogTag_TLALatency, "TLALatency", LogTagType_Float),
+DeclareTag(LogTag_CDGLatency,  "CDGLatency",  LogTagType_Float),
+DeclareTag(LogTag_ISNLatency, "ISNLatency", LogTagType_Float),
+DeclareTag(LogTag_FcsStatus, "FcsStatus", LogTagType_Hex64),
+DeclareTag(LogTag_MachineNeeded, "MachineNeeded", LogTagType_UInt32),
+DeclareTag(LogTag_MachineComplete, "MachineComplete", LogTagType_UInt32),
+DeclareTag(LogTag_MachineTimedOut, "MachineTimedOut", LogTagType_UInt32),
+DeclareTag(LogTag_Federator, "Federator",      LogTagType_String),
+DeclareTag(LogTag_RawUrl, "RawUrl",      LogTagType_VarString),
+DeclareTag(LogTag_StatusCode, "StatusCode",      LogTagType_UInt32),
+
+// ISN Total Time
+DeclareTag(LogTag_ISNMaxTotalMachine, "ISNMaxTotalMachine", LogTagType_String),
+DeclareTag(LogTag_ISNMaxTotalLatency, "ISNMaxTotalLatency", LogTagType_Float),
+DeclareTag(LogTag_ISNMaxTotalStartTime, "ISNMaxTotalStartTime", LogTagType_Time),
+DeclareTag(LogTag_ISNAvgTotalLatency, "ISNAvgTotalLatency", LogTagType_Float),
+
+// ISN Queue Time
+DeclareTag(LogTag_ISNMaxQueueMachine, "ISNMaxQueueMachine", LogTagType_String),
+DeclareTag(LogTag_ISNMaxQueueLatency, "ISNMaxQueueLatency", LogTagType_Float),
+DeclareTag(LogTag_ISNMaxQueueStartTime, "ISNMaxQueueStartTime", LogTagType_Time),
+DeclareTag(LogTag_ISNAvgQueueLatency, "ISNAvgQueueLatency", LogTagType_Float),
+
+// ISN Ranker Time
+DeclareTag(LogTag_ISNMaxRankerMachine, "ISNMaxRankerMachine", LogTagType_String),
+DeclareTag(LogTag_ISNMaxRankerLatency, "ISNMaxRankerLatency", LogTagType_Float),
+DeclareTag(LogTag_ISNMaxRankerStartTime, "ISNMaxRankerStartTime", LogTagType_Time),
+DeclareTag(LogTag_ISNAvgRankerLatency, "ISNAvgRankerLatency", LogTagType_Float),
+
+// ISN Pages Scored Time
+DeclareTag(LogTag_ISNMaxPagesScoredMachine, "ISNMaxPagesScoredMachine", LogTagType_String),
+DeclareTag(LogTag_ISNMaxPagesScored, "ISNMaxPagesScored", LogTagType_UInt64),
+DeclareTag(LogTag_ISNMaxPagesScoredStartTime, "ISNMaxPagesScoredStartTime", LogTagType_Time),
+DeclareTag(LogTag_ISNAvgPagesScored, "ISNAvgPagesScored", LogTagType_Float),
+
+// ISN Pages Matched Time
+DeclareTag(LogTag_ISNMaxPagesMatchedMachine, "ISNMaxPgsMatchedMachine", LogTagType_String),
+DeclareTag(LogTag_ISNMaxPagesMatched, "ISNMaxPgsMatched", LogTagType_UInt64),
+DeclareTag(LogTag_ISNMaxPagesMatchedStartTime, "ISNMaxPgsMatchedStartTime", LogTagType_Time),
+DeclareTag(LogTag_ISNAvgPagesMatched, "ISNAvgPgsMatched", LogTagType_Float),
+
+// ISN Pages In Corpus Time
+DeclareTag(LogTag_ISNMaxPagesInCorpusMachine, "ISNMaxPgsInCorpusMachine", LogTagType_String),
+DeclareTag(LogTag_ISNMaxPagesInCorpus, "ISNMaxPgsInCorpus", LogTagType_UInt64),
+DeclareTag(LogTag_ISNMaxPagesInCorpusStartTime, "ISNMaxPgsInCorpusStartTime", LogTagType_Time),
+DeclareTag(LogTag_ISNAvgPagesInCorpus, "ISNAvgPagesInCorpus", LogTagType_Float),
+
+// ISN QueueLength Length
+DeclareTag(LogTag_ISNMaxQueueLengthMachine, "ISNMaxQueueLengthMachine", LogTagType_String),
+DeclareTag(LogTag_ISNMaxQueueLength, "ISNMaxQueueLength", LogTagType_Float),
+DeclareTag(LogTag_ISNMaxQueueLengthStartTime, "ISNMaxQueueLengthStartTime", LogTagType_Time),
+DeclareTag(LogTag_ISNAvgQueueLength, "ISNAvgQueueLength", LogTagType_Float),
+
+// Reissue
+DeclareTag(LogTag_ReissueLatency, "ReissueLatency", LogTagType_Float),
+DeclareTag(LogTag_ReissueCount, "ReissueCount", LogTagType_UInt32),
+DeclareTag(LogTag_MaxCDLatency, "MaxCDLatency", LogTagType_Float),
+DeclareTag(LogTag_MachineAnswered, "MachineAnswered", LogTagType_UInt32),
+DeclareTag(LogTag_MachineQueried, "MachineQueried", LogTagType_UInt32),
+DeclareTag(LogTag_MaxTierQueried, "MaxTierQueried", LogTagType_UInt32),
+
+// Version
+DeclareTag(LogTag_CDVersion, "CDVersion", LogTagType_UInt32),
+DeclareTag(LogTag_CDVersionCount, "CDVersionCount", LogTagType_UInt32),
+DeclareTag(LogTag_IndexVersion, "IndexVersion", LogTagType_UInt64),
+DeclareTag(LogTag_RankVersion, "RankVersion", LogTagType_UInt64),
+
+// Watchdog
+DeclareTag(LogTag_Property, "Property", LogTagType_String),
+DeclareTag(LogTag_Level, "Level", LogTagType_String),
+DeclareTag(LogTag_Machinename, "machinename", LogTagType_String),
+
+// Caching
+DeclareTag(LogTag_HitCount, "HitCount", LogTagType_UInt64),
+DeclareTag(LogTag_TotalEstimatedMatches, "TotalEstimatedMatches", LogTagType_UInt64),
+
+// fex filter tags
+DeclareTag(LogTag_Mkt, "Mkt", LogTagType_String),
+DeclareTag(LogTag_Flight, "Flight", LogTagType_String),
+DeclareTag(LogTag_Brand, "Brand", LogTagType_String),
+DeclareTag(LogTag_VariantID, "Variant", LogTagType_UInt32),
+
+DeclareTag(LogTag_RequestTime, "RequestTime", LogTagType_String),
+DeclareTag(LogTag_Method, "Method", LogTagType_String),
+DeclareTag(LogTag_Host, "Host", LogTagType_String),
+DeclareTag(LogTag_BytesRecv, "BytesRecv", LogTagType_UInt32),
+DeclareTag(LogTag_BytesSent, "BytesSent", LogTagType_UInt32),
+DeclareTag(LogTag_BlockedStatus, "BlockedStatus",      LogTagType_UInt32),
+DeclareTag(LogTag_RequestCost, "RequestCost",      LogTagType_UInt32),
+DeclareTag(LogTag_CacheHeader, "CacheHeader", LogTagType_String),
+DeclareTag(LogTag_Latency, "Latency", LogTagType_Float),
+DeclareTag(LogTag_UserAgent, "UserAgent", LogTagType_String),
+DeclareTag(LogTag_Referer, "Referer", LogTagType_String),
+DeclareTag(LogTag_Cookies, "Cookies", LogTagType_String),
+DeclareTag(LogTag_GetHeaderXUpSubno, "GetHeaderXUpSubno", LogTagType_String),
+DeclareTag(LogTag_GetHeaderXUpUpLink, "GetHeaderXUpUpLink", LogTagType_String),
+
+DeclareTag(LogTag_RSLMsg, "Msg", LogTagType_String),
+DeclareTag(LogTag_RSLMsgLen, "MsgLen", LogTagType_UInt32),
+DeclareTag(LogTag_Offset, "Offset", LogTagType_UInt64),
+DeclareTag(LogTag_RSLState, "State", LogTagType_UInt32),
+DeclareTag(LogTag_RSLMemberId, "MemberId", LogTagType_UInt64),
+DeclareTag(LogTag_RSLBallotId, "BallotId", LogTagType_UInt32),
+DeclareTag(LogTag_RSLDecree, "Decree", LogTagType_UInt64),
+DeclareTag(LogTag_RSLBallot, "Ballot", LogTagType_String),
+DeclareTag(LogTag_RSLMsgVersion, "MsgVersion", LogTagType_UInt32),
+
+DeclareTag(LogTag_OldIndexVersion, "OldIndexVersion", LogTagType_UInt64),
+DeclareTag(LogTag_NewIndexVersion, "NewIndexVersion", LogTagType_UInt64),
+
+// Clusterbuilder
+DeclareTag(LogTag_NumArticlesLoaded, "NumArticlesLoaded", LogTagType_UInt32),
+DeclareTag(LogTag_NumArticlesRefs, "NumArticleRefs", LogTagType_UInt32),
+DeclareTag(LogTag_NumArticles, "NumArticles", LogTagType_UInt32),
+DeclareTag(LogTag_NumClusterRefs, "NumClusterRefs", LogTagType_UInt32),
+DeclareTag(LogTag_NumClusters, "NumClusters", LogTagType_UInt32),
+DeclareTag(LogTag_NumLanguageModelRowRefs, "NumLanguageModelRowRefs", LogTagType_UInt32),
+DeclareTag(LogTag_NumLanguageModelRows, "NumLanguageModelRows", LogTagType_UInt32),
+DeclareTag(LogTag_NumChunksExpired, "NumChunksExpired", LogTagType_UInt32),
+DeclareTag(LogTag_NumClustersExpired, "NumClustersExpired", LogTagType_UInt32),
+DeclareTag(LogTag_NumArticlesExpired, "NumArticlesExpired", LogTagType_UInt32),
+DeclareTag(LogTag_NumStopTokensLoaded, "NumStopTokensLoaded", LogTagType_UInt32),
+DeclareTag(LogTag_NumNoClusterTokensLoaded, "NumNoClusterTokensLoaded", LogTagType_UInt32),
+
+DeclareTag(LogTag_PrefixString, "Prefix", LogTagType_String),
+DeclareTag(LogTag_PrefixOptions, "PrefixOptions", LogTagType_Hex64),
+
+// ids for fex (latencies in ms)
+DeclareTag(LogTag_FederationLatency,  "FederationLatency",  LogTagType_Float),
+DeclareTag(LogTag_HttpSysLatency,  "HttpSysLatency",  LogTagType_Float),
+DeclareTag(LogTag_TotalLatency,  "TotalLatency",  LogTagType_Float),
+
+// Dryad
+DeclareTag(LogTag_Cluster, "Cluster", LogTagType_String),
+DeclareTag(LogTag_Namespace, "Namespace", LogTagType_String),
+DeclareTag(LogTag_NodeName, "Node", LogTagType_String),
+DeclareTag(LogTag_ServiceType, "SvcType", LogTagType_String),
+
+DeclareTag(LogTag_Command, "Command", LogTagType_String),
+DeclareTag(LogTag_Service, "Service", LogTagType_String),
+
+DeclareTag(LogTag_OID, "OID", LogTagType_Hex64),
+DeclareTag(LogTag_EID, "ExtentID", LogTagType_String),
+DeclareTag(LogTag_RefCount, "RefCount", LogTagType_UInt32),
+
+DeclareTag(LogTag_RemoteMachine,            "RemoteMachine",           LogTagType_String ),
+
+// Other (non-cosmos) tags
+DeclareTag(LogTag_APProxyCommandID, "ID", LogTagType_UInt64),
+
+//Logging for Fex C Logs
+DeclareTag(LogTag_QLocation, "QLoc", LogTagType_String),
+DeclareTag(LogTag_QLatitude, "QLat", LogTagType_Float),
+DeclareTag(LogTag_QLongitude, "QLong", LogTagType_Float),
+
+//Windows Live Searchpane Action tracing
+DeclareTag(LogTag_P4_ActionID, "ActionID", LogTagType_UInt32),
+DeclareTag(LogTag_P4_SessionID, "SessionID", LogTagType_String),
+DeclareTag(LogTag_P4_ActionTime, "ActionTime", LogTagType_String),
+DeclareTag(LogTag_P4_ResultType, "ResultType", LogTagType_UInt32),
+DeclareTag(LogTag_P4_Market, "Market", LogTagType_String),
+DeclareTag(LogTag_P4_SearchSource, "SearchSource", LogTagType_UInt32),
+DeclareTag(LogTag_P4_TargetPage, "TargetPage", LogTagType_UInt32),
+DeclareTag(LogTag_P4_ExceptionID, "ExceptionID", LogTagType_UInt64),
+DeclareTag(LogTag_P4_SearchTerm, "SearchTerm", LogTagType_String),
+DeclareTag(LogTag_P4_ActiveURL, "ActiveURL", LogTagType_String),
+DeclareTag(LogTag_P4_ExceptionMessage, "ExceptionMessage", LogTagType_String),
+DeclareTag(LogTag_P4_QuickSearch, "QuickSearch", LogTagType_UInt32),
+DeclareTag(LogTag_P4_AnswerType, "AnswerType", LogTagType_String),
+DeclareTag(LogTag_P4_TutorialMode, "TutorialMode", LogTagType_UInt32),
+
+//AppID Tracing for SOAP API
+DeclareTag(LogTag_AppID, "AppID", LogTagType_String),
+
+//Logging of Reverse IP Loc, Lat and Long for Fex C Logs
+DeclareTag(LogTag_IPLocation, "IPLoc", LogTagType_String),
+DeclareTag(LogTag_IPLatitude, "IPLat", LogTagType_Float),
+DeclareTag(LogTag_IPLongitude, "IPLong", LogTagType_Float),
+
+//adding one more latency type for logging
+DeclareTag(LogTag_DPSLatency,  "DPSLatency",  LogTagType_Float),
+DeclareTag(LogTag_RawQuery, "RawQuery", LogTagType_String),
+
+// Answers stuff
+DeclareTag(LogTag_Market,  "Market",  LogTagType_String),
+DeclareTag(LogTag_Environment,  "Environment",  LogTagType_String),
+DeclareTag(LogTag_QueryTokenID,  "QueryTokenID",  LogTagType_UInt64),
+DeclareTag(LogTag_GrammarTokenID,  "GrammarTokenID",  LogTagType_UInt64),
+DeclareTag(LogTag_AnswerRequest,  "AnsRequest", LogTagType_String),
+DeclareTag(LogTag_AnswerResponse,  "AnsResponse", LogTagType_String),
+DeclareTag(LogTag_AnswerServiceStatus,  "AnsServiceStatus", LogTagType_String),
+DeclareTag(LogTag_AnswerLogVersion,  "LogVersion", LogTagType_UInt32),
+DeclareTag(LogTag_MatchDuration,  "MatchDuration", LogTagType_UInt32),
+DeclareTag(LogTag_FulfillDuration,  "FulfillDuration", LogTagType_UInt32),
+
+// Latencies for C-logs
+DeclareTag(LogTag_TotalLatency_C,      "TotLat",  LogTagType_Float),
+DeclareTag(LogTag_DPSLatency_C,        "DPSLat",  LogTagType_Float),
+DeclareTag(LogTag_HttpSysLatency_C,    "HttpSysLat",  LogTagType_Float),
+DeclareTag(LogTag_FederationLatency_C, "FedLat",  LogTagType_Float),
+DeclareTag(LogTag_FEXLatency_C,        "FEXLat",  LogTagType_Float),
+DeclareTag(LogTag_Latency_C,           "Latency", LogTagType_Float),
+
+//General CLogging
+DeclareTag(LogTag_CLogVersion,     "CLogVersion", LogTagType_String),
+DeclareTag(LogTag_FEXBuild,        "FexBuild", LogTagType_String),
+DeclareTag(LogTag_DataCenter,      "DataCenter", LogTagType_String),
+
+// Speller request parameters
+DeclareTag(LogTag_SpellerTimeout,          "Timeout",           LogTagType_UInt32),
+DeclareTag(LogTag_SpellerTargetCorrection, "TargetCorrection",  LogTagType_String),
+DeclareTag(LogTag_SpellerConfig,           "Config",            LogTagType_String),
+DeclareTag(LogTag_SpellerOptions,          "Options",           LogTagType_Hex64),
+
+// Tag to identify spilling in indexserve
+DeclareTag(LogTag_SpillStatus,             "SpillStatus",     LogTagType_UInt32),
+
+// Tags for TLA Query Log
+DeclareTag(LogTag_ISNSourceEnvironment, "ISNSourceEnvironment",   LogTagType_String),
+DeclareTag(LogTag_CDGSourceEnvironment, "CDGSourceEnvironment",   LogTagType_String),
+DeclareTag(LogTag_LocalEnvironment,     "LocalEnvironment",       LogTagType_String),
+DeclareTag(LogTag_NumDocuments,         "NumDocuments",           LogTagType_UInt32),
+DeclareTag(LogTag_NumDocumentsActual,   "NumDocumentsActual",     LogTagType_UInt32),
+DeclareTag(LogTag_Tier1ISNLatency,      "Tier1ISNLatency",        LogTagType_Float),
+DeclareTag(LogTag_Tier2ISNLatency,      "Tier2ISNLatency",        LogTagType_Float),
+
+// tag for clusterbuilder
+DeclareTag(LogTag_NumNoAutotermTokensLoaded, "NumNoAutotermTokens", LogTagType_UInt32),
+DeclareTag(LogTag_NumEntityGazetteersLoaded, "NumEntityGazetteersLoaded", LogTagType_UInt32),
+DeclareTag(LogTag_NumPeakTermsLoaded, "NumPeakTerms", LogTagType_UInt32),
+DeclareTag(LogTag_NumHighCTRTermsLoaded, "NumHighCTRTerms", LogTagType_UInt32),
+DeclareTag(LogTag_NumDailyPeakTermsLoaded, "NumDailyHighCTRTerms", LogTagType_UInt32),
+
+// Answers Performance Monitor Results
+DeclareTag(LogTag_APM_ExecutionTime,       "ExecutionTime",        LogTagType_String ),
+DeclareTag(LogTag_APM_TestNumber,          "TestNumber",           LogTagType_String ),
+DeclareTag(LogTag_APM_ExpectedAnswer,      "ExpectedAnswer",       LogTagType_String ),
+DeclareTag(LogTag_APM_ExpectedScenario,    "ExpectedScenario",     LogTagType_String ),
+DeclareTag(LogTag_APM_Environment,         "Environment",          LogTagType_String ),
+DeclareTag(LogTag_APM_HostName,            "HostName",             LogTagType_String ),
+DeclareTag(LogTag_APM_Port,                "Port",                 LogTagType_String ),
+DeclareTag(LogTag_APM_APMQueryResult,      "APMQueryResult",       LogTagType_String ),
+DeclareTag(LogTag_APM_AnswersResponseCode, "AnswersResponseCode",  LogTagType_String ),
+DeclareTag(LogTag_APM_Latency,             "Latency",              LogTagType_UInt32 ),
+DeclareTag(LogTag_APM_ActualAnswer,        "ActualAnswer",         LogTagType_String ),
+DeclareTag(LogTag_APM_ProductionID,        "ProductionID",         LogTagType_String ),
+DeclareTag(LogTag_APM_GrammarID,           "GrammarID",            LogTagType_String ),
+DeclareTag(LogTag_APM_DataSet,             "DataSet",              LogTagType_String ),
+DeclareTag(LogTag_APM_DataSetVersion,      "DataSetVersion",       LogTagType_String ),
+DeclareTag(LogTag_APM_ActualScenario,      "ActualScenario",       LogTagType_String ),
+DeclareTag(LogTag_APM_OutputVersion,       "Version",              LogTagType_UInt32 ),
+DeclareTag(LogTag_APM_AlertStatus,         "AlertStatus",          LogTagType_UInt32 ),
+DeclareTag(LogTag_APM_SuccessRate,         "SuccessRate",          LogTagType_UInt32 ),
+// APM v2 XML blob is all one string.
+DeclareTag(LogTag_APM_TestResultXML,       "TestResultXML",        LogTagType_String ),
+
+// Speller Debugging Tags
+DeclareTag(LogTag_Speller_Query,            "Query",                   LogTagType_String),
+DeclareTag(LogTag_Speller_Status,           "PostWebSpellerStatus",    LogTagType_UInt32),
+DeclareTag(LogTag_Speller_QueryType,        "QueryType",               LogTagType_UInt32),
+DeclareTag(LogTag_Speller_Flags,            "SpellerFlags",            LogTagType_UInt32),
+DeclareTag(LogTag_Speller_NumSuggestions,   "NumSuggestions",          LogTagType_UInt32),
+DeclareTag(LogTag_Speller_Suggestion,       "Suggestion",              LogTagType_String),
+DeclareTag(LogTag_Speller_PPLatency,        "PreprocessLatency",       LogTagType_Float),
+DeclareTag(LogTag_Speller_URLDetectLatency, "URLDetectLatency",        LogTagType_Float),
+DeclareTag(LogTag_Speller_CandGenLatency,   "CandGenLatency",          LogTagType_Float),
+DeclareTag(LogTag_Speller_ViterbiLatency,   "ViterbiLatency",          LogTagType_Float),
+DeclareTag(LogTag_Speller_LMLatency,        "LMLatency",               LogTagType_Float),
+DeclareTag(LogTag_Speller_URLCheckLatency,  "URLCheckLatency",         LogTagType_Float),
+DeclareTag(LogTag_Speller_ConfLatency,      "ConfLatency",             LogTagType_Float),
+DeclareTag(LogTag_Speller_PreWebLatency,    "PreWebSpellerLatency",    LogTagType_Float),
+DeclareTag(LogTag_Speller_PostWebLatency,   "PostWebSpellerLatency",   LogTagType_Float),
+
+// More Tags for TLA Query Log: request info from users
+DeclareTag(LogTag_RequestIP,                "RequestIP",               LogTagType_String),
+DeclareTag(LogTag_RequestMethod,            "RequestMethod",           LogTagType_String),
+DeclareTag(LogTag_RequestDomain,            "RequestDomain",           LogTagType_String),
+DeclareTag(LogTag_RequestUrl,               "RequestUrl",              LogTagType_String),
+DeclareTag(LogTag_RequestReferer,           "RequestReferer",          LogTagType_String),
+DeclareTag(LogTag_RequestAppID,             "RequestAppID",            LogTagType_String),
+DeclareTag(LogTag_RequestPort,              "RequestPort",             LogTagType_UInt32),
+
+// Log Service
+DeclareTag(LogTag_IG,                       "IG",                      LogTagType_String),
+
+// Tags for TLAQueryStatJoin Log
+DeclareTag(LogTag_TLATraceID,          "TLATraceID",             LogTagType_String),
+DeclareTag(LogTag_CacheTraceID,        "CacheTraceID",           LogTagType_String),
+// tag for cache service
+DeclareTag(LogTag_CacheStatus,         "CacheStatus",            LogTagType_Hex64),
+DeclareTag(LogTag_CacheID,             "CacheID",                LogTagType_String),
+
+// Vox Populi (answers) service tags
+DeclareTag(LogTag_VoxPopuli_Rank,           "Rank",                    LogTagType_Float),
+DeclareTag(LogTag_VoxPopuli_Pos,            "Pos",                     LogTagType_UInt32),
+
+DeclareTag(LogTag_VarQueryString,           "Q",                       LogTagType_VarString),
+DeclareTag(LogTag_VarMarket,                "Market",                  LogTagType_VarString),
+DeclareTag(LogTag_AnswerService,            "Service",                 LogTagType_VarString),
+DeclareTag(LogTag_AnswerScenario,           "Scenario",                LogTagType_VarString),
+DeclareTag(LogTag_AnswerFeed,               "Feed",                    LogTagType_VarString),
+DeclareTag(LogTag_AnswerEffectiveConstraint,"AnswerEffectiveConstraint",LogTagType_VarString),
+DeclareTag(LogTag_EffectiveConstraint,      "EffectiveConstraint",     LogTagType_VarString),
+DeclareTag(LogTag_AnswerEffectiveFlight,    "AnswerEffectiveFlight",   LogTagType_VarString),
+DeclareTag(LogTag_EffectiveFlight,          "EffectiveFlight",         LogTagType_VarString),
+DeclareTag(LogTag_Time1,                    "Time1",                   LogTagType_Time),
+DeclareTag(LogTag_Time2,                    "Time2",                   LogTagType_Time),
+
+// TLA Query Log Tags
+DeclareTag(LogTag_RemoteFcsLatency,         "RemoteFcsLatency",        LogTagType_Float),
+DeclareTag(LogTag_RemoteFcsNetworkLatency,  "RemoteFcsNetworkLatency", LogTagType_Float),
+
+// tag for tiers queried within request
+DeclareTag(LogTag_TiersQueried,        "TiersQueried",                 LogTagType_String),
+
+// IDs for alterations
+DeclareTag(LogTag_AlterationName,          "AlterationName",           LogTagType_String),
+
+DeclareTag(LogTag_Mean,                     "Mean",                    LogTagType_Float),
+DeclareTag(LogTag_WeightedMean,             "WeightedMean",            LogTagType_Float),
+DeclareTag(LogTag_StdDev,                   "StdDev",                  LogTagType_Float),
+DeclareTag(LogTag_Samples,                  "Samples",                 LogTagType_Int32),
+DeclareTag(LogTag_Metric,                   "Metric",                  LogTagType_String),
+DeclareTag(LogTag_MetricParameters,         "MetricParameters",        LogTagType_String),
+
+DeclareTag(LogTag_VarFlight,                "Flight",                  LogTagType_VarString),
+
+DeclareTag(LogTag_FcsResultsNumber,        "FcsResultsNumber",         LogTagType_UInt64),
+DeclareTag(LogTag_QueryResults,            "QueryResults",             LogTagType_String),
+DeclareTag(LogTag_DuplicatedQueryType,     "DuplicatedQueryType",      LogTagType_UInt32),
+
+//
+// Additional speller latencies for experimentation
+//
+DeclareTag(LogTag_Speller_RemoveDupCandLat,        "RemoveDupCandLat",         LogTagType_Float),
+DeclareTag(LogTag_Speller_PopCandLat,              "PopCandLat",               LogTagType_Float),
+DeclareTag(LogTag_Speller_FiltCandLat,             "FiltCandLat",              LogTagType_Float),
+DeclareTag(LogTag_Speller_BldLatticeLat,           "BldLatticeLat",            LogTagType_Float),
+DeclareTag(LogTag_Speller_CalcBestPathLat,         "CalcBestPathLat",          LogTagType_Float),
+DeclareTag(LogTag_Speller_PrnLatticeLat,           "PrnLatticeLat",            LogTagType_Float),
+DeclareTag(LogTag_Speller_GetTopPathsLat,          "GenTopPathsLat",           LogTagType_Float),
+DeclareTag(LogTag_Speller_ScorePathLat,            "ScorePathLat",             LogTagType_Float),
+DeclareTag(LogTag_Speller_TokenizeLat,             "TokenizeLat",              LogTagType_Float),
+DeclareTag(LogTag_Speller_FcsParseLat,             "FcsParseLat",              LogTagType_Float),
+DeclareTag(LogTag_Speller_GenCandLat,              "GenCandLat",               LogTagType_Float),
+DeclareTag(LogTag_Speller_PreCheckLat,             "PreCheckLat",              LogTagType_Float),
+DeclareTag(LogTag_Speller_JaJpSpellCheckLat,       "JaJpSpellCheckLat",        LogTagType_Float),
+DeclareTag(LogTag_Speller_PreSpellCheckLat,        "PreSpellCheckLat",         LogTagType_Float),
+DeclareTag(LogTag_Speller_SpellCheckLat,           "SpellCheckLat",            LogTagType_Float),
+DeclareTag(LogTag_Speller_PostSpellCheckLat,       "PostSpellCheckLat",        LogTagType_Float),
+DeclareTag(LogTag_Speller_PhraseCheckLat1,         "PhraseCheckLat1",          LogTagType_Float),
+DeclareTag(LogTag_Speller_PhraseCheckLat2,         "PhraseCheckLat2",          LogTagType_Float),
+DeclareTag(LogTag_Speller_PhraseCheckLat3,         "PhraseCheckLat3",          LogTagType_Float),
+DeclareTag(LogTag_Speller_PhraseCheckLat4,         "PhraseCheckLat4",          LogTagType_Float),
+DeclareTag(LogTag_Speller_PhraseCheckLat5,         "PhraseCheckLat5",          LogTagType_Float),
+DeclareTag(LogTag_Speller_PhraseCheckLat6,         "PhraseCheckLat6",          LogTagType_Float),
+
+//
+// Tag ids for Commerce Answer Service (CAS) metadata logging
+//
+DeclareTag(LogTag_CAS_ProductID,            "ProductID",                     LogTagType_VarString),
+DeclareTag(LogTag_CAS_VendID,               "VendorID",                      LogTagType_VarString),
+DeclareTag(LogTag_CAS_Shingle,              "Shingle",                       LogTagType_VarString),
+DeclareTag(LogTag_CAS_MSNShopItemID,        "MSNShopItemID",                 LogTagType_VarString),
+DeclareTag(LogTag_CAS_TraceID,              "TraceID",                       LogTagType_VarString),
+DeclareTag(LogTag_CAS_PTraceID,             "PTraceID",                      LogTagType_String),
+DeclareTag(LogTag_CAS_RawQuery,             "RawQuery",                      LogTagType_VarString),
+DeclareTag(LogTag_CAS_Metadata,             "CASMetadata",                   LogTagType_String),
+DeclareTag(LogTag_CAS_CommDocType,          "CommDocType",                   LogTagType_String),
+DeclareTag(LogTag_CAS_DpID,                 "DpID",                          LogTagType_VarString),
+DeclareTag(LogTag_CAS_Category,             "Category",                      LogTagType_VarString),
+DeclareTag(LogTag_CAS_MCATId,               "MCATID",                        LogTagType_VarString),
+DeclareTag(LogTag_CAS_Scenario,             "Scenario",                      LogTagType_String),
+DeclareTag(LogTag_CAS_Brand,                "Brand",                         LogTagType_VarString),
+DeclareTag(LogTag_CAS_ProductName,          "ProductName",                   LogTagType_VarString),
+DeclareTag(LogTag_CAS_ProductLine,          "ProductLine",                   LogTagType_VarString),
+DeclareTag(LogTag_CAS_ReviewRating,         "ReviewRating",                  LogTagType_VarString),
+DeclareTag(LogTag_CAS_Title,                "Title",                         LogTagType_VarString),
+DeclareTag(LogTag_CAS_ResultPosition,       "ResultPosition",                LogTagType_Int32),
+DeclareTag(LogTag_CAS_FeatureName,          "FeatureName",                   LogTagType_VarString),
+DeclareTag(LogTag_CAS_CRFUsedToMatch,       "CRFUsedToMatch",                LogTagType_Int32),
+DeclareTag(LogTag_CAS_CRFConfidenceLevel,   "CRFConfidenceLevel",            LogTagType_Float),
+DeclareTag(LogTag_CAS_QDRConfidenceLevel,   "QDRConfidenceLevel",            LogTagType_Float),
+DeclareTag(LogTag_CAS_CRFAlteredRawQuery,   "CRFAlteredRawQuery",            LogTagType_String),
+DeclareTag(LogTag_CAS_CRFLabeledQuery,      "CRFLabeledQuery",               LogTagType_String),
+
+// WebPM Performance Monitor Results
+DeclareTag(LogTag_WebPM_CurrentEnvironment,  "CurrentEnvironment",   LogTagType_String ),
+DeclareTag(LogTag_WebPM_TargetEnvironment,   "TargetEnvironment",    LogTagType_String ),
+DeclareTag(LogTag_WebPM_TargetEnvironmentName,"TargetEnvironmentName",LogTagType_String ),
+DeclareTag(LogTag_WebPM_TargetPort,          "Port",                 LogTagType_UInt32 ),
+DeclareTag(LogTag_WebPM_RAWQuery,            "RAWQuery",             LogTagType_String ),
+DeclareTag(LogTag_WebPM_QueryResult,         "QueryResult",          LogTagType_String ),
+DeclareTag(LogTag_WebPM_ResponseStatus,      "ResponseStatus",       LogTagType_UInt64 ),
+DeclareTag(LogTag_WebPM_HttpResponseCode,    "HttpResponseCode",     LogTagType_String ),
+DeclareTag(LogTag_WebPM_E2ELatency,          "E2ELatency",           LogTagType_UInt32 ),
+DeclareTag(LogTag_WebPM_FcsCacheFindLatency, "FcsCacheFindLatency",  LogTagType_Float ),
+DeclareTag(LogTag_WebPM_FcsISNLatency,       "FcsISNLatency",        LogTagType_Float ),
+DeclareTag(LogTag_WebPM_FcsCDGLatency,       "FcsCDGLatency",        LogTagType_Float ),
+DeclareTag(LogTag_WebPM_FcsTotalLatency,     "FcsTotalLatency",      LogTagType_Float ),
+DeclareTag(LogTag_WebPM_ProductionID,        "ProductionID",         LogTagType_UInt32 ),
+DeclareTag(LogTag_WebPM_FDRSourceISNEnv,     "FDRSourceISNEnv",      LogTagType_String ),
+DeclareTag(LogTag_WebPM_FDRSourceCDGEnv,     "FDRSourceCDGEnv",      LogTagType_String ),
+DeclareTag(LogTag_WebPM_MaxTierQueried,      "MaxTierQueried",       LogTagType_UInt32 ),
+DeclareTag(LogTag_WebPM_ISNResponseFoundInCache, "ISNResponseFoundInCache", LogTagType_String ),
+DeclareTag(LogTag_WebPM_CDGResponseFoundInCache, "CDGResponseFoundInCache", LogTagType_String ),
+DeclareTag(LogTag_WebPM_OutputVersion,       "Version",              LogTagType_UInt32 ),
+DeclareTag(LogTag_WebPM_AlertStatus,         "AlertStatus",          LogTagType_UInt32 ),
+DeclareTag(LogTag_WebPM_FcsEstimatedMatches, "FcsEstimatedMatches",  LogTagType_UInt64 ),
+DeclareTag(LogTag_WebPM_FcsNumberofResults,  "FcsNumberofResults",   LogTagType_UInt64 ),
+DeclareTag(LogTag_WebPM_ExecutionTime,       "ExecutionTime",        LogTagType_String ),
+DeclareTag(LogTag_WebPM_TestNumber,          "TestNumber",           LogTagType_UInt32 ),
+DeclareTag(LogTag_WebPM_RemoteEnvironmentName,"RemoteEnvironmentName",LogTagType_String ),
+DeclareTag(LogTag_WebPM_NumberOfHops,        "NumberOfHops",         LogTagType_UInt32 ),
+
+// Moonshot information
+DeclareTag(LogTag_Docs,                     "DODR",                    LogTagType_String),
+
+DeclareTag(LogTag_FEXAnswerServiceName,        "AnswerServiceName",       LogTagType_String),
+DeclareTag(LogTag_FEXAnswerScenario,           "AnswerScenario",          LogTagType_String),
+DeclareTag(LogTag_FEXAnswerUXDisplayHint,      "AnswerUXDisplayHint",     LogTagType_String),
+
+DeclareTag(LogTag_VarConstraint,               "Constraint",              LogTagType_VarString),
+DeclareTag(LogTag_Constraint,                  "Constraint",              LogTagType_String),
+
+// Count of adult documents rendered
+DeclareTag(LogTag_AdultDocumentCount,		 "AdultDocumentCount",		LogTagType_UInt32),
+
+// User state
+DeclareTag(LogTag_ULS,                      "ULS",                     LogTagType_UInt32),
+
+DeclareTag(LogTag_IndexName,               "IndexName",                LogTagType_String),
+DeclareTag(LogTag_ResultSource,            "ResultSource",             LogTagType_String),
+DeclareTag(LogTag_MUID,                     "MUID",                    LogTagType_String),
+
+// ApSDk tags 
+DeclareTag(LogTag_ApCommMsgVersion,         "Msg",             LogTagType_String),
+DeclareTag(LogTag_ApCommSeqNo,              "SeqNo",           LogTagType_UInt64),
+
+// Performance Tracking
+DeclareTag(LogTag_LoadBalanceId,      "LoadBalanceId",     LogTagType_String),
+DeclareTag(LogTag_LoadBalanceTS,      "LoadBalanceTS",     LogTagType_String),
+
+// FEX / FrontDoor 
+DeclareTag(LogTag_FrontDoorAction,    "FDAction",          LogTagType_String),
+DeclareTag(LogTag_FullUrl,             "FullUrl",          LogTagType_String),
+
+DeclareTag(LogTag_JSON,                "JSON",              LogTagType_String),
+
+//webmaster
+DeclareTag(LogTag_Email, "Email", LogTagType_String),
+DeclareTag(LogTag_PUID, "PUID", LogTagType_String),
+DeclareTag(LogTag_UserProfileID, "UserProfileID", LogTagType_String),
+
+// Query Diagnostic
+DeclareTag(LogTag_QueryProcessLogVersion,   "LogVersion",      LogTagType_UInt32),
+DeclareTag(LogTag_QueryProcessTracking,     "QPTracking",      LogTagType_String),
+
+// This must be the final tag, and it must have type None
+DeclareTag(LogTag_End,         "End",               LogTagType_None),

DryadVertex/VertexHost/system/classlib/include/MSMutex.h

@@ -0,0 +1,101 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "RefCount.h"
+
+//JCnamespace apsdk
+//JC{
+
+class MSMutex : public RefCount {
+public:
+  CRITICAL_SECTION m_Section;
+
+  MSMutex() {
+    InitializeCriticalSection(&m_Section);
+  }
+  ~MSMutex() {
+    DeleteCriticalSection(&m_Section);
+  }
+
+  void Acquire() {
+    EnterCriticalSection(&m_Section);
+  }
+
+  void Release() {
+    LeaveCriticalSection(&m_Section);
+  }
+  
+  BOOL TryAcquire() {
+    return TryEnterCriticalSection(&m_Section);
+  }
+  
+};
+
+struct MutexLock {
+  Ptr<MSMutex>        m_Lock;
+  MutexLock(MSMutex *am) : m_Lock(am) {
+    LogAssert (m_Lock);
+    m_Lock->Acquire();
+  }
+  ~MutexLock() {
+    Release();
+  }
+  void Release() {
+    if (m_Lock) {
+      m_Lock->Release();
+      m_Lock = NULL;
+    }
+  }
+};
+    
+    
+  
+struct MutexTryLock {
+  Ptr<MSMutex>        m_Lock;
+  MutexTryLock(MSMutex * am) {
+    BOOL lockAcquired = am->TryAcquire(); 
+    if (lockAcquired)
+      m_Lock = am;
+  }
+  
+  ~MutexTryLock() {
+    Release();
+  }
+  void Release() { 
+    if (m_Lock) {
+      m_Lock->Release();
+      m_Lock = NULL;
+    }
+  }
+  bool operator!() { return (m_Lock == NULL); }
+  operator BOOL () { return (m_Lock != NULL); }
+};
+
+#define MUTEX_LOCK(_l, _m) MutexLock _l(_m)
+#define MUTEX_TRY_LOCK(_l,_m) MutexTryLock _l(_m)
+#define MUTEX_RELEASE(_l) (_l).Release()
+
+//JC} // namespace apsdk
+
+#ifdef USING_APSDK_NAMESPACE
+using namespace apsdk;
+#endif

DryadVertex/VertexHost/system/classlib/include/PropertyIds.h

@@ -0,0 +1,49 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "basic_types.h"
+
+// property type flag
+const UInt16 PropTypeMask                       = 0xc000;
+
+// PropType_Atom is a leaf property which is an element of a list or a
+// set. There may be nested properties within the leaf.
+const UInt16 PropType_Atom                      = 0x0000;
+
+// length type flag
+const UInt16 PropLengthMask                     = 0x2000;
+
+// A property with PropLength_Short has a 1-byte length field
+const UInt16 PropLength_Short                   = 0x0000;
+// A property with PropLength_Long has a 4-byte length field
+const UInt16 PropLength_Long                    = 0x2000;
+
+// mask for the remaining 13-bit namespace
+
+const UInt16 PropValueMask                      = 0x1fff;
+
+#define PROP_SHORTATOM(x_) ((x_) | PropType_Atom | PropLength_Short)
+#define PROP_LONGATOM(x_) ((x_) | PropType_Atom | PropLength_Long)
+
+// Propries for Dryad
+const UInt16 Prop_Stream_BeginTag          = PROP_SHORTATOM(0x1200);
+const UInt16 Prop_Stream_EndTag            = PROP_SHORTATOM(0x1201);

DryadVertex/VertexHost/system/classlib/include/RefCount.h

@@ -0,0 +1,119 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+#include <windows.h>
+
+class RefCount
+{
+    mutable volatile long m_Crefs;
+    public:
+    
+    RefCount(void) { m_Crefs = 0; }
+    
+    virtual ~RefCount() {}
+    
+    int UpCount(void) const
+    {
+        return InterlockedIncrement(&m_Crefs);
+    }
+    
+    int DownCount(void) const
+    {
+        int val = InterlockedDecrement(&m_Crefs);
+        if (!val) 
+	{
+            delete this;
+	}
+        return val;
+    }
+};
+
+template <class T> class Ptr
+{
+    T* m_Ptr;
+    public:
+
+    Ptr(const Ptr<T>& other) : m_Ptr(other.m_Ptr)
+    {
+        if (m_Ptr)
+        {
+            m_Ptr->UpCount();
+        }
+    }
+    
+    Ptr(T* ptr = 0) : m_Ptr(ptr)
+    {
+        if (m_Ptr)
+        {
+            m_Ptr->UpCount();
+        }
+    }
+    
+    ~Ptr(void)
+    {
+        if (m_Ptr)
+        {
+            m_Ptr->DownCount();
+        }
+    }
+
+    operator T*(void) const { return m_Ptr; }
+    
+    operator T*(void) { return m_Ptr; }
+    
+    T& operator*(void) const { return *m_Ptr; }
+    
+    T& operator*(void) { return *m_Ptr; }
+
+	T* operator->(void) const { return m_Ptr; }
+    
+    T* operator->(void) { return m_Ptr; }
+    
+    bool operator == (const T* ptr) const { return (m_Ptr == ptr); }
+    
+    bool operator == (const Ptr<T> &ptr) const { return (m_Ptr == ptr.m_Ptr); }
+    
+    bool operator != (const T * ptr) const { return (m_Ptr != ptr); }
+    
+    bool operator != (const Ptr<T> &ptr) const { return (m_Ptr != ptr.m_Ptr); }
+
+	bool operator !() const { return (m_Ptr == 0); }
+    
+    Ptr& operator=(Ptr<T> &ptr) {return operator=((T *) ptr);}
+    
+    Ptr& operator=(T* ptr)
+    {
+        if (m_Ptr != ptr)
+        {
+            if (m_Ptr) 
+            {
+                m_Ptr->DownCount(); 
+            }
+            m_Ptr = ptr; 
+            if (m_Ptr)
+            {
+                m_Ptr->UpCount();
+            }
+        }
+        return *this;
+    }
+};
+

DryadVertex/VertexHost/system/classlib/include/basic_types.h

@@ -0,0 +1,321 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#pragma warning (disable: 4619) // #pragma warning : there is no warning number 'NNNN'
+
+#pragma warning (push)
+
+//
+// Disabled warnings
+//
+
+#pragma warning (disable: 4255)
+#pragma warning (disable: 4668) // XXX is not defined as preprocessor macro, replacing with 0
+#pragma warning (disable: 4820) // 'N' bytes padding added after data member 'XXX'
+#pragma warning (disable: 4365) // conversion from 'LONG64' to 'DWORD64', signed/unsigned mismat
+#pragma warning (disable: 4548) // malloc.h(245) & STL: expression before comma has no effect; expected expression with side-effect
+#pragma warning (disable: 4995) // deprecated functions
+
+#include <Winsock2.h>
+#include <mswsock.h>
+#include <windows.h>
+#include <limits.h>
+#include <malloc.h>
+// include for DNS hostname support
+//#include <WinDNS.h>
+
+#pragma warning (pop)
+
+#if defined (ELEMENTCOUNT)
+#undef ELEMENTCOUNT
+#endif
+#define ELEMENTCOUNT(x) (sizeof(x)/sizeof(x[0]))
+
+typedef __int8 Int8;
+typedef unsigned __int8 UInt8;
+typedef __int16 Int16;
+typedef unsigned __int16 UInt16;
+typedef __int32 Int32;
+typedef unsigned __int32 UInt32;
+typedef __int64 Int64;
+typedef unsigned __int64 UInt64;
+typedef size_t Size_t;
+
+#ifdef _M_IX86
+typedef              int  xint;
+typedef unsigned     int uxint;
+#else
+typedef          __int64  xint;
+typedef unsigned __int64 uxint;
+#endif
+
+#define MAX_UINT8  ((UInt8)-1)
+#define MAX_UINT16 ((UInt16)-1)
+#define MAX_UINT32 ((UInt32)-1)
+#define MIN_INT32  ((Int32)0x80000000)
+#define MAX_INT32  ((Int32)0x7FFFFFFF)      // 2147483647
+#define MAX_UINT64 ((UInt64)-1)
+#define MAX_INT64  0x7FFFFFFFFFFFFFFFi64
+#define MAX_FLOAT  (3.402823466e+38F)
+#define MAX_SIZE_T ((size_t)((size_t)0 - (size_t)1))
+
+#define PF_I64D "%I64d"
+#define PF_I64X "%016I64x"
+#define PF_I64O "%022I64o"
+
+// A structure to represent a fixed-size chunk of
+// memory
+struct SIZED_STRING
+{
+    union
+    {
+        const UInt8 *pbData;
+        const char  *pcData;
+    };
+    size_t cbData;
+};
+
+// A helper macro for defining a SIZED_STRING as part of a constant
+#define INLINE_SIZED_STRING(str) { (const UInt8 *)str, sizeof(str) - 1 }
+
+//  Structure to store a wchar_t version of a user dictionary word in memory
+struct WCHAR_SIZED_STRING
+{
+    wchar_t* pData;
+    size_t cchData;
+};
+
+// A helper macro for defining a SIZED_STRING as part of a constant
+#define INLINE_WCHAR_SIZED_STRING(wstr) { wstr, sizeof(wstr)/sizeof(wstr[0]) - 1 }
+
+//  Structure to store a wchar_t version of a user dictionary word in memory
+struct WCHAR_SIZED_STRING_CONST
+{
+    const wchar_t* pData;
+    size_t cchData;
+};
+
+// A utility class for creating temporary SIZED_STRINGs
+class CStackSizedString : public SIZED_STRING
+{
+public:
+    // Null-terminated input
+    CStackSizedString(const char *szValue)
+    {
+        pbData = (const UInt8 *)szValue;
+        cbData = strlen(szValue);
+    }
+
+    // Name/size pair
+    CStackSizedString(
+        const UInt8 *pbValue,
+        size_t cbValue)
+    {
+        pbData = pbValue;
+        cbData = cbValue;
+    }
+
+    // Name/size pair
+    CStackSizedString(
+        const char *pcValue,
+        size_t cbValue)
+    {
+        pcData = pcValue;
+        cbData = cbValue;
+    }
+
+private:
+    // prevent heap allocation
+    void *operator new(size_t);
+};
+
+// A utility class for creating temporary WCHAR_SIZED_STRINGs
+class CStackSizedWString : public WCHAR_SIZED_STRING_CONST
+{
+public:
+    // Null-terminated input
+    CStackSizedWString(const wchar_t *wzValue)
+    {
+        pData = wzValue;
+        cchData = wcslen(wzValue);
+    }
+
+    // Name/size pair
+    CStackSizedWString(
+        const wchar_t *wzValue,
+        size_t cchValue)
+    {
+        pData = wzValue;
+        cchData = cchValue;
+    }
+
+private:
+    // prevent heap allocation
+    void *operator new(size_t);
+};
+
+//This is an interface to disallow heap construction.
+class INoHeapInstance
+{
+private:
+    void* operator new(size_t);
+};
+
+// DDWORD is used to easily access a 64bit number as both a 64bit
+//   and as two 32bit numbers
+typedef union
+{
+    struct
+    {
+        DWORD    low;
+        DWORD    high;
+    } dw;
+    DWORD64    ddw;
+} DDWORD;
+
+// critical section wrapper
+//
+class CRITSEC
+{
+private:
+    CRITICAL_SECTION m_critsec;
+
+public:
+    CRITSEC()
+    {
+        InitializeCriticalSection(&m_critsec);
+    }
+
+    ~CRITSEC()
+    {
+        DeleteCriticalSection(&m_critsec);
+    }
+
+    void    Enter()
+    {
+        EnterCriticalSection(&m_critsec);
+    }
+
+    BOOL TryEnter()
+    {
+        return TryEnterCriticalSection(&m_critsec);
+    }
+
+    void    Leave()
+    {
+        LeaveCriticalSection(&m_critsec);
+    }
+
+    DWORD SetSpinCount(DWORD spinCount = 4000)
+    {
+        return SetCriticalSectionSpinCount(&m_critsec, spinCount);
+    }
+};
+
+//Smart wrapper around CRITSEC so that we dont need to call enter/leave
+class AutoCriticalSection : public INoHeapInstance
+{
+private:
+    CRITSEC* m_pCritSec;
+public:
+    AutoCriticalSection(CRITSEC* pCritSec) : m_pCritSec(pCritSec)
+    {
+        this->m_pCritSec->Enter();
+    }
+    ~AutoCriticalSection()
+    {
+        this->m_pCritSec->Leave();
+    }
+};
+
+///////////////////////////////////
+#pragma warning(push)
+#pragma warning (disable: 4201) // nonstandard extension used : nameless struct/union
+
+typedef union tagFAInt64
+{
+    UInt64  n64;
+    LONG64  i64;
+    UInt8   nBytes[8];
+    FILETIME  ft;
+
+    struct
+    {
+        UInt32 nData;
+        UInt32 nCount;
+    };
+    struct
+    {
+        Int32  i32_low;
+        Int32  i32_hi;
+    };
+    struct
+    {
+        UInt32  n32_low;
+        UInt32  n32_hi;
+    };
+
+    tagFAInt64():n64(0){};
+    tagFAInt64( UInt64 nVal ):n64(nVal){};
+    tagFAInt64( Int64  iVal ):i64(iVal){};
+    tagFAInt64( UInt32 nHigh, UInt32 nLow ): n32_low(nLow), n32_hi(nHigh){};
+    tagFAInt64( Int32  iHigh, Int32  iLow ): i32_low(iLow), i32_hi(iHigh){};
+    tagFAInt64( DWORD  nHigh, DWORD  nLow ): n32_low(nLow), n32_hi(nHigh){};
+    tagFAInt64( const FILETIME&  rftSrc ): ft(rftSrc){};
+
+} FAInt64;  // flexible access 64 bit integer
+
+typedef union tagFAInt32
+{
+    UInt32  n32;
+    Int32   i32;
+    UInt8   nBytes[4];
+    struct
+    {
+        Int16  i16_low;
+        Int16  i16_hi;
+    };
+    struct
+    {
+        UInt16  n16_low;
+        UInt16  n16_hi;
+    };
+    tagFAInt32():n32(0){};
+    tagFAInt32( UInt32 nVal ): n32(nVal){};
+    tagFAInt32( Int32  iVal ): i32(iVal){};
+
+} FAInt32;  // flexible access 32 bit integer
+
+#define NUMELEM(p)  (sizeof(p)/sizeof((p)[0]))
+
+#pragma warning(pop)
+
+//
+// disabling a couple of warnings that show up with /Wall and are pretty much useless -- they come mostly from macros
+//
+#pragma warning (disable: 4514) // unreferenced inline function has been removed
+#pragma warning (disable: 4820) // 'N' bytes padding added after data member 'XXX'
+#pragma warning (disable: 4265) // class has virtual functions, but destructor is not virtual
+#pragma warning (disable: 4668) // XXX is not defined as preprocessor macro, replacing with 0
+#pragma warning (disable: 4711) // function XXX selected for automatic inline expansion
+#pragma warning (disable: 4548) // malloc.h(245) & STL: expression before comma has no effect; expected expression with side-effect
+#pragma warning (disable: 4127) // conditional expression is constant

DryadVertex/VertexHost/system/classlib/include/fingerprint.h

@@ -0,0 +1,54 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#pragma once
+
+#include "basic_types.h"
+#include "ms_fprint.h"
+
+class FingerPrint64
+{
+public:
+    static FingerPrint64* GetInstance();
+
+    UInt64 GetFingerPrint(
+        const void* data,
+        const size_t length
+        );
+
+    class FingerPrint64Init 
+    {
+    public:
+        FingerPrint64Init();
+        ~FingerPrint64Init();
+    private:
+        static UInt64 count;
+    };
+    static void Init();
+    static void Dispose();
+    FingerPrint64(UInt64 poly);
+    FingerPrint64();
+    ~FingerPrint64();
+private:
+    ms_fprint_data_t fp;
+    static FingerPrint64* instance;
+};
+
+static FingerPrint64::FingerPrint64Init fpInit;

DryadVertex/VertexHost/system/classlib/include/ms_fprint.h

@@ -0,0 +1,57 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+/* (c) Microsoft Corporation.  All rights reserved. */
+
+#if !defined(_MS_FPRINT_H_)
+#define _MS_FPRINT_H_
+
+#if defined(__GNUC__)
+typedef unsigned long long ms_fprint_t;
+#endif
+
+#if defined(_MSC_VER)
+typedef unsigned __int64 ms_fprint_t;
+#pragma warning(disable:4127)
+#include <stddef.h>
+#endif
+
+typedef struct ms_fprint_data_s *ms_fprint_data_t;
+/* an opaque type used to keep the data structures need to compute
+   fingerprints.  */
+
+ms_fprint_data_t ms_fprint_new ();
+/* Computes the tables needed for fingerprint manipulations. */
+
+ms_fprint_data_t ms_fprint_new (ms_fprint_t poly);
+/* Computes the tables needed for fingerprint manipulations. */
+
+ms_fprint_t ms_fprint_of (ms_fprint_data_t fp,
+			    void *data, size_t len);
+/* if fp was generated with polynomial P, and bytes
+   "data[0, ..., len-1]" contain string B,
+   return the fingerprint under P of the concatenation of B.
+   Strings are treated as polynomials.  The low-order bit in the first
+   byte is the highest degree coefficient in the polynomial.*/
+
+void ms_fprint_destroy (ms_fprint_data_t fp);
+/* discard the data associated with "fp" */
+
+#endif

DryadVertex/VertexHost/system/classlib/src/DrCriticalSection.cpp

@@ -0,0 +1,196 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#include "DrCommon.h"
+
+#pragma unmanaged
+
+void DrCriticalSectionBase::Init(
+    __in PCSTR name,
+    DWORD spinCount,
+    bool logUsage,
+    DrTimeInterval logHeldTooLongTimeout
+    )
+{
+    _name = (name != NULL) ? name : "UnknownCritSec";
+    _logUsage = logUsage;
+    _logHeldTooLongTimeoutMs = DrGetTimerMsFromInterval(logHeldTooLongTimeout);
+    _lastFunctionName = NULL;
+    _lastFileName = NULL;
+    _lastLineNumber = 0;
+    _enterTimeMs = 0;
+    LogAssert( InitializeCriticalSectionAndSpinCount( this, spinCount ) );
+}
+
+void DrCriticalSectionBase::SetCriticalSectionLoggingParameters(
+    bool logUsage,
+    DrTimeInterval logHeldTooLongTimeout
+)
+{
+    _logHeldTooLongTimeoutMs = DrGetTimerMsFromInterval(logHeldTooLongTimeout);
+    _logUsage = logUsage;
+}
+
+void DrCriticalSectionBase::SetCriticalSectionLogging(
+    bool logUsage
+)
+{
+    _logUsage = logUsage;
+}
+
+void DrCriticalSectionBase::SetCriticalSectionLogHeldTooLongTimeout(
+    DrTimeInterval logHeldTooLongTimeout
+)
+{
+    _logHeldTooLongTimeoutMs = DrGetTimerMsFromInterval(logHeldTooLongTimeout);
+}
+
+void DrCriticalSectionBase::Enter( PCSTR functionName, PCSTR fileName, UINT lineNumber )
+{
+    UInt32 tms = GetTickCount();
+
+    EnterCriticalSection( this );
+
+    _enterTimeMs = GetTickCount();
+
+    tms = _enterTimeMs - tms;
+
+    Int32 entryCount = -1;
+    Int32 contentionCount = -1;
+    if ( DebugInfo != NULL )
+    {
+        entryCount = (Int32)DebugInfo->EntryCount;
+        contentionCount = (Int32)DebugInfo->ContentionCount;
+    }
+
+    _lastFunctionName = functionName;
+    _lastFileName = fileName;
+    _lastLineNumber = lineNumber;
+
+    if ( tms > _logHeldTooLongTimeoutMs )
+    {
+        if (_lastFileName != NULL) {
+            DrLogW( "CritSect WAITED TO ENTER TOO LONG %s at %s %s(%u), entryCount=%d, contentionCount=%d, waited for %ums, addr=%08Ix",
+                _name, _lastFunctionName, _lastFileName, _lastLineNumber, entryCount, contentionCount, tms, this );
+        }
+        else
+        {
+            DrLogW( "CritSect WAITED TO ENTER TOO LONG %s, entryCount=%d, contentionCount=%d, waited for %ums, addr=%08Ix",
+                _name, entryCount, contentionCount, tms, this );
+        }
+    }
+
+    if ( _logUsage )
+    {
+        if (_lastFileName != NULL) {
+            DrLogD( "CritSect ENTER %s at %s %s(%u), entryCount=%d, contentionCount=%d, waited for %ums, addr=%08Ix",
+                _name, _lastFunctionName, _lastFileName, _lastLineNumber, entryCount, contentionCount, tms, this );
+        }
+        else
+        {
+            DrLogD( "CritSect ENTER %s at %s, entryCount=%d, contentionCount=%d, waited for %ums, addr=%08Ix",
+                _name, _lastFunctionName, entryCount, contentionCount, tms, this );
+        }
+    }
+}
+
+void DrCriticalSectionBase::Leave( PCSTR functionName, PCSTR fileName, UINT lineNumber )
+{
+    DebugLogAssert( Aquired() );
+
+    UInt32 tms = 0;
+    if ( _enterTimeMs != 0 )        // technically wrong, but it will fail to complain on 1 out of 4 billion slow locks
+    {
+        tms = GetTickCount() - _enterTimeMs;
+        _enterTimeMs = 0;
+    }
+
+    Int32 entryCount = -1;
+    Int32 contentionCount = -1;
+    if ( DebugInfo != NULL )
+    {
+        entryCount = (Int32)DebugInfo->EntryCount;
+        contentionCount = (Int32)DebugInfo->ContentionCount;
+    }
+
+    if ( tms > _logHeldTooLongTimeoutMs )
+    {
+        if (fileName != NULL)
+        {
+            if (_lastFileName != NULL)
+            {
+                DrLogW( "CritSect LEAVE, HELD TOO LONG %s at %s %s(%u) entered at %s %s(%u), entryCount=%d, contentionCount=%d, time held=%ums, addr=%08Ix",
+                    _name,  functionName, fileName, lineNumber, _lastFunctionName, _lastFileName, _lastLineNumber, entryCount, contentionCount, tms, this );
+            }
+            else
+            {
+                DrLogW( "CritSect LEAVE, HELD TOO LONG %s at %s %s(%u), entryCount=%d, contentionCount=%d, time held=%ums, addr=%08Ix",
+                    _name,  functionName, fileName, lineNumber, entryCount, contentionCount, tms, this );
+            }
+        }
+        else
+        {
+            if (_lastFileName != NULL)
+            {
+                DrLogW( "CritSect LEAVE, HELD TOO LONG %s entered at %s %s(%u), entryCount=%d, contentionCount=%d, time held=%ums, addr=%08Ix",
+                    _name,  _lastFunctionName, _lastFileName, _lastLineNumber, entryCount, contentionCount, tms, this );
+            }
+            else
+            {
+                DrLogW( "CritSect LEAVE, HELD TOO LONG %s, entryCount=%d, contentionCount=%d, time held=%ums, addr=%08Ix",
+                    _name, entryCount, contentionCount, tms, this );
+            }
+        }
+    }
+
+    if ( _logUsage )
+    {
+        if (fileName != NULL)
+        {
+            if (_lastFileName != NULL)
+            {
+                DrLogD( "CritSect LEAVE %s at %s %s(%u) entered at %s %s(%u), entryCount=%d, contentionCount=%d, time held=%ums, addr=%08Ix",
+                    _name,  functionName, fileName, lineNumber, _lastFunctionName, _lastFileName, _lastLineNumber, entryCount, contentionCount, tms, this );
+            }
+            else
+            {
+                DrLogD( "CritSect LEAVE %s at %s %s(%u), entryCount=%d, contentionCount=%d, time held=%ums, addr=%08Ix",
+                    _name,  functionName, fileName, lineNumber, entryCount, contentionCount, tms, this );
+            }
+        }
+        else
+        {
+            if (_lastFileName != NULL)
+            {
+                DrLogD( "CritSect LEAVE %s entered at %s %s(%u), entryCount=%d, contentionCount=%d, time held=%ums, addr=%08Ix",
+                    _name, _lastFunctionName, _lastFileName, _lastLineNumber, entryCount, contentionCount, tms, this );
+            }
+            else
+            {
+                DrLogD( "CritSect LEAVE %s, entryCount=%d, contentionCount=%d, time held=%ums, addr=%08Ix",
+                    _name, entryCount, contentionCount, tms, this );
+            }
+        }
+    }
+
+    LeaveCriticalSection( this );
+}
+
+

DryadVertex/VertexHost/system/classlib/src/DrError.cpp

@@ -0,0 +1,480 @@
+/*
+Copyright (c) Microsoft Corporation
+
+All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in 
+compliance with the License.  You may obtain a copy of the License 
+at http://www.apache.org/licenses/LICENSE-2.0   
+
+
+THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 
+EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF 
+TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.  
+
+
+See the Apache Version 2.0 License for specific language governing permissions and 
+limitations under the License. 
+
+*/
+
+#include "DrCommon.h"
+#include <lmerr.h>
+#include <winhttp.h>
+
+#pragma unmanaged
+
+typedef struct {
+    DrError value;
+    const char *pszDescription;
+} ErrEntry;
+
+
+typedef struct _ErrHashEntry {
+    DrError value;
+    const char *pszDescription;
+    struct _ErrHashEntry *pNext;
+} ErrHashEntry;
+
+
+static ErrEntry g_DryadErrorMap[] = {
+
+#undef DEFINE_DR_ERROR
+#undef COMMON_DR_ERRORS_DEFINED
+
+#define DEFINE_DR_ERROR(name, number, description) {name, description},
+
+#include "DrError.h"
+
+#undef DEFINE_DR_ERROR
+};
+
+
+class DrErrorTable : public DrTempStringPool
+{
+public:
+    static const DWORD k_hashTableSize = 111;
+    
+public:
+    DrErrorTable()
+    {
+        m_fInitialized = false;
+        for (int i = 0; i < k_hashTableSize; i++) {
+            m_pBucket[i] = NULL;
+        }
+        m_hModuleNetMsg = NULL;
+        m_hModuleWinHttp = NULL;
+    }
+
+    ~DrErrorTable()
+    {
+        ErrHashEntry *p;
+
+        for (int i = 0; i < k_hashTableSize; i++) {
+            while ((p = m_pBucket[i]) != NULL) {
+                m_pBucket[i] = p->pNext;
+                delete p;
+            }
+            
+        }
+        
+        if (m_hModuleNetMsg != NULL) {
+            FreeLibrary(m_hModuleNetMsg);
+        }
+
+        if (m_hModuleWinHttp != NULL) {
+            FreeLibrary(m_hModuleWinHttp);
+        }
+    }
+
+    void Lock()
+    {
+        m_lock.Enter();
+    }
+
+    void Unlock()
+    {
+        m_lock.Leave();
+    }
+    
+    // The returned error string should be freed with free();
+    //
+    // If the error code is unknown, a generic error description is returned.
+    char *GetSystemErrorText(DWORD dwError)
+    {
+        LPSTR MessageBuffer;
+        DWORD dwBufferLength;
+        HANDLE hModule = NULL;
+
+        DWORD dwUse = dwError;
+        DWORD dwNormalized = dwError;
+        if ((dwNormalized & 0xFFFF0000) == ((FACILITY_WIN32 << 16) | 0x80000000)) {
+            dwNormalized = dwNormalized & 0xFFFF;
+        }
+
+        DWORD dwFormatFlags = FORMAT_MESSAGE_ALLOCATE_BUFFER |
+            FORMAT_MESSAGE_IGNORE_INSERTS |
+            FORMAT_MESSAGE_FROM_SYSTEM ;
+
+        //
+        // If dwLastError is in the network range, 
+        //  load the message source.
+        //
+
+        if (dwNormalized >= NERR_BASE && dwNormalized <= MAX_NERR) {
+            Lock();
+            if (m_hModuleNetMsg == NULL) {
+                m_hModuleNetMsg = LoadLibraryEx(
+                    TEXT("netmsg.dll"),
+                    NULL,
+                    LOAD_LIBRARY_AS_DATAFILE
+                    );
+            }
+            Unlock();
+
+            if(m_hModuleNetMsg != NULL) {
+                dwFormatFlags |= FORMAT_MESSAGE_FROM_HMODULE;
+                hModule = m_hModuleNetMsg;
+            }
+        } else if (dwNormalized >= WINHTTP_ERROR_BASE && dwNormalized <= WINHTTP_ERROR_LAST) {
+            Lock();
+            if (m_hModuleWinHttp == NULL) {
+                m_hModuleWinHttp = LoadLibraryEx(
+                    TEXT("winhttp.dll"),
+                    NULL,
+                    LOAD_LIBRARY_AS_DATAFILE
+                    );
+            }
+            Unlock();
+
+            if(m_hModuleWinHttp != NULL) {
+                dwFormatFlags |= FORMAT_MESSAGE_FROM_HMODULE;
+                hModule = m_hModuleWinHttp;
+                dwUse = dwNormalized;
+            }
+        }
+
+        //
+        // Call FormatMessage() to allow for message 
+        //  text to be acquired from the system 
+        //  or from the supplied module handle.
+        //
+        // For perf, we assume here that all ANSI error messages are also valid UTF-8. If
+        // this turns out not to be true, we'll have to get the unicode message and convert to UTF-8
+        if ((dwBufferLength = FormatMessageA(
+                    dwFormatFlags,
+                    hModule, 
+                    dwUse,
+                    MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // default language
+                    (LPSTR) &MessageBuffer,
+                    0,
+                    NULL
+                )) != 0) {
+
+            char *pszBuffer = (char *)malloc(dwBufferLength + 1);
+            LogAssert(pszBuffer != NULL);
+            memcpy(pszBuffer, MessageBuffer, dwBufferLength+1);
+            LocalFree(MessageBuffer);
+
+            DWORD i;
+            for (i = 0; i < dwBufferLength; ++i)
+                if (pszBuffer[i] == '\r' || pszBuffer[i] == '\n')
+                    pszBuffer[i] = ' ';
+
+            return pszBuffer;
+        }
+
+
+        char *pszBuffer = (char *)malloc(64);
+        LogAssert(pszBuffer != NULL);
+        _snprintf(pszBuffer, 64, "Error code %u (0x%08x)", dwError, dwError);
+        return pszBuffer;
+    }
+
+    // The returned error string should be freed with free();
+    //
+    // If the error code is unknown, a generic error description is returned.
+/*JC
+    WCHAR *GetSystemErrorTextW(DWORD dwError)
+    {
+        LPWSTR MessageBuffer;
+        DWORD dwBufferLength;
+
+        DWORD dwFormatFlags = FORMAT_MESSAGE_ALLOCATE_BUFFER |
+            FORMAT_MESSAGE_IGNORE_INSERTS |
+            FORMAT_MESSAGE_FROM_SYSTEM ;
+
+        //
+        // If dwLastError is in the network range, 
+        //  load the message source.
+        //
+
+        if (dwError >= NERR_BASE && dwError <= MAX_NERR) {
+            Lock();
+            if (m_hModuleNetMsg == NULL) {
+                m_hModuleNetMsg = LoadLibraryEx(
+                    TEXT("netmsg.dll"),
+                    NULL,
+                    LOAD_LIBRARY_AS_DATAFILE
+                    );
+            }
+            Unlock();
+
+            if(m_hModuleNetMsg != NULL) {
+                dwFormatFlags |= FORMAT_MESSAGE_FROM_HMODULE;
+            }
+    }
+
+        //
+        // Call FormatMessage() to allow for message 
+        //  text to be acquired from the system 
+        //  or from the supplied module handle.
+        //
+
+        if ((dwBufferLength = FormatMessageW(
+                    dwFormatFlags,
+                    m_hModuleNetMsg, 
+                    dwError,
+                    MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // default language
+                    (LPWSTR) &MessageBuffer,
+                    0,
+                    NULL
+                )) != 0) {
+
+            WCHAR *pszBuffer = (WCHAR *)malloc((dwBufferLength + 1) * sizeof(WCHAR));
+            LogAssert(pszBuffer != NULL);
+            memcpy(pszBuffer, MessageBuffer, (dwBufferLength+1) * sizeof(WCHAR));
+            LocalFree(MessageBuffer);
+            return pszBuffer;
+        }
+
+
+        WCHAR*pszBuffer = (WCHAR *)malloc(64 * sizeof(WCHAR));
+        LogAssert(pszBuffer != NULL);
+        _snwprintf(pszBuffer, 64, L"Error code %u (0x%08x)", dwError, dwError);
+        return pszBuffer;
+    }
+*/
+
+    static DWORD IndexOf(DrError err) {
+        return ((DWORD)err % k_hashTableSize);
+    }
+
+    // Returns NULL if not found
+    // Must be called under lock
+    ErrHashEntry *FindEntry(DrError err)
+    {
+        DWORD dw = IndexOf(err);
+        ErrHashEntry *p = m_pBucket[dw];
+        while (p != NULL) {
+            if (p->value == err) {
+                return p;
+            }
+            p = p->pNext;
+        }
+        return NULL;
+    }
+
+    // returns ERROR_ALREADY_ASSIGNED if the error has already been defined
+    DrError AddEntry(DrError code, const char *pszDescription)
+    {
+        DrError err = DrError_Fail;
+        Lock();
+
+        if (FindEntry(code) != NULL) {
+            err = HRESULT_FROM_WIN32( ERROR_ALREADY_ASSIGNED );
+            goto done;
+        }
+        DWORD dw = IndexOf(code);
+        ErrHashEntry *p = new ErrHashEntry;
+        LogAssert(p != NULL);
+        p->value = code;
+        p->pszDescription = dupStr(pszDescription);
+        p->pNext = m_pBucket[dw];
+        m_pBucket[dw] = p;
+        err = DrError_OK;
+        
+    done:
+        Unlock();
+        return err;
+    }
+
+    // The returned error string should be freed with free();
+    //
+    // If the error code is unknown, a generic error description is returned.
+    char *GetErrorText(DrError err)
+    {
+        char *pszText = NULL;
+        Lock();
+        Initialize();
+        ErrHashEntry *p = FindEntry(err);
+        if (p != NULL) {
+            const char *pszOrig = p->pszDescription;
+            if (pszOrig != NULL) {
+                Size_t len = strlen(pszOrig)+1;
+                pszText = (char *)malloc(len);
+                memcpy(pszText, pszOrig, len);
+            }
+        }
+        Unlock();
+        if (pszText == NULL) {
+            pszText = GetSystemErrorText((DWORD) err);
+        }
+        LogAssert(pszText != NULL);
+        return pszText;
+    }
+
+/*JC
+    WCHAR *GetErrorTextW(DrError err)
+    {
+        WCHAR *pszText = NULL;
+        Lock();
+        Initialize();
+        ErrHashEntry *p = FindEntry(err);
+        if (p != NULL) {
+            const char *pszOrig = p->pszDescription;
+            if (pszOrig != NULL) {
+                DrWStr128 wstrText(pszOrig);
+                Size_t len = wstrText.GetLength()+1;
+                pszText = (WCHAR *)malloc(len * sizeof(WCHAR));
+                memcpy(pszText, wstrText.GetString(), len * sizeof(WCHAR));
+            }
+        }
+        Unlock();
+        if (pszText == NULL) {
+            pszText = GetSystemErrorTextW((DWORD) err);
+        }
+        LogAssert(pszText != NULL);
+        return pszText;
+    }
+*/
+    inline void Initialize()
+    {
+        if (!m_fInitialized) {
+            Lock();
+            AddStaticCodes();
+            m_fInitialized = true;
+            Unlock();
+        }
+    }
+    
+    void AddStaticCodes()
+    {
+        int n = sizeof(g_DryadErrorMap) / sizeof(g_DryadErrorMap[0]);
+        for (int i = 0; i < n; i++) {
+            DrError err = AddEntry(g_DryadErrorMap[i].value, g_DryadErrorMap[i].pszDescription);
+            if (err != DrError_OK) {
+                DrLogE( "DrErrorTable::AddStaticCodes - Could not add error code 0x%08x: [%s] to table, error: 0x%08x", g_DryadErrorMap[i].value, g_DryadErrorMap[i].pszDescription, err);
+            }
+            LogAssert(err == DrError_OK);
+        }
+}
+    
+private:
+    ErrHashEntry *m_pBucket[k_hashTableSize];
+    HMODULE m_hModuleNetMsg;
+    HMODULE m_hModuleWinHttp;
+    DrCriticalSection m_lock;
+    bool m_fInitialized;
+};
+
+
+DrErrorTable g_csErrorTable;
+
+// The returned error string should be freed with free();
+//
+// If the error code is unknown, a generic error description is returned.
+char *DrGetErrorText(DrError err)
+{
+    return g_csErrorTable.GetErrorText(err);
+}
+
+// The returned error string should be freed with free();
+//
+// If the error code is unknown, a generic error description is returned.
+/*JC
+WCHAR *DrGetErrorTextW(DrError err)
+{
+    return g_csErrorTable.GetErrorTextW(err);
+}
+*/
+
+void DrInitErrorTable(void)
+{
+    g_csErrorTable.Initialize();
+}
+
+extern DrError DrAddErrorDescription(DrError code, const char *pszDescription)
+{
+    return g_csErrorTable.AddEntry(code, pszDescription);
+}
+
+// The buffer must be at least 64 bytes long to guarantee a result. If the result won't fit in the buffer, a generic
+// error message is generated.
+const char *DrGetErrorDescription(DrError err, char *pBuffer, int buffLen)
+{
+    if (buffLen < 64) {
+        return "Error Description Buffer too short!";
+    }
+    char *pszText = DrGetErrorText(err);
+    LogAssert(pszText != NULL);
+    Size_t len = strlen(pszText)+1;
+    if ((int) len <= buffLen) {         // AKadatch: don't use "int". // TODO: fix this hack.
+        memcpy(pBuffer, pszText, len);
+    } else {
+        _snprintf(pBuffer, (size_t) (buffLen-1), "Error code %u (0x%08x)", err, err);
+    }
+
+    free(pszText);
+    
+    return pBuffer;
+}
+
+// The buffer must be at least 64 chars long to guarantee a result. If the result won't fit in the buffer, a generic
+// error message is generated.
+/* JC
+const WCHAR *DrGetErrorDescription(DrError err, WCHAR *pBuffer, int buffLen)
+{
+    if (buffLen < 64) {
+        return L"Error Description Buffer too short!";
+    }
+    WCHAR *pszText = DrGetErrorTextW(err);
+    LogAssert(pszText != NULL);
+    Size_t len = wcslen(pszText)+1;
+    if ((int) len <= buffLen) {         // AKadatch: don't use "int". // TODO: fix this hack.
+        memcpy(pBuffer, pszText, len * sizeof(WCHAR));
+    } else {
+        _snwprintf(pBuffer, (size_t) (buffLen-1), L"Error code %u (0x%08x)", err, err);
+    }
+
+    free(pszText);
+    
+    return pBuffer;
+}
+*/
+
+// The buffer must be at least 64 bytes long to guarantee a result. If the result won't fit in the buffer, a generic
+// error message is generated.
+DrStr& DrAppendErrorDescription(DrStr& strOut, DrError err)
+{
+    char *pszText = DrGetErrorText(err);
+    LogAssert(pszText != NULL);
+    strOut.Append(pszText);
+    free(pszText);
+    return strOut;
+}
+
+// The buffer must be at least 64 chars long to guarantee a result. If the result won't fit in the buffer, a generic
+// error message is generated.
+/* JC
+DrWStr& DrAppendErrorDescription(DrWStr& strOut, DrError err)
+{
+    WCHAR *pszText = DrGetErrorTextW(err);
+    LogAssert(pszText != NULL);
+    strOut.Append(pszText);
+    free(pszText);
+    return strOut;
+}
+*/
+
+