using Microsoft.Research.DryadLinq; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using System; using System.Collections.Generic; using System.IO; using System.Linq; namespace DryadLinqTests { public static class Config { public static string accountName = @"MyAccountName"; public static string storageKey = @"MyStorageKey"; public static string containerName = @"MyContainerName"; public static string cluster = "MyCcluster"; } public class DataGenerator { public DataGenerator() { } public static IEnumerable> CreateSimpleFileSets() { IEnumerable> data = new int[][] { new[] { 1, 2, 3, 4 }, new[] { 5, 6, 7, 8 }, new[] { 9, 10, 11, 12 }, }; return data; } public static IEnumerable> CreateGroupByReduceDataSet() { // we need quite a few elements to ensure the combiner will be activated in Stage#1 groupBy. // 33 elements per partition should suffice, but 100 per partition is safer. IEnumerable> data = new int[][] { Enumerable.Range(1,100).ToArray(), Enumerable.Range(101,100).ToArray(), }; return data; } } public class Utils { public static bool DeleteFile(string accountName, string accountKey, string containerName, string fileName, bool delSubDirs) { try { CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + accountName + ";AccountKey=" + accountKey); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(containerName); container.CreateIfNotExists(); BlobContainerPermissions containerPermissions = new BlobContainerPermissions(); containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob; container.SetPermissions(containerPermissions); if (false == delSubDirs) { CloudBlockBlob remoteFile = container.GetBlockBlobReference(fileName); remoteFile.DeleteIfExists(); } if (true == delSubDirs) { foreach (IListBlobItem item in container.ListBlobs(fileName, true)) { CloudBlockBlob blob = (CloudBlockBlob)item; blob.DeleteIfExists(); } } } catch (Exception) { return false; } return true; } public static bool FileExists(string accountName, string accountKey, string containerName, string fileName) { try { CloudStorageAccount storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=" + accountName + ";AccountKey=" + accountKey); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(containerName); container.CreateIfNotExists(); BlobContainerPermissions containerPermissions = new BlobContainerPermissions(); containerPermissions.PublicAccess = BlobContainerPublicAccessType.Blob; container.SetPermissions(containerPermissions); CloudBlockBlob remoteFile = container.GetBlockBlobReference(fileName); if (!remoteFile.Exists()) return false; } catch (Exception) { return false; } return true; } internal static DryadLinqContext MakeBasicConfig(string cluster) //??? { var context = new DryadLinqContext(cluster); try { context.JobFriendlyName = "DryadLinq_DevUnitTests"; context.CompileForVertexDebugging = true; context.JobEnvironmentVariables.Add("DummyEnvVar", "hello"); //note: this is consumed by a unit-test. if (File.Exists("Microsoft.Hpc.Linq.pdb")) // TODO: fix references { context.ResourcesToAdd.Add("Microsoft.Hpc.Linq.pdb"); } if (File.Exists("Microsoft.Hpc.Dsc.Client.pdb")) // TODO: fix references { context.ResourcesToAdd.Add("Microsoft.Hpc.Dsc.Client.pdb"); } // To prevent job from running forever, and blocking other test context.JobRuntimeLimit = (int)TimeSpan.FromMinutes(30).TotalSeconds; //config.AllowConcurrentUserDelegatesInSingleProcess = false; // If we are on Azure, we have to set the nodeGroup to "NodeRole" so that the default of "ComputeNodes" is not used // This fixes "FromEnumerableTests" on Azure which queries the active node-group. // Note also, the headnode for an azure deployment defaults to "HPCCluster" (at least from James' script) int onAzureInt = 0; string onAzureString = Environment.GetEnvironmentVariable("CCP_SCHEDULERONAZURE"); if (onAzureString != null) { int.TryParse(onAzureString, out onAzureInt); } if (onAzureInt == 1) { context.NodeGroup = "NodeRole"; } } catch (DryadLinqException) { } return context; } } }