VM Sizes merge with Azure/dev

src/ServiceManagement/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj

@@ -66,11 +66,13 @@
     <Reference Include="Microsoft.Data.Services.Client, Version=5.6.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\..\..\packages\Microsoft.Data.Services.Client.5.6.0\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
     </Reference>
-    <Reference Include="Microsoft.Hadoop.Client, Version=1.3.3.2, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.Hadoop.Client.dll</HintPath>
+    <Reference Include="Microsoft.Hadoop.Client, Version=1.4.5.0, Culture=neutral, PublicKeyToken=c66ce9294aae1300, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.Hadoop.Client.dll</HintPath>
     </Reference>
-    <Reference Include="Microsoft.HDInsight.Net.Http.Formatting, Version=1.3.3.2, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.HDInsight.Net.Http.Formatting.dll</HintPath>
+    <Reference Include="Microsoft.HDInsight.Net.Http.Formatting, Version=1.4.5.0, Culture=neutral, PublicKeyToken=c66ce9294aae1300, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.HDInsight.Net.Http.Formatting.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.IdentityModel.Clients.ActiveDirectory, Version=2.11.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
@@ -96,14 +98,21 @@
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\..\packages\Microsoft.WindowsAzure.Management.4.0.1\lib\net40\Microsoft.WindowsAzure.Management.dll</HintPath>
     </Reference>
-    <Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.3.3.2, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.3.3.2\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
+    <Reference Include="Microsoft.WindowsAzure.Management.HDInsight, Version=1.4.5.0, Culture=neutral, PublicKeyToken=c66ce9294aae1300, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.dll</HintPath>
     </Reference>
-    <Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.3.3.2, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
+    <Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Contracts, Version=1.4.5.0, Culture=neutral, PublicKeyToken=c66ce9294aae1300, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.Contracts.dll</HintPath>
     </Reference>
-    <Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.3.3.2, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
+    <Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework, Version=1.4.5.0, Culture=neutral, PublicKeyToken=c66ce9294aae1300, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.WindowsAzure.Management.HDInsight.Framework.Core, Version=1.4.5.0, Culture=neutral, PublicKeyToken=c66ce9294aae1300, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.WindowsAzure.Storage, Version=3.0.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\..\..\packages\WindowsAzure.Storage.3.0.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
@@ -224,8 +233,11 @@
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
+    <None Include="app.config" />
     <None Include="MSSharedLibKey.snk" />
-    <None Include="packages.config" />
+    <None Include="packages.config">
+      <SubType>Designer</SubType>
+    </None>
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\..\Common\Commands.Common.Test\Commands.Common.Test.csproj">

src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/AddConfigValuesCmdletTests.cs

@@ -83,6 +83,18 @@ public void CanCallTheAddConfigValuesCmdletTestsCmdlet_StormConfig()
 
         }
 
+        [TestMethod]
+        [TestCategory("CheckIn")]
+        public void CanCallTheAddConfigValuesCmdletTestsCmdlet_SparkConfig()
+        {
+            using (IRunspace runspace = this.GetPowerShellRunspace())
+            {
+                var sparkConfig = new Hashtable();
+                sparkConfig.Add("spark.fakeconfig.value", "12345");
+                RunConfigOptionstest(runspace, CmdletConstants.SparkConfig, sparkConfig, c => c.SparkConfiguration);
+            }
+        }
+
         [TestMethod]
         [TestCategory("CheckIn")]
         public void CanCallTheAddConfigValuesCmdletTestsCmdlet_HiveConfig()
@@ -373,7 +385,9 @@ public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveConfig()
                 var coreConfig = new Hashtable();
                 var yarnConfig = new Hashtable();
                 var stormConfig = new Hashtable();
+                var sparkConfig = new Hashtable();
                 stormConfig.Add("storm.fakekey", "123");
+                sparkConfig.Add("spark.fakekey", "123");
                 var clusterConfig = new AzureHDInsightConfig
                 {
                     HiveMetastore =
@@ -401,12 +415,14 @@ public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveConfig()
                             .WithParameter(CmdletConstants.CoreConfig, coreConfig)
                             .WithParameter(CmdletConstants.YarnConfig, yarnConfig)
                             .WithParameter(CmdletConstants.StormConfig, stormConfig)
+                            .WithParameter(CmdletConstants.SparkConfig, sparkConfig)
                             .Invoke();
                 AzureHDInsightConfig config = results.Results.ToEnumerable<AzureHDInsightConfig>().First();
 
                 Assert.AreEqual(config.CoreConfiguration.Count, coreConfig.Count);
                 Assert.AreEqual(config.YarnConfiguration.Count, yarnConfig.Count);
                 Assert.AreEqual(config.StormConfiguration.Count, stormConfig.Count);
+                Assert.AreEqual(config.SparkConfiguration.Count, sparkConfig.Count);
 
                 foreach (object entry in coreConfig.Keys)
                 {
@@ -432,6 +448,14 @@ public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveConfig()
                     Assert.AreEqual(stormConfig[entry], configUnderTest.Value, "value doesn't match for storm config option with name '{0}'", entry);
                 }
 
+                foreach (object entry in sparkConfig.Keys)
+                {
+                    KeyValuePair<string, string> configUnderTest =
+                        config.SparkConfiguration.FirstOrDefault(c => string.Equals(c.Key, entry.ToString(), StringComparison.Ordinal));
+                    Assert.IsNotNull(configUnderTest, "Unable to find spark config option with name '{0}'", entry);
+                    Assert.AreEqual(sparkConfig[entry], configUnderTest.Value, "value doesn't match for spark config option with name '{0}'", entry);
+                }
+
                 Assert.AreEqual(clusterConfig.HiveMetastore.DatabaseName, config.HiveMetastore.DatabaseName);
                 Assert.AreEqual(clusterConfig.HiveMetastore.SqlAzureServerName, config.HiveMetastore.SqlAzureServerName);
                 Assert.AreEqual(clusterConfig.HiveMetastore.Credential.UserName, config.HiveMetastore.Credential.UserName);

src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/CmdletHardCodes.cs

@@ -91,6 +91,7 @@ public static class CmdletConstants
         public const string StatusFolder = "StatusFolder";
         public const string YarnConfig = "Yarn";
         public const string StormConfig = "Storm";
+        public const string SparkConfig = "Spark";
         public const string HBaseConfig = "HBase";
 
         public const string StdErr = "StandardError";

src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/NewClusterCmdletTests.cs

@@ -420,7 +420,7 @@ public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_Hive_Oozie()
                             .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
                             .Invoke();
 
-                ClusterCreateParameters request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest;
+                ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest;
                 Assert.IsNotNull(request.HiveMetastore);
                 Assert.IsNotNull(request.OozieMetastore);
 
@@ -493,7 +493,7 @@ public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_Hive_Oozie_An
                             .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
                             .Invoke();
 
-                ClusterCreateParameters request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest;
+                ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest;
                 Assert.IsNotNull(request.HiveMetastore);
                 Assert.IsNotNull(request.OozieMetastore);
 
@@ -565,7 +565,7 @@ public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_ScriptAction(
                             .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword()))
                             .Invoke();
 
-                ClusterCreateParameters request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest;
+                ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest;
                 Assert.IsTrue(request.ConfigActions != null && request.ConfigActions.Count == 2);
                 Assert.IsTrue(
                     request.ConfigActions.ElementAt(0).Name == "test1" &&

src/ServiceManagement/HDInsight/Commands.HDInsight.Test/Models/Simulators/AzureHDInsightClusterManagementClientSimulator.cs

@@ -30,7 +30,7 @@ namespace Microsoft.WindowsAzure.Commands.Test.Utilities.HDInsight.Simulators
 {
     internal class AzureHDInsightClusterManagementClientSimulator : IHDInsightClient
     {
-        internal static ClusterCreateParameters LastCreateRequest;
+        internal static ClusterCreateParametersV2 LastCreateRequest;
 
         private static readonly Collection<SimulatorClusterContainer> Clusters = new Collection<SimulatorClusterContainer>
         {
@@ -176,9 +176,12 @@ public void Cancel()
         {
         }
 
+
+
         public ClusterDetails CreateCluster(ClusterCreateParameters cluster)
         {
-            Task<ClusterDetails> createTask = this.CreateClusterAsync(cluster);
+            Task<ClusterDetails> createTask = this.CreateClusterAsync(
+                new ClusterCreateParametersV2( cluster));
             createTask.Wait();
             return createTask.Result;
         }
@@ -188,7 +191,24 @@ public ClusterDetails CreateCluster(ClusterCreateParameters cluster, TimeSpan ti
             return this.CreateCluster(cluster);
         }
 
+        public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster)
+        {
+            Task<ClusterDetails> createTask = this.CreateClusterAsync(cluster);
+            createTask.Wait();
+            return createTask.Result;
+        }
+
+        public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster, TimeSpan timeout)
+        {
+            return this.CreateCluster(cluster);
+        }
+
         public Task<ClusterDetails> CreateClusterAsync(ClusterCreateParameters clusterCreateParameters)
+        {
+            return CreateClusterAsync(new ClusterCreateParametersV2(clusterCreateParameters));
+        }
+
+        public Task<ClusterDetails> CreateClusterAsync(ClusterCreateParametersV2 clusterCreateParameters)
         {
             this.LogMessage("Creating cluster '{0}' in location {1}", clusterCreateParameters.Name, clusterCreateParameters.Location);
             LastCreateRequest = clusterCreateParameters;
@@ -218,6 +238,16 @@ public void DeleteCluster(string dnsName, TimeSpan timeout)
             this.DeleteClusterAsync(dnsName).Wait();
         }
 
+        public void DeleteCluster(string dnsName, string location)
+        {
+            throw new NotImplementedException();
+        }
+
+        public void DeleteCluster(string dnsName, string location, TimeSpan timeout)
+        {
+            throw new NotImplementedException();
+        }
+
         public async Task DeleteClusterAsync(string name)
         {
             ClusterDetails cluster = await this.GetClusterAsync(name);
@@ -230,6 +260,11 @@ public async Task DeleteClusterAsync(string name)
             Clusters.Remove(GetClusterInternal(name));
         }
 
+        public Task DeleteClusterAsync(string name, string location)
+        {
+            throw new NotImplementedException();
+        }
+
         public void DisableHttp(string dnsName, string location)
         {
             this.DisableHttpAsync(dnsName, location).Wait();
@@ -265,6 +300,13 @@ public ClusterDetails GetCluster(string dnsName)
             return getTask.Result;
         }
 
+        public ClusterDetails GetCluster(string dnsName, string location)
+        {
+            Task<ClusterDetails> getTask = this.GetClusterAsync(dnsName, location);
+            getTask.Wait();
+            return getTask.Result;
+        }
+
         public async Task<ClusterDetails> GetClusterAsync(string name)
         {
             this.LogMessage("Getting hdinsight clusters for subscriptionid : {0}", this.credentials.SubscriptionId.ToString());
@@ -273,6 +315,11 @@ public async Task<ClusterDetails> GetClusterAsync(string name)
             return cluster;
         }
 
+        public Task<ClusterDetails> GetClusterAsync(string name, string location)
+        {
+            throw new NotImplementedException();
+        }
+
         public ClusterDetails ChangeClusterSize(string dnsName, string location, int newSize)
         {
             return ChangeClusterSizeAsync(dnsName, location, newSize).WaitForResult();

src/ServiceManagement/HDInsight/Commands.HDInsight.Test/app.config

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
+</configuration>

src/ServiceManagement/HDInsight/Commands.HDInsight.Test/packages.config

@@ -11,12 +11,12 @@
   <package id="Microsoft.Data.Edm" version="5.6.0" targetFramework="net45" />
   <package id="Microsoft.Data.OData" version="5.6.0" targetFramework="net45" />
   <package id="Microsoft.Data.Services.Client" version="5.6.0" targetFramework="net45" />
-  <package id="Microsoft.Hadoop.Client" version="1.3.3.2" targetFramework="net45" />
+  <package id="Microsoft.Hadoop.Client" version="1.4.5" targetFramework="net45" />
   <package id="Microsoft.IdentityModel.Clients.ActiveDirectory" version="2.11.10918.1222" targetFramework="net45" />
   <package id="Microsoft.Net.Http" version="2.2.28" targetFramework="net45" />
   <package id="Microsoft.WindowsAzure.ConfigurationManager" version="1.8.0.0" targetFramework="net45" />
   <package id="Microsoft.WindowsAzure.Management" version="4.0.1" targetFramework="net45" />
-  <package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.3.3.2" targetFramework="net45" />
+  <package id="Microsoft.WindowsAzure.Management.HDInsight" version="1.4.5" targetFramework="net45" />
   <package id="Moq" version="4.2.1402.2112" targetFramework="net45" />
   <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net45" />
   <package id="System.Spatial" version="5.6.0" targetFramework="net45" />

src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/AddAzureHDInsightConfigValuesCmdlet.cs

@@ -81,6 +81,7 @@ public AzureHDInsightConfig Config
                 this.command.Config.SubnetName = value.SubnetName;
                 this.command.Config.StormConfiguration.AddRange(value.StormConfiguration);
                 this.command.Config.HBaseConfiguration.ConfigurationCollection.AddRange(value.HBaseConfiguration.ConfigurationCollection);
+                this.command.Config.SparkConfiguration.AddRange(value.SparkConfiguration);
             }
         }
 
@@ -161,6 +162,17 @@ public Hashtable Storm
             set { this.command.Storm = value; }
         }
 
+        /// <summary>
+        ///     Gets or sets a collection of configuration properties to customize the Spark service.
+        /// </summary>
+        [SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly", Justification = "Ease of use in Powershell")]
+        [Parameter(Mandatory = false, HelpMessage = "a collection of configuration properties to customize the Spark service.")]
+        public Hashtable Spark
+        {
+            get { return this.command.Spark; }
+            set { this.command.Spark = value; }
+        }
+
         /// <summary>
         ///     Gets or sets a collection of configuration properties to customize the HBase service.
         /// </summary>