-
Notifications
You must be signed in to change notification settings - Fork 4k
Make storage data cmdlets work with ARM and ASM #1041
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
f794738
0a39e08
cd7d07f
a96738e
e7d911d
214b928
d922821
9e9422b
9fe922f
951c275
cab4d2d
2948d32
34e3c3e
dd4eb8b
463f127
bf14f68
d772790
7ee8f0c
6c1c812
eb49df1
b13ef44
4372922
1fbfe45
e2efa78
320939f
621d66f
9396469
fb993b0
edfd4f2
a368dce
fed6d76
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
// ---------------------------------------------------------------------------------- | ||
// | ||
// Copyright Microsoft Corporation | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
// ---------------------------------------------------------------------------------- | ||
|
||
using System; | ||
using System.Collections.Generic; | ||
using Microsoft.WindowsAzure.Commands.Common; | ||
using Microsoft.Azure.Common.Authentication.Models; | ||
using Microsoft.WindowsAzure.Commands.Common.Storage; | ||
using ArmStorage = Microsoft.Azure.Management.Storage; | ||
using SmStorage = Microsoft.WindowsAzure.Management.Storage; | ||
using Microsoft.WindowsAzure.Storage; | ||
using Microsoft.Azure.Common.Authentication; | ||
using Microsoft.WindowsAzure.Storage.Auth; | ||
|
||
namespace Microsoft.WindowsAzure.Commands.Utilities.Common | ||
{ | ||
public static class AzureContextExtensions | ||
{ | ||
/// <summary> | ||
/// Set the current storage account using the given connection string | ||
/// </summary> | ||
/// <param name="context">The current context.</param> | ||
/// <param name="connectionString">The connection string to check.</param> | ||
public static void SetCurrentStorageAccount(this AzureContext context, string connectionString) | ||
{ | ||
if (context.Subscription != null) | ||
{ | ||
context.Subscription.SetProperty(AzureSubscription.Property.StorageAccount, connectionString); | ||
} | ||
} | ||
|
||
/// <summary> | ||
/// Set the current storage account using the given connection string. | ||
/// </summary> | ||
/// <param name="context">The current context.</param> | ||
/// <param name="account">A storage account.</param> | ||
public static void SetCurrentStorageAccount(this AzureContext context, IStorageContextProvider account) | ||
{ | ||
if (context.Subscription != null && account != null && account.Context != null | ||
&& account.Context.StorageAccount != null) | ||
{ | ||
context.SetCurrentStorageAccount(account.Context.StorageAccount.ToString(true)); | ||
} | ||
} | ||
|
||
/// <summary> | ||
/// Get the current storage account. | ||
/// </summary> | ||
/// <param name="context">The current context.</param> | ||
/// <returns>The current storage account, or null, if no current storage account is set.</returns> | ||
public static CloudStorageAccount GetCurrentStorageAccount(this AzureContext context) | ||
{ | ||
if (context != null && context.Subscription != null) | ||
{ | ||
try | ||
{ | ||
return | ||
CloudStorageAccount.Parse( | ||
context.Subscription.GetProperty(AzureSubscription.Property.StorageAccount)); | ||
} | ||
catch | ||
{ | ||
// return null if we could not parse the connection string | ||
} | ||
} | ||
|
||
return null; | ||
} | ||
|
||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
// ---------------------------------------------------------------------------------- | ||
// | ||
// Copyright Microsoft Corporation | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
// ---------------------------------------------------------------------------------- | ||
|
||
using System; | ||
using System.Collections.Generic; | ||
using System.Linq; | ||
using System.Text; | ||
using System.Threading.Tasks; | ||
|
||
namespace Microsoft.WindowsAzure.Commands.Common.Storage | ||
{ | ||
public interface IStorageContextProvider | ||
{ | ||
AzureStorageContext Context | ||
{ | ||
get; | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
// ---------------------------------------------------------------------------------- | ||
// | ||
// Copyright Microsoft Corporation | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
// ---------------------------------------------------------------------------------- | ||
|
||
using System; | ||
using System.IO; | ||
using System.Text.RegularExpressions; | ||
|
||
namespace Microsoft.WindowsAzure.Commands.Common.Storage | ||
{ | ||
public class StorageIdentity | ||
{ | ||
private const string StorageIdentityRegex = | ||
"/subscriptions/([^/]+)/resourceGroups/([^/]+)/microsoft.storage/storageAccounts/(\\w+)"; | ||
public StorageIdentity(string identity) | ||
{ | ||
var matcher = new Regex(StorageIdentityRegex); | ||
var result = matcher.Match(identity); | ||
if (!result.Success || result.Groups == null || result.Groups.Count < 3) | ||
{ | ||
throw new InvalidOperationException(string.Format("Cannot find resource grpoup name and storage account name from resource identity {0}", identity)); | ||
} | ||
|
||
this.ResourceGroupName = result.Groups[1].Value; | ||
this.StorageAccountName = result.Groups[2].Value; | ||
} | ||
|
||
public string ResourceGroupName { get; private set; } | ||
|
||
public string StorageAccountName { get; private set; } | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,9 @@ | ||
| ||
using System.CodeDom; | ||
using System.Diagnostics.Eventing.Reader; | ||
using System.Text; | ||
using Microsoft.Azure.Management.Storage; | ||
|
||
namespace Microsoft.WindowsAzure.Commands.Common.Storage | ||
{ | ||
using System; | ||
|
@@ -8,6 +13,7 @@ namespace Microsoft.WindowsAzure.Commands.Common.Storage | |
using Microsoft.WindowsAzure.Storage.Auth; | ||
using Microsoft.WindowsAzure.Storage.Blob; | ||
using Microsoft.WindowsAzure.Storage.Table; | ||
using Arm = Microsoft.Azure.Management.Storage; | ||
|
||
public class StorageUtilities | ||
{ | ||
|
@@ -26,38 +32,143 @@ public static Uri CreateHttpsEndpoint(string endpointUri) | |
return new Uri(endpoint); | ||
} | ||
|
||
public static CloudStorageAccount GenerateCloudStorageAccount(StorageManagementClient storageClient, string accountName) | ||
/// <summary> | ||
/// Create a cloud storage account using an ARM storage management client | ||
/// </summary> | ||
/// <param name="storageClient">The client to use to get storage account details.</param> | ||
/// <param name="resourceGroupName">The resource group contining the storage account.</param> | ||
/// <param name="accountName">The name of the storage account.</param> | ||
/// <returns>A CloudStorageAccount that can be used by windows azure storage library to manipulate objects in the storage account.</returns> | ||
public static CloudStorageAccount GenerateCloudStorageAccount(Arm.IStorageManagementClient storageClient, | ||
string resourceGroupName, string accountName) | ||
{ | ||
var storageServiceResponse = storageClient.StorageAccounts.Get(accountName); | ||
var storageKeysResponse = storageClient.StorageAccounts.GetKeys(accountName); | ||
|
||
Uri fileEndpoint = null; | ||
Uri blobEndpoint = null; | ||
Uri queueEndpoint = null; | ||
Uri tableEndpoint = null; | ||
if (!TestMockSupport.RunningMocked) | ||
{ | ||
var storageServiceResponse = storageClient.StorageAccounts.GetProperties(resourceGroupName, accountName); | ||
Uri blobEndpoint = storageServiceResponse.StorageAccount.PrimaryEndpoints.Blob; | ||
Uri queueEndpoint = storageServiceResponse.StorageAccount.PrimaryEndpoints.Queue; | ||
Uri tableEndpoint = storageServiceResponse.StorageAccount.PrimaryEndpoints.Table; | ||
return new CloudStorageAccount( | ||
GenerateStorageCredentials(storageClient, resourceGroupName, accountName), | ||
blobEndpoint, | ||
queueEndpoint, | ||
tableEndpoint, null); | ||
} | ||
else | ||
{ | ||
return new CloudStorageAccount( | ||
new StorageCredentials(accountName, | ||
Convert.ToBase64String(Encoding.UTF8.GetBytes(Guid.NewGuid().ToString()))), | ||
new Uri(string.Format("https://{0}.blob.core.windows.net", accountName)), | ||
new Uri(string.Format("https://{0}.queue.core.windows.net", accountName)), | ||
new Uri(string.Format("https://{0}.table.core.windows.net", accountName)), | ||
null); | ||
} | ||
} | ||
|
||
if (storageServiceResponse.StorageAccount.Properties.Endpoints.Count >= 4) | ||
/// <summary> | ||
/// Create a cloud storage account using a service management storage client | ||
/// </summary> | ||
/// <param name="storageClient">The client to use to get storage account details.</param> | ||
/// <param name="accountName">The name of the storage account.</param> | ||
/// <returns>A CloudStorageAccount that can be used by windows azure storage library to manipulate objects in the storage account.</returns> | ||
public static CloudStorageAccount GenerateCloudStorageAccount(IStorageManagementClient storageClient, string accountName) | ||
{ | ||
if (!TestMockSupport.RunningMocked) | ||
{ | ||
fileEndpoint = StorageUtilities.CreateHttpsEndpoint(storageServiceResponse.StorageAccount.Properties.Endpoints[3].ToString()); | ||
var storageServiceResponse = storageClient.StorageAccounts.Get(accountName); | ||
|
||
Uri fileEndpoint = null; | ||
Uri blobEndpoint = null; | ||
Uri queueEndpoint = null; | ||
Uri tableEndpoint = null; | ||
|
||
if (storageServiceResponse.StorageAccount.Properties.Endpoints.Count >= 4) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: Would be good to describe what endpoint count of 4 means and what endpoints count of 3 means. |
||
{ | ||
fileEndpoint = | ||
StorageUtilities.CreateHttpsEndpoint( | ||
storageServiceResponse.StorageAccount.Properties.Endpoints[3].ToString()); | ||
} | ||
|
||
if (storageServiceResponse.StorageAccount.Properties.Endpoints.Count >= 3) | ||
{ | ||
tableEndpoint = | ||
StorageUtilities.CreateHttpsEndpoint( | ||
storageServiceResponse.StorageAccount.Properties.Endpoints[2].ToString()); | ||
queueEndpoint = | ||
StorageUtilities.CreateHttpsEndpoint( | ||
storageServiceResponse.StorageAccount.Properties.Endpoints[1].ToString()); | ||
} | ||
|
||
if (storageServiceResponse.StorageAccount.Properties.Endpoints.Count >= 1) | ||
{ | ||
blobEndpoint = | ||
StorageUtilities.CreateHttpsEndpoint( | ||
storageServiceResponse.StorageAccount.Properties.Endpoints[0].ToString()); | ||
} | ||
|
||
return new CloudStorageAccount( | ||
GenerateStorageCredentials(storageClient, storageServiceResponse.StorageAccount.Name), | ||
blobEndpoint, | ||
queueEndpoint, | ||
tableEndpoint, | ||
fileEndpoint); | ||
} | ||
|
||
if (storageServiceResponse.StorageAccount.Properties.Endpoints.Count >= 3) | ||
else | ||
{ | ||
tableEndpoint = StorageUtilities.CreateHttpsEndpoint(storageServiceResponse.StorageAccount.Properties.Endpoints[2].ToString()); | ||
queueEndpoint = StorageUtilities.CreateHttpsEndpoint(storageServiceResponse.StorageAccount.Properties.Endpoints[1].ToString()); | ||
return new CloudStorageAccount( | ||
new StorageCredentials(accountName, | ||
Convert.ToBase64String(Encoding.UTF8.GetBytes(Guid.NewGuid().ToString()))), | ||
new Uri(string.Format("https://{0}.blob.core.windows.net", accountName)), | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same comment above There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. same |
||
new Uri(string.Format("https://{0}.queue.core.windows.net", accountName)), | ||
new Uri(string.Format("https://{0}.table.core.windows.net", accountName)), | ||
new Uri(string.Format("https://{0}.file.core.windows.net", accountName))); | ||
} | ||
} | ||
|
||
if (storageServiceResponse.StorageAccount.Properties.Endpoints.Count >= 1) | ||
/// <summary> | ||
/// Create storage credentials for the given account | ||
/// </summary> | ||
/// <param name="storageClient">The ARM storage management client.</param> | ||
/// <param name="resourceGroupName">The resource group containing the storage account.</param> | ||
/// <param name="accountName">The storage account name.</param> | ||
/// <returns>Storage credentials for the given account.</returns> | ||
public static StorageCredentials GenerateStorageCredentials(Arm.IStorageManagementClient storageClient, | ||
string resourceGroupName, string accountName) | ||
{ | ||
if (!TestMockSupport.RunningMocked) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would be better if the whole class can be mocked in the tests instead of having hard-coded branching based on tests. If this class can be changed to be some interface and then we have a Mock version that we inject upon running tests that would be perfect There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed. I was thinking this when I added to this class. I think this is OK for now, but we should add in a singleton for storage utilities that we mock out for tests in the next release. I fear it would e a good deal of work to identify all the tests that used storage and do the mocking for each.right at the moment, so I will fiel an issue to do this in the next sprint. |
||
{ | ||
var storageKeysResponse = storageClient.StorageAccounts.ListKeys(resourceGroupName, accountName); | ||
return new StorageCredentials(accountName, | ||
storageKeysResponse.StorageAccountKeys.Key1); | ||
} | ||
else | ||
{ | ||
blobEndpoint = StorageUtilities.CreateHttpsEndpoint(storageServiceResponse.StorageAccount.Properties.Endpoints[0].ToString()); | ||
return new StorageCredentials(accountName, | ||
Convert.ToBase64String(Encoding.UTF8.GetBytes(Guid.NewGuid().ToString()))); | ||
} | ||
} | ||
|
||
return new CloudStorageAccount( | ||
new StorageCredentials(storageServiceResponse.StorageAccount.Name, storageKeysResponse.PrimaryKey), | ||
blobEndpoint, | ||
queueEndpoint, | ||
tableEndpoint, | ||
fileEndpoint); | ||
/// <summary> | ||
/// Create storage credentials for the given account | ||
/// </summary> | ||
/// <param name="storageClient">The RDFE storage management client.</param> | ||
/// <param name="accountName">The storage account name.</param> | ||
/// <returns>Storage credentials for the given account.</returns> | ||
public static StorageCredentials GenerateStorageCredentials(IStorageManagementClient storageClient, | ||
string accountName) | ||
{ | ||
if (!TestMockSupport.RunningMocked) | ||
{ | ||
var storageKeysResponse = storageClient.StorageAccounts.GetKeys(accountName); | ||
return new StorageCredentials(accountName, | ||
storageKeysResponse.PrimaryKey); | ||
} | ||
else | ||
{ | ||
return new StorageCredentials(accountName, | ||
Convert.ToBase64String(Encoding.UTF8.GetBytes(Guid.NewGuid().ToString()))); | ||
} | ||
} | ||
|
||
public static string GenerateTableStorageSasUrl(string connectionString, string tableName, DateTime expiryTime, SharedAccessTablePermissions permissions) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Looks scary for me that we have hard-coded endpoints here regardless of the environment. Would propose using AzureEnvironment.BlobSuffixEndpoint instead of the hard-coded value. Feel free to ignore that comment in case I miss some context/assumption that's made here.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this only occurs when running mocked tests, when the values should not matter. in production, the endpoint values from the account are used. In the context where this function is used, this data is not always available. Given that this should not matter, tabling this for now.