diff --git a/src/Microsoft.Azure.WebJobs.Host/Executors/JobHostContextFactory.cs b/src/Microsoft.Azure.WebJobs.Host/Executors/JobHostContextFactory.cs index 953203b53..0e89246f9 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Executors/JobHostContextFactory.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Executors/JobHostContextFactory.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; @@ -25,7 +24,6 @@ using Microsoft.Azure.WebJobs.Host.Storage.Queue; using Microsoft.Azure.WebJobs.Host.Timers; using Microsoft.Azure.WebJobs.Host.Triggers; -using Newtonsoft.Json.Serialization; namespace Microsoft.Azure.WebJobs.Host.Executors { @@ -117,8 +115,7 @@ public static async Task CreateAndLogHostStartedAsync( if (singletonManager == null) { - IStorageAccount storageAccount = await storageAccountProvider.GetStorageAccountAsync(cancellationToken); - singletonManager = new SingletonManager(storageAccount.CreateBlobClient(), backgroundExceptionDispatcher, config.Singleton, trace, config.NameResolver); + singletonManager = new SingletonManager(storageAccountProvider, backgroundExceptionDispatcher, config.Singleton, trace, config.NameResolver); } using (CancellationTokenSource combinedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, shutdownToken)) diff --git a/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonLock.cs b/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonLock.cs index 6ac5b85ef..4bca49553 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonLock.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonLock.cs @@ -110,7 +110,7 @@ public async Task ReleaseAsync(CancellationToken cancellationToken) /// public async Task GetOwnerAsync(CancellationToken cancellationToken) { - return await _singletonManager.GetLockOwnerAsync(_lockId, cancellationToken); + return await _singletonManager.GetLockOwnerAsync(_attribute, _lockId, cancellationToken); } } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonManager.cs b/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonManager.cs index f95951b8b..8f14a7f76 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonManager.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonManager.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; @@ -11,6 +12,7 @@ using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; using Microsoft.Azure.WebJobs.Host.Bindings.Path; +using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Azure.WebJobs.Host.Storage; using Microsoft.Azure.WebJobs.Host.Storage.Blob; using Microsoft.Azure.WebJobs.Host.Timers; @@ -28,7 +30,8 @@ internal class SingletonManager private readonly INameResolver _nameResolver; private readonly IBackgroundExceptionDispatcher _backgroundExceptionDispatcher; private readonly SingletonConfiguration _config; - private IStorageBlobDirectory _directory; + private readonly IStorageAccountProvider _accountProvider; + private ConcurrentDictionary _lockDirectoryMap = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); private TimeSpan _minimumLeaseRenewalInterval = TimeSpan.FromSeconds(1); private TraceWriter _trace; @@ -37,12 +40,11 @@ internal SingletonManager() { } - public SingletonManager(IStorageBlobClient blobClient, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, SingletonConfiguration config, TraceWriter trace, INameResolver nameResolver = null) + public SingletonManager(IStorageAccountProvider accountProvider, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, SingletonConfiguration config, TraceWriter trace, INameResolver nameResolver = null) { + _accountProvider = accountProvider; _nameResolver = nameResolver; _backgroundExceptionDispatcher = backgroundExceptionDispatcher; - _directory = blobClient.GetContainerReference(HostContainerNames.Hosts) - .GetDirectoryReference(HostDirectoryNames.SingletonLocks); _config = config; _trace = trace; } @@ -85,7 +87,8 @@ public async virtual Task LockAsync(string lockId, string functionInstan public async virtual Task TryLockAsync(string lockId, string functionInstanceId, SingletonAttribute attribute, CancellationToken cancellationToken) { - IStorageBlockBlob lockBlob = _directory.GetBlockBlobReference(lockId); + IStorageBlobDirectory lockDirectory = GetLockDirectory(attribute.Account); + IStorageBlockBlob lockBlob = lockDirectory.GetBlockBlobReference(lockId); await TryCreateAsync(lockBlob, cancellationToken); _trace.Verbose(string.Format(CultureInfo.InvariantCulture, "Waiting for Singleton lock ({0})", lockId), source: TraceSource.Execution); @@ -220,9 +223,10 @@ public static SingletonAttribute GetListenerSingletonOrNull(Type listenerType, M return singletonAttribute; } - public async virtual Task GetLockOwnerAsync(string lockId, CancellationToken cancellationToken) + public async virtual Task GetLockOwnerAsync(SingletonAttribute attribute, string lockId, CancellationToken cancellationToken) { - IStorageBlockBlob lockBlob = _directory.GetBlockBlobReference(lockId); + IStorageBlobDirectory lockDirectory = GetLockDirectory(attribute.Account); + IStorageBlockBlob lockBlob = lockDirectory.GetBlockBlobReference(lockId); await ReadLeaseBlobMetadata(lockBlob, cancellationToken); @@ -240,6 +244,27 @@ public async virtual Task GetLockOwnerAsync(string lockId, CancellationT return owner; } + internal IStorageBlobDirectory GetLockDirectory(string accountName) + { + if (string.IsNullOrEmpty(accountName)) + { + accountName = ConnectionStringNames.Storage; + } + + IStorageBlobDirectory storageDirectory = null; + if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory)) + { + Task task = _accountProvider.GetAccountAsync(accountName, CancellationToken.None); + IStorageAccount storageAccount = task.Result; + IStorageBlobClient blobClient = storageAccount.CreateBlobClient(); + storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts) + .GetDirectoryReference(HostDirectoryNames.SingletonLocks); + _lockDirectoryMap[accountName] = storageDirectory; + } + + return storageDirectory; + } + private ITaskSeriesTimer CreateLeaseRenewalTimer(IStorageBlockBlob leaseBlob, string leaseId, string lockId, TimeSpan leasePeriod, IBackgroundExceptionDispatcher backgroundExceptionDispatcher) { diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus/ServiceBusAccountAttribute.cs b/src/Microsoft.Azure.WebJobs.ServiceBus/ServiceBusAccountAttribute.cs index bb4163328..d2fe9e352 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus/ServiceBusAccountAttribute.cs +++ b/src/Microsoft.Azure.WebJobs.ServiceBus/ServiceBusAccountAttribute.cs @@ -6,7 +6,7 @@ namespace Microsoft.Azure.WebJobs { /// - /// Attribute used to override the default ServiceBus account used. + /// Attribute used to override the default ServiceBus account used by triggers and binders. /// /// /// This attribute can be applied at the parameter/method/class level, and the precedence diff --git a/src/Microsoft.Azure.WebJobs/SingletonAttribute.cs b/src/Microsoft.Azure.WebJobs/SingletonAttribute.cs index 8c0cfa6a1..61d7c3189 100644 --- a/src/Microsoft.Azure.WebJobs/SingletonAttribute.cs +++ b/src/Microsoft.Azure.WebJobs/SingletonAttribute.cs @@ -8,7 +8,7 @@ namespace Microsoft.Azure.WebJobs /// /// This attribute can be applied to a job functions to ensure that only a single /// instance of the function is executed at any given time (even across host instances). - /// Distributed locks are used to serialize invocations across all host instances. + /// A blob lease is used behind the scenes to implement the lock. /// /// This attribute can also be used in mode to ensure that /// the listener for a triggered function is only running on a single instance. Trigger bindings @@ -25,11 +25,8 @@ public sealed class SingletonAttribute : Attribute /// /// Constructs a new instance. /// - /// The this singleton should use. - /// Defaults to if not explicitly specified. - /// - public SingletonAttribute(SingletonMode mode = SingletonMode.Function) - : this(string.Empty, mode) + public SingletonAttribute() + : this(string.Empty) { } @@ -38,20 +35,12 @@ public SingletonAttribute(SingletonMode mode = SingletonMode.Function) /// /// The scope for the singleton lock. When applied to triggered /// job functions, this value can include binding parameters. - /// The this singleton should use. - /// Defaults to if not explicitly specified. - /// - public SingletonAttribute(string scope, SingletonMode mode = SingletonMode.Function) + public SingletonAttribute(string scope) { Scope = scope; - Mode = mode; + Mode = SingletonMode.Function; } - /// - /// Gets the . - /// - public SingletonMode Mode { get; private set; } - /// /// Gets the scope identifier for the singleton lock. /// @@ -61,6 +50,21 @@ public string Scope private set; } + /// + /// Gets or sets the this singleton should use. + /// Defaults to if not explicitly specified. + /// + public SingletonMode Mode { get; set; } + + /// + /// Gets the name of the Azure Storage account that the blob lease should be + /// created in. + /// + /// + /// If not specified, the default AzureWebJobs storage account will be used. + /// + public string Account { get; set; } + /// /// Gets or sets the timeout value in seconds for lock acquisition. /// If the lock is not obtained within this interval, the invocation will fail. diff --git a/src/Microsoft.Azure.WebJobs/StorageAccountAttribute.cs b/src/Microsoft.Azure.WebJobs/StorageAccountAttribute.cs index 7989872c6..9d632563c 100644 --- a/src/Microsoft.Azure.WebJobs/StorageAccountAttribute.cs +++ b/src/Microsoft.Azure.WebJobs/StorageAccountAttribute.cs @@ -6,7 +6,7 @@ namespace Microsoft.Azure.WebJobs { /// - /// Attribute used to override the default Azure Storage account used. + /// Attribute used to override the default Azure Storage account used by triggers and binders. /// /// /// This attribute can be applied at the parameter/method/class level, and the precedence @@ -29,7 +29,7 @@ public StorageAccountAttribute(string account) } /// - /// Gets the Azure Storage account name to use. + /// Gets the name of the Azure Storage account to use. /// public string Account { get; private set; } } diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AsyncChainEndToEndTests.cs b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AsyncChainEndToEndTests.cs index e7d8eb4b8..031c6852d 100644 --- a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AsyncChainEndToEndTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AsyncChainEndToEndTests.cs @@ -126,7 +126,7 @@ public async Task AsyncChainEndToEnd_CustomFactories() Assert.True(queueProcessorFactory.CustomQueueProcessors.Sum(p => p.BeginProcessingCount) >= 2); Assert.True(queueProcessorFactory.CustomQueueProcessors.Sum(p => p.CompleteProcessingCount) >= 2); - Assert.Equal(14, storageClientFactory.TotalBlobClientCount); + Assert.Equal(13, storageClientFactory.TotalBlobClientCount); Assert.Equal(6, storageClientFactory.TotalQueueClientCount); Assert.Equal(0, storageClientFactory.TotalTableClientCount); diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/SingletonEndToEndTests.cs b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/SingletonEndToEndTests.cs index ec875c257..a23d26429 100644 --- a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/SingletonEndToEndTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/SingletonEndToEndTests.cs @@ -26,10 +26,12 @@ public class SingletonEndToEndTests : IClassFixture(IStorageAccount s IFunctionOutputLogger functionOutputLogger = task.Result; FunctionExecutor executor = new FunctionExecutor(functionInstanceLogger, functionOutputLogger, backgroundExceptionDispatcher, new TestTraceWriter(TraceLevel.Verbose)); - Task storageAccountTask = storageAccountProvider.GetStorageAccountAsync(CancellationToken.None); - storageAccountTask.Wait(); SingletonConfiguration singletonConfig = new SingletonConfiguration(); TestTraceWriter trace = new TestTraceWriter(TraceLevel.Verbose); - SingletonManager singletonManager = new SingletonManager(storageAccountTask.Result.CreateBlobClient(), backgroundExceptionDispatcher, singletonConfig, trace); + SingletonManager singletonManager = new SingletonManager(storageAccountProvider, backgroundExceptionDispatcher, singletonConfig, trace); ITypeLocator typeLocator = new FakeTypeLocator(programType); FunctionIndexProvider functionIndexProvider = new FunctionIndexProvider( diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonLockTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonLockTests.cs index b840c0191..e88c93052 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonLockTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonLockTests.cs @@ -79,7 +79,7 @@ public async Task GetOwnerAsync_InvokesSingletonManager_WithExpectedValues() Mock mockSingletonManager = new Mock(MockBehavior.Strict); string lockOwner = "ownerid"; - mockSingletonManager.Setup(p => p.GetLockOwnerAsync(TestLockId, cancellationToken)).ReturnsAsync(lockOwner); + mockSingletonManager.Setup(p => p.GetLockOwnerAsync(attribute, TestLockId, cancellationToken)).ReturnsAsync(lockOwner); SingletonLock singletonLock = new SingletonLock(TestLockId, TestInstanceId, attribute, mockSingletonManager.Object); diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonManagerTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonManagerTests.cs index 2aa9ebd3e..1d89f1772 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonManagerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonManagerTests.cs @@ -8,6 +8,8 @@ using System.Reflection; using System.Threading; using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Azure.WebJobs.Host.Storage; using Microsoft.Azure.WebJobs.Host.Storage.Blob; using Microsoft.Azure.WebJobs.Host.TestCommon; using Microsoft.Azure.WebJobs.Host.Timers; @@ -23,10 +25,15 @@ public class SingletonManagerTests private const string TestLockId = "testid"; private const string TestInstanceId = "testinstance"; private const string TestLeaseId = "testleaseid"; + private const string Secondary = "SecondaryStorage"; private SingletonManager _singletonManager; private SingletonConfiguration _singletonConfig; + private Mock _mockAccountProvider; private Mock _mockBlobDirectory; + private Mock _mockSecondaryBlobDirectory; + private Mock _mockStorageAccount; + private Mock _mockSecondaryStorageAccount; private Mock _mockExceptionDispatcher; private Mock _mockStorageBlob; private TestTraceWriter _trace = new TestTraceWriter(TraceLevel.Verbose); @@ -35,11 +42,25 @@ public class SingletonManagerTests public SingletonManagerTests() { + _mockAccountProvider = new Mock(MockBehavior.Strict); _mockBlobDirectory = new Mock(MockBehavior.Strict); + _mockSecondaryBlobDirectory = new Mock(MockBehavior.Strict); + _mockStorageAccount = new Mock(MockBehavior.Strict); + _mockSecondaryStorageAccount = new Mock(MockBehavior.Strict); Mock mockBlobClient = new Mock(MockBehavior.Strict); + Mock mockSecondaryBlobClient = new Mock(MockBehavior.Strict); Mock mockBlobContainer = new Mock(MockBehavior.Strict); mockBlobContainer.Setup(p => p.GetDirectoryReference(HostDirectoryNames.SingletonLocks)).Returns(_mockBlobDirectory.Object); mockBlobClient.Setup(p => p.GetContainerReference(HostContainerNames.Hosts)).Returns(mockBlobContainer.Object); + Mock mockSecondaryBlobContainer = new Mock(MockBehavior.Strict); + mockSecondaryBlobContainer.Setup(p => p.GetDirectoryReference(HostDirectoryNames.SingletonLocks)).Returns(_mockSecondaryBlobDirectory.Object); + mockSecondaryBlobClient.Setup(p => p.GetContainerReference(HostContainerNames.Hosts)).Returns(mockSecondaryBlobContainer.Object); + _mockStorageAccount.Setup(p => p.CreateBlobClient(null)).Returns(mockBlobClient.Object); + _mockSecondaryStorageAccount.Setup(p => p.CreateBlobClient(null)).Returns(mockSecondaryBlobClient.Object); + _mockAccountProvider.Setup(p => p.GetAccountAsync(ConnectionStringNames.Storage, It.IsAny())) + .ReturnsAsync(_mockStorageAccount.Object); + _mockAccountProvider.Setup(p => p.GetAccountAsync(Secondary, It.IsAny())) + .ReturnsAsync(_mockSecondaryStorageAccount.Object); _mockExceptionDispatcher = new Mock(MockBehavior.Strict); _mockStorageBlob = new Mock(MockBehavior.Strict); @@ -54,11 +75,24 @@ public SingletonManagerTests() _singletonConfig.LockAcquisitionTimeout = TimeSpan.FromMilliseconds(200); _nameResolver = new TestNameResolver(); - _singletonManager = new SingletonManager(mockBlobClient.Object, _mockExceptionDispatcher.Object, _singletonConfig, _trace, _nameResolver); + _singletonManager = new SingletonManager(_mockAccountProvider.Object, _mockExceptionDispatcher.Object, _singletonConfig, _trace, _nameResolver); _singletonManager.MinimumLeaseRenewalInterval = TimeSpan.FromMilliseconds(250); } + [Fact] + public void GetLockDirectory_HandlesMultipleAccounts() + { + IStorageBlobDirectory directory = _singletonManager.GetLockDirectory(ConnectionStringNames.Storage); + Assert.Same(_mockBlobDirectory.Object, directory); + + directory = _singletonManager.GetLockDirectory(null); + Assert.Same(_mockBlobDirectory.Object, directory); + + directory = _singletonManager.GetLockDirectory(Secondary); + Assert.Same(_mockSecondaryBlobDirectory.Object, directory); + } + [Fact] public async Task TryLockAsync_CreatesBlobLease_WithAutoRenewal() { @@ -195,11 +229,12 @@ public async Task GetLockOwnerAsync_LeaseLocked_ReturnsOwner() mockBlobProperties.Setup(p => p.LeaseState).Returns(LeaseState.Leased); _mockStorageBlob.SetupGet(p => p.Properties).Returns(mockBlobProperties.Object); - string lockOwner = await _singletonManager.GetLockOwnerAsync(TestLockId, CancellationToken.None); + SingletonAttribute attribute = new SingletonAttribute(); + string lockOwner = await _singletonManager.GetLockOwnerAsync(attribute, TestLockId, CancellationToken.None); Assert.Equal(null, lockOwner); _mockBlobMetadata.Add(SingletonManager.FunctionInstanceMetadataKey, TestLockId); - lockOwner = await _singletonManager.GetLockOwnerAsync(TestLockId, CancellationToken.None); + lockOwner = await _singletonManager.GetLockOwnerAsync(attribute, TestLockId, CancellationToken.None); Assert.Equal(TestLockId, lockOwner); mockBlobProperties.VerifyAll(); @@ -217,7 +252,8 @@ public async Task GetLockOwnerAsync_LeaseAvailable_ReturnsNull() mockBlobProperties.Setup(p => p.LeaseStatus).Returns(LeaseStatus.Unlocked); _mockStorageBlob.SetupGet(p => p.Properties).Returns(mockBlobProperties.Object); - string lockOwner = await _singletonManager.GetLockOwnerAsync(TestLockId, CancellationToken.None); + SingletonAttribute attribute = new SingletonAttribute(); + string lockOwner = await _singletonManager.GetLockOwnerAsync(attribute, TestLockId, CancellationToken.None); Assert.Equal(null, lockOwner); mockBlobProperties.VerifyAll(); @@ -346,7 +382,7 @@ private static void TestJob() { } - [Singleton("Function", SingletonMode.Listener)] + [Singleton("Function", Mode = SingletonMode.Listener)] private static void TestJob_ListenerSingleton() { } @@ -357,13 +393,13 @@ private static void TestJob_MultipleFunctionSingletons() { } - [Singleton("bar", SingletonMode.Listener)] - [Singleton("foo", SingletonMode.Listener)] + [Singleton("bar", Mode = SingletonMode.Listener)] + [Singleton("foo", Mode = SingletonMode.Listener)] private static void TestJob_MultipleListenerSingletons() { } - [Singleton("Listener", SingletonMode.Listener)] + [Singleton("Listener", Mode = SingletonMode.Listener)] private class TestListener { } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonValueProviderTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonValueProviderTests.cs index 699c7a01d..0226827cf 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonValueProviderTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonValueProviderTests.cs @@ -66,7 +66,7 @@ public void ToInvokeString_ReturnsExpectedValue() public void SingletonWatcher_GetStatus_ReturnsExpectedValue() { Mock mockSingletonManager = new Mock(MockBehavior.Strict); - mockSingletonManager.Setup(p => p.GetLockOwnerAsync(_lockId, CancellationToken.None)).ReturnsAsync("someotherguy"); + mockSingletonManager.Setup(p => p.GetLockOwnerAsync(_attribute, _lockId, CancellationToken.None)).ReturnsAsync("someotherguy"); SingletonValueProvider localValueProvider = new SingletonValueProvider(_method, _attribute.Scope, TestInstanceId, _attribute, mockSingletonManager.Object); SingletonLock localSingletonLock = (SingletonLock)localValueProvider.GetValue();