Increasing the default listener lock period to the max of 60 seconds

to decrease steady state lock renewals. Also increasing the default polling
interval to 3 seconds.

src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonConfiguration.cs

@@ -11,7 +11,12 @@ namespace Microsoft.Azure.WebJobs.Host
     /// </summary>
     public sealed class SingletonConfiguration
     {
+        // These are the min/max values supported by Azure Storage
+        private static readonly TimeSpan MinimumLeasePeriod = TimeSpan.FromSeconds(15);
+        private static readonly TimeSpan MaximumLeasePeriod = TimeSpan.FromSeconds(60);
+
         private TimeSpan _lockPeriod;
+        private TimeSpan _listenerLockPeriod;
         private TimeSpan _lockAcquisitionTimeout;
         private TimeSpan _lockAcquisitionPollingInterval;
         private TimeSpan _listenerLockRecoveryPollingInterval;
@@ -21,19 +26,16 @@ public sealed class SingletonConfiguration
         /// </summary>
         public SingletonConfiguration()
         {
-            _lockPeriod = TimeSpan.FromSeconds(15);
+            _lockPeriod = MinimumLeasePeriod;
+            _listenerLockPeriod = MaximumLeasePeriod;
             _lockAcquisitionTimeout = TimeSpan.FromMinutes(1);
-            _lockAcquisitionPollingInterval = TimeSpan.FromSeconds(1);
+            _lockAcquisitionPollingInterval = TimeSpan.FromSeconds(3);
             _listenerLockRecoveryPollingInterval = TimeSpan.FromMinutes(1);
         }
 
         /// <summary>
-        /// Gets or sets the default duration of a lock. As this period nears expiry,
-        /// the lock will be automatically renewed.
-        /// <remarks>
-        /// Since there is auto-renewal, changing this to a value other than default
-        /// will only change how often renewals occur.
-        /// </remarks>
+        /// Gets or sets the default duration of <see cref="SingletonMode.Function"/> locks.
+        /// As this period nears expiry, the lock will be automatically renewed.
         /// </summary>
         public TimeSpan LockPeriod
         {
@@ -43,19 +45,30 @@ public TimeSpan LockPeriod
             }
             set
             {
-                if (value < TimeSpan.FromSeconds(15) ||
-                    value > TimeSpan.FromSeconds(60))
-                {
-                    throw new ArgumentOutOfRangeException("value");
-                }
+                ValidateLockPeriod(value);
                 _lockPeriod = value;
             }
         }
 
         /// <summary>
-        /// Gets or sets the timeout value for lock acquisition. If the lock for a
-        /// particular function invocation is not obtained within this interval, the
-        /// invocation will fail.
+        /// Gets or sets the default duration of <see cref="SingletonMode.Listener"/> locks.
+        /// As this period nears expiry, the lock will be automatically renewed.
+        /// </summary>
+        public TimeSpan ListenerLockPeriod
+        {
+            get
+            {
+                return _listenerLockPeriod;
+            }
+            set
+            {
+                ValidateLockPeriod(value);
+                _listenerLockPeriod = value;
+            }
+        }
+
+        /// <summary>
+        /// Gets or sets the timeout value for lock acquisition.
         /// </summary>
         public TimeSpan LockAcquisitionTimeout
         {
@@ -76,7 +89,7 @@ public TimeSpan LockAcquisitionTimeout
         /// <summary>
         /// Gets or sets the polling interval governing how often retries are made
         /// when waiting to acquire a lock. The system will retry on this interval
-        /// until the <see cref="LockAcquisitionTimeout"/> expiry is exceeded.
+        /// until the <see cref="LockAcquisitionTimeout"/> is exceeded.
         /// </summary>
         public TimeSpan LockAcquisitionPollingInterval 
         {
@@ -95,17 +108,15 @@ public TimeSpan LockAcquisitionPollingInterval
         }
 
         /// <summary>
-        /// Gets or sets the polling interval used by singleton locks of type
-        /// <see cref="SingletonMode.Listener"/> to periodically reattempt to
-        /// acquire their lock if they failed to acquire it on startup.
+        /// Gets or sets the polling interval used by <see cref="SingletonMode.Listener"/> locks
+        /// to acquire their lock if they failed to acquire it on startup.
         /// </summary>
         /// <remarks>
-        /// On startup, singleton listeners for triggered functions will each attempt to
-        /// acquire their locks. If they are unable to within the timeout window, the
-        /// listener won't start (and the triggered function won't be running). However,
-        /// behind the scenes, the listener will periodically reattempt to acquire the lock
-        /// based on this value.
-        /// To disable this behavior, set the value to <see cref="Timeout.InfiniteTimeSpan"/>.
+        /// On startup, singleton listeners for triggered functions make a single attempt to acquire
+        /// their locks. If unable to acquire the lock (e.g. if another instance has it) the listener
+        /// won't start (and the triggered function won't be running). However, the listener will
+        /// periodically reattempt to acquire the lock based on this value. To disable this behavior
+        /// set the value to <see cref="Timeout.InfiniteTimeSpan"/>.
         /// </remarks>
         public TimeSpan ListenerLockRecoveryPollingInterval
         {
@@ -116,12 +127,21 @@ public TimeSpan ListenerLockRecoveryPollingInterval
             set
             {
                 if (value != Timeout.InfiniteTimeSpan &&
-                    value < TimeSpan.FromSeconds(15))
+                    value < MinimumLeasePeriod)
                 {
                     throw new ArgumentOutOfRangeException("value");
                 }
                 _listenerLockRecoveryPollingInterval = value;
             }
         }
+
+        private static void ValidateLockPeriod(TimeSpan value)
+        {
+            if (value < MinimumLeasePeriod ||
+                value > MaximumLeasePeriod)
+            {
+                throw new ArgumentOutOfRangeException("value");
+            }
+        }
     }
 }

src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonListener.cs

@@ -14,6 +14,7 @@ internal class SingletonListener : IListener
         private readonly MethodInfo _method;
         private readonly SingletonAttribute _attribute;
         private readonly SingletonManager _singletonManager;
+        private readonly SingletonConfiguration _singletonConfig;
         private readonly IListener _innerListener;
         private string _lockId;
         private object _lockHandle;
@@ -24,6 +25,7 @@ public SingletonListener(MethodInfo method, SingletonAttribute attribute, Single
             _method = method;
             _attribute = attribute;
             _singletonManager = singletonManager;
+            _singletonConfig = _singletonManager.Config;
             _innerListener = innerListener;
 
             string boundScope = _singletonManager.GetBoundScope(_attribute.Scope);
@@ -36,30 +38,22 @@ public SingletonListener(MethodInfo method, SingletonAttribute attribute, Single
 
         public async Task StartAsync(CancellationToken cancellationToken)
         {
-            // for listener locks, if the user hasn't explicitly set an override on the
-            // attribute, we want to default the timeout to the lock period. We want to
-            // stop as soon as possible (since we want startup to be relatively fast)
-            // however we don't want to give up before waiting for a natural lease expiry.
-            // If we miss the lock, the recovery timer will periodically reattempt to acquire.
-            if (_attribute.LockAcquisitionTimeout == null)
-            {
-                _attribute.LockAcquisitionTimeout = (int)_singletonManager.Config.LockPeriod.TotalSeconds;
-            }
-
-            _lockHandle = await _singletonManager.TryLockAsync(_lockId, null, _attribute, cancellationToken);
+            // When recovery is enabled, we don't do retries on the individual lock attempts,
+            // since retries are being done outside
+            bool recoveryEnabled = _singletonConfig.ListenerLockRecoveryPollingInterval != Timeout.InfiniteTimeSpan;
+            _lockHandle = await _singletonManager.TryLockAsync(_lockId, null, _attribute, cancellationToken, retry: !recoveryEnabled);
 
             if (_lockHandle == null)
             {
                 // If we're unable to acquire the lock, it means another listener
-                // has it so we return w/o starting our listener
+                // has it so we return w/o starting our listener.
                 //
                 // However, we also start a periodic background "recovery" timer that will recheck
                 // occasionally for the lock. This ensures that if the host that has the lock goes
                 // down for whatever reason, others will have a chance to resume the work.
-                TimeSpan recoveryPollingInterval = _singletonManager.Config.ListenerLockRecoveryPollingInterval;
-                if (recoveryPollingInterval != Timeout.InfiniteTimeSpan)
+                if (recoveryEnabled)
                 {
-                    LockTimer = new System.Timers.Timer(recoveryPollingInterval.TotalMilliseconds);
+                    LockTimer = new System.Timers.Timer(_singletonConfig.ListenerLockRecoveryPollingInterval.TotalMilliseconds);
                     LockTimer.Elapsed += OnLockTimer;
                     LockTimer.Start();
                 }
@@ -117,7 +111,7 @@ private void OnLockTimer(object sender, ElapsedEventArgs e)
 
         internal async Task TryAcquireLock()
         {
-            _lockHandle = await _singletonManager.TryLockAsync(_lockId, null, _attribute, CancellationToken.None);
+            _lockHandle = await _singletonManager.TryLockAsync(_lockId, null, _attribute, CancellationToken.None, retry: false);
 
             if (_lockHandle != null)
             {

src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonManager.cs

@@ -85,16 +85,17 @@ public async virtual Task<object> LockAsync(string lockId, string functionInstan
             return lockHandle;
         }
 
-        public async virtual Task<object> TryLockAsync(string lockId, string functionInstanceId, SingletonAttribute attribute, CancellationToken cancellationToken)
+        public async virtual Task<object> TryLockAsync(string lockId, string functionInstanceId, SingletonAttribute attribute, CancellationToken cancellationToken, bool retry = true)
         {
             IStorageBlobDirectory lockDirectory = GetLockDirectory(attribute.Account);
             IStorageBlockBlob lockBlob = lockDirectory.GetBlockBlobReference(lockId);
             await TryCreateAsync(lockBlob, cancellationToken);
 
             _trace.Verbose(string.Format(CultureInfo.InvariantCulture, "Waiting for Singleton lock ({0})", lockId), source: TraceSource.Execution);
 
-            string leaseId = await TryAcquireLeaseAsync(lockBlob, _config.LockPeriod, cancellationToken);
-            if (string.IsNullOrEmpty(leaseId))
+            TimeSpan lockPeriod = GetLockPeriod(attribute, _config);
+            string leaseId = await TryAcquireLeaseAsync(lockBlob, lockPeriod, cancellationToken);
+            if (string.IsNullOrEmpty(leaseId) && retry)
             {
                 // Someone else has the lease. Continue trying to periodically get the lease for
                 // a period of time
@@ -105,7 +106,7 @@ public async virtual Task<object> TryLockAsync(string lockId, string functionIns
                 while (string.IsNullOrEmpty(leaseId) && remainingWaitTime > 0)
                 {
                     await Task.Delay(_config.LockAcquisitionPollingInterval);
-                    leaseId = await TryAcquireLeaseAsync(lockBlob, _config.LockPeriod, cancellationToken);
+                    leaseId = await TryAcquireLeaseAsync(lockBlob, lockPeriod, cancellationToken);
                     remainingWaitTime -= _config.LockAcquisitionPollingInterval.TotalMilliseconds;
                 }
             }
@@ -128,7 +129,7 @@ public async virtual Task<object> TryLockAsync(string lockId, string functionIns
                 LeaseId = leaseId,
                 LockId = lockId,
                 Blob = lockBlob,
-                LeaseRenewalTimer = CreateLeaseRenewalTimer(lockBlob, leaseId, lockId, _config.LockPeriod, _backgroundExceptionDispatcher)
+                LeaseRenewalTimer = CreateLeaseRenewalTimer(lockBlob, leaseId, lockId, lockPeriod, _backgroundExceptionDispatcher)
             };
 
             // start the renewal timer, which ensures that we maintain our lease until
@@ -265,6 +266,12 @@ internal IStorageBlobDirectory GetLockDirectory(string accountName)
             return storageDirectory;
         }
 
+        internal static TimeSpan GetLockPeriod(SingletonAttribute attribute, SingletonConfiguration config)
+        {
+            return attribute.Mode == SingletonMode.Listener ?
+                    config.ListenerLockPeriod : config.LockPeriod;
+        }
+
         private ITaskSeriesTimer CreateLeaseRenewalTimer(IStorageBlockBlob leaseBlob, string leaseId, string lockId, TimeSpan leasePeriod, 
             IBackgroundExceptionDispatcher backgroundExceptionDispatcher)
         {

test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonConfigurationTests.cs

@@ -15,8 +15,9 @@ public void ConstructorDefaults()
             SingletonConfiguration config = new SingletonConfiguration();
 
             Assert.Equal(TimeSpan.FromSeconds(15), config.LockPeriod);
+            Assert.Equal(TimeSpan.FromSeconds(60), config.ListenerLockPeriod);
             Assert.Equal(TimeSpan.FromMinutes(1), config.LockAcquisitionTimeout);
-            Assert.Equal(TimeSpan.FromSeconds(1), config.LockAcquisitionPollingInterval);
+            Assert.Equal(TimeSpan.FromSeconds(3), config.LockAcquisitionPollingInterval);
         }
 
         [Fact]
@@ -50,6 +51,37 @@ public void LockPeriod_RangeValidation()
             }
         }
 
+        [Fact]
+        public void ListenerLockPeriod_RangeValidation()
+        {
+            SingletonConfiguration config = new SingletonConfiguration();
+
+            TimeSpan[] invalidValues = new TimeSpan[]
+            {
+                TimeSpan.Zero,
+                TimeSpan.FromSeconds(14),
+                TimeSpan.FromSeconds(61)
+            };
+            foreach (TimeSpan value in invalidValues)
+            {
+                Assert.Throws<ArgumentOutOfRangeException>(() =>
+                {
+                    config.ListenerLockPeriod = value;
+                });
+            }
+
+            TimeSpan[] validValues = new TimeSpan[]
+            {
+                TimeSpan.FromSeconds(16),
+                TimeSpan.FromSeconds(59)
+            };
+            foreach (TimeSpan value in validValues)
+            {
+                config.ListenerLockPeriod = value;
+                Assert.Equal(value, config.ListenerLockPeriod);
+            }
+        }
+
         [Fact]
         public void LockAcquisitionPollingInterval_RangeValidation()
         {