diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/LeaseManagement/DocumentServiceLeaseManagerCosmos.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/LeaseManagement/DocumentServiceLeaseManagerCosmos.cs index 260bff3161..d5a75abb47 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/LeaseManagement/DocumentServiceLeaseManagerCosmos.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/LeaseManagement/DocumentServiceLeaseManagerCosmos.cs @@ -190,7 +190,17 @@ await this.leaseUpdater.UpdateLeaseAsync( if (serverLease.Owner != lease.Owner) { DefaultTrace.TraceInformation("Lease with token {0} no need to release lease. The lease was already taken by another host '{1}'.", lease.CurrentLeaseToken, serverLease.Owner); - throw new LeaseLostException(lease); + throw new LeaseLostException( + lease, + CosmosExceptionFactory.Create( + statusCode: HttpStatusCode.PreconditionFailed, + message: $"{lease.CurrentLeaseToken} lease token was taken over by owner '{serverLease.Owner}'", + headers: new Headers(), + stackTrace: default, + trace: NoOpTrace.Singleton, + error: default, + innerException: default), + isGone: false); } serverLease.Owner = null; return serverLease; @@ -232,7 +242,17 @@ public override async Task RenewAsync(DocumentServiceLease if (serverLease.Owner != lease.Owner) { DefaultTrace.TraceInformation("Lease with token {0} was taken over by owner '{1}'", lease.CurrentLeaseToken, serverLease.Owner); - throw new LeaseLostException(lease); + throw new LeaseLostException( + lease, + CosmosExceptionFactory.Create( + statusCode: HttpStatusCode.PreconditionFailed, + message: $"{lease.CurrentLeaseToken} lease token was taken over by owner '{serverLease.Owner}'", + headers: new Headers(), + stackTrace: default, + trace: NoOpTrace.Singleton, + error: default, + innerException: default), + isGone: false); } return serverLease; }).ConfigureAwait(false); @@ -245,7 +265,17 @@ public override async Task UpdatePropertiesAsync(DocumentS if (lease.Owner != this.options.HostName) { DefaultTrace.TraceInformation("Lease with token '{0}' was taken over by owner '{1}' before lease properties update", lease.CurrentLeaseToken, lease.Owner); - throw new LeaseLostException(lease); + throw new LeaseLostException( + lease, + CosmosExceptionFactory.Create( + statusCode: HttpStatusCode.PreconditionFailed, + message: $"{lease.CurrentLeaseToken} lease token was taken over by owner '{lease.Owner}'", + headers: new Headers(), + stackTrace: default, + trace: NoOpTrace.Singleton, + error: default, + innerException: default), + isGone: false); } return await this.leaseUpdater.UpdateLeaseAsync( @@ -257,7 +287,17 @@ public override async Task UpdatePropertiesAsync(DocumentS if (serverLease.Owner != lease.Owner) { DefaultTrace.TraceInformation("Lease with token '{0}' was taken over by owner '{1}'", lease.CurrentLeaseToken, serverLease.Owner); - throw new LeaseLostException(lease); + throw new LeaseLostException( + lease, + CosmosExceptionFactory.Create( + statusCode: HttpStatusCode.PreconditionFailed, + message: $"{lease.CurrentLeaseToken} lease token was taken over by owner '{serverLease.Owner}'", + headers: new Headers(), + stackTrace: default, + trace: NoOpTrace.Singleton, + error: default, + innerException: default), + isGone: false); } serverLease.Properties = lease.Properties; return serverLease; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/DocumentServiceLeaseManagerCosmosTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/DocumentServiceLeaseManagerCosmosTests.cs index 19a0710e66..88f945ae43 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/DocumentServiceLeaseManagerCosmosTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/DocumentServiceLeaseManagerCosmosTests.cs @@ -356,6 +356,201 @@ public async Task IfOwnerChangedThrow() && innerCosmosException.StatusCode == HttpStatusCode.PreconditionFailed); } + /// + /// Verifies that if the renewed read a different Owner from the captured in memory, throws a LeaseLost + /// + [TestMethod] + public async Task IfOwnerChangedThrowOnRenew() + { + DocumentServiceLeaseStoreManagerOptions options = new DocumentServiceLeaseStoreManagerOptions + { + HostName = Guid.NewGuid().ToString() + }; + + DocumentServiceLeaseCore lease = new DocumentServiceLeaseCore() + { + LeaseToken = "0", + Owner = Guid.NewGuid().ToString(), + FeedRange = new FeedRangePartitionKeyRange("0") + }; + + Mock mockUpdater = new Mock(); + + Func, bool> validateUpdater = (Func updater) => + { + // Simulate dirty read from db + DocumentServiceLeaseCore serverLease = new DocumentServiceLeaseCore() + { + LeaseToken = "0", + Owner = Guid.NewGuid().ToString(), + FeedRange = new FeedRangePartitionKeyRange("0") + }; + DocumentServiceLease afterUpdateLease = updater(serverLease); + return true; + }; + + mockUpdater.Setup(c => c.UpdateLeaseAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.Is>(f => validateUpdater(f)))) + .ReturnsAsync(lease); + + ResponseMessage leaseResponse = new ResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new CosmosJsonDotNetSerializer().ToStream(lease) + }; + + Mock leaseContainer = new Mock(); + leaseContainer.Setup(c => c.ReadItemStreamAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())).ReturnsAsync(leaseResponse); + + DocumentServiceLeaseManagerCosmos documentServiceLeaseManagerCosmos = new DocumentServiceLeaseManagerCosmos( + Mock.Of(), + leaseContainer.Object, + mockUpdater.Object, + options, + Mock.Of()); + + LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync(() => documentServiceLeaseManagerCosmos.RenewAsync(lease)); + + Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException + && innerCosmosException.StatusCode == HttpStatusCode.PreconditionFailed); + } + + /// + /// Verifies that if the update properties read a different Owner from the captured in memory, throws a LeaseLost + /// + [TestMethod] + public async Task IfOwnerChangedThrowOnUpdateProperties() + { + DocumentServiceLeaseCore lease = new DocumentServiceLeaseCore() + { + LeaseToken = "0", + Owner = Guid.NewGuid().ToString(), + FeedRange = new FeedRangePartitionKeyRange("0") + }; + + DocumentServiceLeaseStoreManagerOptions options = new DocumentServiceLeaseStoreManagerOptions + { + HostName = lease.Owner + }; + + Mock mockUpdater = new Mock(); + + Func, bool> validateUpdater = (Func updater) => + { + // Simulate dirty read from db + DocumentServiceLeaseCore serverLease = new DocumentServiceLeaseCore() + { + LeaseToken = "0", + Owner = Guid.NewGuid().ToString(), + FeedRange = new FeedRangePartitionKeyRange("0") + }; + DocumentServiceLease afterUpdateLease = updater(serverLease); + return true; + }; + + mockUpdater.Setup(c => c.UpdateLeaseAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.Is>(f => validateUpdater(f)))) + .ReturnsAsync(lease); + + ResponseMessage leaseResponse = new ResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new CosmosJsonDotNetSerializer().ToStream(lease) + }; + + Mock leaseContainer = new Mock(); + leaseContainer.Setup(c => c.ReadItemStreamAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())).ReturnsAsync(leaseResponse); + + DocumentServiceLeaseManagerCosmos documentServiceLeaseManagerCosmos = new DocumentServiceLeaseManagerCosmos( + Mock.Of(), + leaseContainer.Object, + mockUpdater.Object, + options, + Mock.Of()); + + LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync(() => documentServiceLeaseManagerCosmos.UpdatePropertiesAsync(lease)); + + Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException + && innerCosmosException.StatusCode == HttpStatusCode.PreconditionFailed); + } + + /// + /// Verifies that if the update properties read a different Owner from the captured in memory, throws a LeaseLost + /// + [TestMethod] + public async Task IfOwnerChangedThrowOnRelease() + { + DocumentServiceLeaseStoreManagerOptions options = new DocumentServiceLeaseStoreManagerOptions + { + HostName = Guid.NewGuid().ToString() + }; + + DocumentServiceLeaseCore lease = new DocumentServiceLeaseCore() + { + LeaseToken = "0", + Owner = Guid.NewGuid().ToString(), + FeedRange = new FeedRangePartitionKeyRange("0") + }; + + Mock mockUpdater = new Mock(); + + Func, bool> validateUpdater = (Func updater) => + { + // Simulate dirty read from db + DocumentServiceLeaseCore serverLease = new DocumentServiceLeaseCore() + { + LeaseToken = "0", + Owner = Guid.NewGuid().ToString(), + FeedRange = new FeedRangePartitionKeyRange("0") + }; + DocumentServiceLease afterUpdateLease = updater(serverLease); + return true; + }; + + mockUpdater.Setup(c => c.UpdateLeaseAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.Is>(f => validateUpdater(f)))) + .ReturnsAsync(lease); + + ResponseMessage leaseResponse = new ResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new CosmosJsonDotNetSerializer().ToStream(lease) + }; + + Mock leaseContainer = new Mock(); + leaseContainer.Setup(c => c.ReadItemStreamAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())).ReturnsAsync(leaseResponse); + + DocumentServiceLeaseManagerCosmos documentServiceLeaseManagerCosmos = new DocumentServiceLeaseManagerCosmos( + Mock.Of(), + leaseContainer.Object, + mockUpdater.Object, + options, + Mock.Of()); + + LeaseLostException leaseLost = await Assert.ThrowsExceptionAsync(() => documentServiceLeaseManagerCosmos.ReleaseAsync(lease)); + + Assert.IsTrue(leaseLost.InnerException is CosmosException innerCosmosException + && innerCosmosException.StatusCode == HttpStatusCode.PreconditionFailed); + } + /// /// When a lease is missing the range information, check that we are adding it ///