[WIP] Ensure popReceipt is updated between RenewMessage operations

azure-pipelines-release.yml

@@ -17,10 +17,6 @@ steps:
     verbosityRestore: Minimal
     projects: |
       src/DurableTask.AzureStorage/DurableTask.AzureStorage.sln
-      src/DurableTask.Emulator/DurableTask.Emulator.csproj
-      src/DurableTask.ServiceBus/DurableTask.ServiceBus.csproj
-      src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj
-      src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj
 
 # Build the filtered solution in release mode, specifying the continuous integration flag.
 - task: VSBuild@1
@@ -32,43 +28,6 @@ steps:
     configuration: Release
     msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true
 
-- task: VSBuild@1
-  displayName: 'Build (ApplicationInsights)'
-  inputs:
-    solution: 'src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj'
-    vsVersion: '16.0'
-    logFileVerbosity: minimal
-    configuration: Release
-    msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true
-
-- task: VSBuild@1
-  displayName: 'Build (Emulator)'
-  inputs:
-    solution: 'src/DurableTask.Emulator/DurableTask.Emulator.csproj'
-    vsVersion: '16.0'
-    logFileVerbosity: minimal
-    configuration: Release
-    msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true
-
-- task: VSBuild@1
-  displayName: 'Build (ServiceBus)'
-  inputs:
-    solution: 'src/DurableTask.ServiceBus/DurableTask.ServiceBus.csproj'
-    vsVersion: '16.0'
-    logFileVerbosity: minimal
-    configuration: Release
-    msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true
-
-- task: VSBuild@1
-  displayName: 'Build (AzureServiceFabric)'
-  inputs:
-    solution: 'src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj'
-    vsVersion: '16.0'
-    logFileVerbosity: minimal
-    configuration: Release
-    platform: x64
-    msbuildArgs: /p:FileVersionRevision=$(Build.BuildId) /p:ContinuousIntegrationBuild=true
-
 - task: UseDotNet@2
   displayName: 'Use the .NET Core 2.1 SDK (required for build signing)'
   inputs:
@@ -127,74 +86,6 @@ steps:
     packDirectory: $(build.artifactStagingDirectory)
     packagesToPack: 'src/DurableTask.AzureStorage/DurableTask.AzureStorage.sln'
 
-- task: DotNetCoreCLI@2
-  displayName: Generate nuget packages
-  inputs:
-    command: pack
-    verbosityPack: Minimal
-    configuration: Release
-    nobuild: true
-    packDirectory: $(build.artifactStagingDirectory)
-    packagesToPack: 'src/DurableTask.ApplicationInsights/DurableTask.ApplicationInsights.csproj'
-
-- task: DotNetCoreCLI@2
-  displayName: Generate nuget packages
-  inputs:
-    command: pack
-    verbosityPack: Minimal
-    configuration: Release
-    nobuild: true
-    packDirectory: $(build.artifactStagingDirectory)
-    packagesToPack: 'src/DurableTask.Emulator/DurableTask.Emulator.csproj'
-
-- task: DotNetCoreCLI@2
-  displayName: Generate nuget packages
-  inputs:
-    command: pack
-    verbosityPack: Minimal
-    configuration: Release
-    nobuild: true
-    packDirectory: $(build.artifactStagingDirectory)
-    packagesToPack: 'src/DurableTask.ServiceBus/DurableTask.ServiceBus.csproj'
-
-- task: DotNetCoreCLI@2
-  displayName: Generate nuget packages
-  inputs:
-    command: pack
-    verbosityPack: Minimal
-    configuration: Release
-    nobuild: true
-    packDirectory: $(build.artifactStagingDirectory)
-    packagesToPack: 'src/DurableTask.AzureServiceFabric/DurableTask.AzureServiceFabric.csproj'
-    buildProperties: 'Platform=x64'
-
-# Digitally sign all the nuget packages with the Microsoft certificate.
-# This appears to be an in-place signing job, which is convenient.
-- task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
-  displayName: 'ESRP CodeSigning: Nupkg'
-  inputs:
-    ConnectedServiceName: 'ESRP Service'
-    FolderPath: $(build.artifactStagingDirectory)
-    Pattern: '*.nupkg'
-    signConfigType: inlineSignParams
-    inlineOperation: |
-     [    
-        {
-            "KeyCode": "CP-401405",
-            "OperationCode": "NuGetSign",
-            "Parameters": {},
-            "ToolName": "sign",
-            "ToolVersion": "1.0"
-        },
-        {
-            "KeyCode": "CP-401405",
-            "OperationCode": "NuGetVerify",
-            "Parameters": {},
-            "ToolName": "sign",
-            "ToolVersion": "1.0"
-        }
-     ]
-
 # Make the nuget packages available for download in the ADO portal UI
 - publish: $(build.artifactStagingDirectory)
   displayName: 'Publish nuget packages to Artifacts'

src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj

@@ -24,7 +24,7 @@
     <PatchVersion>0</PatchVersion>
 
     <VersionPrefix>$(MajorVersion).$(MinorVersion).$(PatchVersion)</VersionPrefix>
-    <VersionSuffix>preview.5</VersionSuffix>
+    <VersionSuffix>preview.5.popReceiptfix.1</VersionSuffix>
     <FileVersion>$(VersionPrefix).0</FileVersion>
     <!-- FileVersionRevision is expected to be set by the CI. This is useful for distinguishing between multiple builds of the same version. -->
     <FileVersion Condition="'$(FileVersionRevision)' != ''">$(VersionPrefix).$(FileVersionRevision)</FileVersion>
@@ -45,7 +45,6 @@
 
   <ItemGroup Condition="'$(TargetFramework)' == 'net462'">
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="1.1.1" />
-    <PackageReference Include="Newtonsoft.Json" Version="7.0.1" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">

src/DurableTask.AzureStorage/Storage/Queue.cs

@@ -27,12 +27,15 @@ class Queue
         readonly AzureStorageClient azureStorageClient;
         readonly AzureStorageOrchestrationServiceStats stats;
         readonly QueueClient queueClient;
+        // TODO: I don't love this map. I think we should use our own AzStorage Queue class to keep track of popReceipts instead. We may need to change quite a few interfaces in response though, so they take our Queue class instead of the Az Storage SDK Queue abstraction
+        readonly Dictionary<string, string> messageIdPopReceipts;
 
         public Queue(AzureStorageClient azureStorageClient, QueueServiceClient queueServiceClient, string queueName)
         {
             this.azureStorageClient = azureStorageClient;
             this.stats = this.azureStorageClient.Stats;
             this.queueClient = queueServiceClient.GetQueueClient(queueName);
+            this.messageIdPopReceipts = new Dictionary<string, string>();
         }
 
         public string Name => this.queueClient.Name;
@@ -61,15 +64,26 @@ await this.queueClient
 
         public async Task UpdateMessageAsync(QueueMessage queueMessage, TimeSpan visibilityTimeout, Guid? clientRequestId = null, CancellationToken cancellationToken = default)
         {
+            string popReceipt = queueMessage.PopReceipt; // default case
+            if (this.messageIdPopReceipts.TryGetValue(queueMessage.MessageId, out string foundReceipt))
+            {
+                // TODO: we should log something if we cannot find a pop receipt
+                popReceipt = foundReceipt;
+            }
+
             using IDisposable scope = OperationContext.CreateClientRequestScope(clientRequestId);
-            await this.queueClient
+            Response<UpdateReceipt> response = await this.queueClient
                 .UpdateMessageAsync(
                     queueMessage.MessageId,
-                    queueMessage.PopReceipt,
+                    popReceipt,
                     visibilityTimeout: visibilityTimeout,
                     cancellationToken: cancellationToken)
                 .DecorateFailure();
 
+            this.messageIdPopReceipts[queueMessage.MessageId] = response.Value.PopReceipt;
+
+            UpdateReceipt receipt = response.Value;
+
             this.stats.MessagesUpdated.Increment();
         }
 
@@ -82,6 +96,7 @@ await this.queueClient
                     queueMessage.PopReceipt,
                     cancellationToken)
                 .DecorateFailure();
+            this.messageIdPopReceipts.Remove(queueMessage.MessageId);
         }
 
         public async Task<QueueMessage?> GetMessageAsync(TimeSpan visibilityTimeout, CancellationToken cancellationToken = default)
@@ -93,6 +108,7 @@ await this.queueClient
                 return null;
             }
 
+            this.messageIdPopReceipts.Add(message.MessageId, message.PopReceipt);
             this.stats.MessagesRead.Increment();
             return message;
         }

src/DurableTask.Core/DurableTask.Core.csproj

@@ -32,7 +32,7 @@
 
   <ItemGroup Condition="'$(TargetFramework)' == 'net462'">
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="1.1.1" />
-    <PackageReference Include="Newtonsoft.Json" Version="7.0.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="7.0.1" NoWarn="NU1903"/>
   </ItemGroup>
 
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">