Skip to content

Expand platform-level retry checking #3214

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: dev
Choose a base branch
from
Draft

Expand platform-level retry checking #3214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6a0d87b
Expand platform-level retry checking
andystaples Oct 2, 2025
a94f717
Merge branch 'dev' into andystaples/expand-platform-error-checking
andystaples Oct 14, 2025
8529c97
IsAssignableFrom fix
andystaples Oct 14, 2025
6dceb42
Cleanup + extra checks
andystaples Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 51 additions & 11 deletions src/WebJobs.Extensions.DurableTask/OutOfProcMiddleware.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,12 +176,8 @@ await this.LifeCycleNotificationHelper.OrchestratorStartingAsync(
// - a timeout
// - an out of memory exception
// - a worker process exit
if (functionResult.Exception is Host.FunctionTimeoutException
|| functionResult.Exception?.InnerException is SessionAbortedException // see RemoteOrchestrationContext.TrySetResultInternal for details on OOM-handling
|| (functionResult.Exception?.InnerException?.GetType().ToString().Contains("WorkerProcessExitException") ?? false))
if (functionResult.Exception != null && this.IsPlatformLevelError(functionResult))
{
// TODO: the `WorkerProcessExitException` type is not exposed in our dependencies, it's part of WebJobs.Host.Script.
// Should we add that dependency or should it be exposed in WebJobs.Host?
throw functionResult.Exception;
}
}
Expand All @@ -205,9 +201,9 @@ await this.LifeCycleNotificationHelper.OrchestratorStartingAsync(
OrchestratorExecutionResult orchestratorResult;
if (functionResult.Succeeded)
{
if (workerRequiresHistory)
{
throw new SessionAbortedException("The worker has since ended the extended session and needs an orchestration history to execute the orchestration request.");
if (workerRequiresHistory)
{
throw new SessionAbortedException("The worker has since ended the extended session and needs an orchestration history to execute the orchestration request.");
}

orchestratorResult = context.GetResult();
Expand Down Expand Up @@ -295,6 +291,40 @@ await this.LifeCycleNotificationHelper.OrchestratorFailedAsync(
dispatchContext.SetProperty(orchestratorResult);
}

private bool IsPlatformLevelError(FunctionResult functionResult)
{
if (this.ExceptionIsInstanceOrIsCausedBy(functionResult.Exception, checkType: typeof(Host.FunctionTimeoutException))
|| this.ExceptionIsInstanceOrIsCausedBy(functionResult.Exception, checkType: typeof(SessionAbortedException)) // see RemoteOrchestrationContext.TrySetResultInternal for details on OOM-handling
|| this.ExceptionIsInstanceOrIsCausedBy(functionResult.Exception, checkTypeString: "WorkerProcessExitException"))
{
// TODO: the `WorkerProcessExitException` type is not exposed in our dependencies, it's part of WebJobs.Host.Script.
// Should we add that dependency or should it be exposed in WebJobs.Host?
return true;
}

return false;
}

private bool ExceptionIsInstanceOrIsCausedBy(Exception? sourceException, Type? checkType = null, string? checkTypeString = null)
{
if (sourceException is null)
{
return false;
}

if (checkType != null && checkType.IsAssignableFrom(sourceException.GetType()))
{
return true;
}

if (checkTypeString != null && sourceException.GetType().ToString().Contains(checkTypeString))
{
return true;
}

return this.ExceptionIsInstanceOrIsCausedBy(sourceException.InnerException, checkType, checkTypeString);
}

/// <summary>
/// Durable Task Framework entity middleware that invokes an out-of-process orchestrator function.
/// </summary>
Expand Down Expand Up @@ -401,6 +431,11 @@ void SetErrorResult(FailureDetails failureDetails)
// Re-throw so we can abort this invocation.
this.HostLifetimeService.OnStopping.ThrowIfCancellationRequested();
}

if (functionResult.Exception != null && this.IsPlatformLevelError(functionResult))
{
throw functionResult.Exception;
}
}
catch (Exception hostRuntimeException)
{
Expand Down Expand Up @@ -444,7 +479,7 @@ void SetErrorResult(FailureDetails failureDetails)
errorType: "FunctionInvocationFailed",
errorMessage: $"Invocation of function '{functionName}' failed with an exception.",
stackTrace: null,
innerFailure: new FailureDetails(functionResult.Exception),
innerFailure: functionResult.Exception != null ? new FailureDetails(functionResult.Exception) : null,
isNonRetriable: true));
}

Expand Down Expand Up @@ -545,6 +580,11 @@ public async Task CallActivityAsync(DispatchMiddlewareContext dispatchContext, F
// Re-throw so we can abort this invocation.
this.HostLifetimeService.OnStopping.ThrowIfCancellationRequested();
}

if (result.Exception != null && this.IsPlatformLevelError(result))
{
throw result.Exception;
}
}
catch (Exception hostRuntimeException)
{
Expand Down Expand Up @@ -596,7 +636,7 @@ public async Task CallActivityAsync(DispatchMiddlewareContext dispatchContext, F
isReplay: false,
scheduledEvent.EventId);

bool detailsParsedFromSerializedException;
bool detailsParsedFromSerializedException = false;

activityResult = new ActivityExecutionResult
{
Expand All @@ -605,7 +645,7 @@ public async Task CallActivityAsync(DispatchMiddlewareContext dispatchContext, F
taskScheduledId: scheduledEvent.EventId,
reason: $"Function '{functionName}' failed with an unhandled exception.",
details: null,
GetFailureDetails(result.Exception, out detailsParsedFromSerializedException)),
result.Exception != null ? GetFailureDetails(result.Exception, out detailsParsedFromSerializedException) : null),
};

if (!detailsParsedFromSerializedException && this.extension.PlatformInformationService.GetWorkerRuntimeType() == WorkerRuntimeType.DotNetIsolated)
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/Apps/BasicJava/extensions.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<ItemGroup>
<!-- This reference will be dynamically updated by build-e2e-tests.ps1 as part of test setup. -->
<!-- Do not commit changes to this line unless necessary -->
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="3.4.1" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="3.5.0" />
<PackageReference Include="Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator" Version="1.1.3" />
<PackageReference Include="Microsoft.DurableTask.SqlServer.AzureFunctions" Version="1.5.1" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.2" />
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/Apps/BasicNode/extensions.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<ItemGroup>
<!-- This reference will be dynamically updated by build-e2e-tests.ps1 as part of test setup. -->
<!-- Do not commit changes to this line unless necessary -->
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="3.4.1" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="3.5.0" />
<PackageReference Include="Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator" Version="1.1.3" />
<PackageReference Include="Microsoft.DurableTask.SqlServer.AzureFunctions" Version="1.5.1" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.2" />
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/Apps/BasicPowerShell/extensions.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<ItemGroup>
<!-- This reference will be dynamically updated by build-e2e-tests.ps1 as part of test setup. -->
<!-- Do not commit changes to this line unless necessary -->
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="3.4.1" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="3.5.0" />
<PackageReference Include="Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator" Version="1.1.3" />
<PackageReference Include="Microsoft.DurableTask.SqlServer.AzureFunctions" Version="1.5.1" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.2" />
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/Apps/BasicPython/extensions.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
<ItemGroup>
<!-- This reference will be dynamically updated by build-e2e-tests.ps1 as part of test setup. -->
<!-- Do not commit changes to this line unless necessary -->
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="3.4.1" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="3.5.0" />
<PackageReference Include="Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator" Version="1.1.3" />
<PackageReference Include="Microsoft.DurableTask.SqlServer.AzureFunctions" Version="1.5.1" />
<PackageReference Include="Microsoft.Data.SqlClient" Version="5.2.2" />
Expand Down
Loading