Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

BUG 1972388: vsbuild task in YAML build pipeline hangs forever in ADO even though the task has already logged completion #3979

Merged
merged 5 commits into from
Oct 20, 2022
Merged

BUG 1972388: vsbuild task in YAML build pipeline hangs forever in ADO even though the task has already logged completion #3979

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4ddd15c
moved logic to drain queue in separate method "DrainQueues"
sergey-koryshev Oct 4, 2022
ff2a107
Implemented a call of method "DrainQueues" once step is completed
sergey-koryshev Oct 4, 2022
0d2afd5
Simplified syntax of property 'JobServerQueue'
sergey-koryshev Oct 13, 2022
37b00d9
Merge branch 'master' of github.com:microsoft/azure-pipelines-agent i…
sergey-koryshev Oct 13, 2022
1854b93
Merge branch 'master' into users/sergey.koryshev/1972388-vsbuild-task…
sergey-koryshev Oct 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions src/Agent.Worker/StepsRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using System.Threading.Tasks;
using Microsoft.TeamFoundation.DistributedTask.Expressions;
using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines;
using Microsoft.VisualStudio.Services.CircuitBreaker;

namespace Microsoft.VisualStudio.Services.Agent.Worker
{
Expand All @@ -33,6 +34,10 @@ public interface IStepsRunner : IAgentService

public sealed class StepsRunner : AgentService, IStepsRunner
{
private IJobServerQueue _jobServerQueue;

private IJobServerQueue JobServerQueue => _jobServerQueue ??= HostContext.GetService<IJobServerQueue>();

// StepsRunner should never throw exception to caller
public async Task RunAsync(IExecutionContext jobContext, IList<IStep> steps)
{
Expand Down Expand Up @@ -303,6 +308,17 @@ private async Task RunStepAsync(IStep step, CancellationToken jobCancellationTok
// Complete the step context.
step.ExecutionContext.Section(StringUtil.Loc("StepFinishing", step.DisplayName));
step.ExecutionContext.Complete();

try
{
// We need to drain the queues after a task just in case if
// there are a lot of items since it can cause some UI hangs.
await JobServerQueue.DrainQueues();
} catch (Exception ex)
{
Trace.Error($"Error has occurred while draining queues, it can cause some UI glitches but it doesn't affect a pipeline execution itself: {ex}");
step.ExecutionContext.Error(ex);
}
}

private async Task SwitchToUtf8Codepage(IStep step)
Expand Down
42 changes: 24 additions & 18 deletions src/Microsoft.VisualStudio.Services.Agent/JobServerQueue.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ public interface IJobServerQueue : IAgentService, IThrottlingReporter
{
event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
Task ShutdownAsync();
Task DrainQueues();
void Start(Pipelines.AgentJobRequestMessage jobRequest);
void QueueWebConsoleLine(Guid stepRecordId, string line, long lineNumber);
void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
Expand Down Expand Up @@ -91,6 +92,28 @@ public override void Initialize(IHostContext hostContext)
_jobServer = hostContext.GetService<IJobServer>();
}

public async Task DrainQueues()
{
// Drain the queue
// ProcessWebConsoleLinesQueueAsync() will never throw exception, live console update is always best effort.
Trace.Verbose("Draining web console line queue.");
await ProcessWebConsoleLinesQueueAsync(runOnce: true);
Trace.Info("Web console line queue drained.");

// ProcessFilesUploadQueueAsync() will never throw exception, log file upload is always best effort.
Trace.Verbose("Draining file upload queue.");
await ProcessFilesUploadQueueAsync(runOnce: true);
Trace.Info("File upload queue drained.");

// ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown
// if there is any timeline records that failed to update contains output variabls.
Trace.Verbose("Draining timeline update queue.");
await ProcessTimelinesUpdateQueueAsync(runOnce: true);
Trace.Info("Timeline update queue drained.");

Trace.Info("All queues are drained.");
}

public void Start(Pipelines.AgentJobRequestMessage jobRequest)
{
Trace.Entering();
Expand Down Expand Up @@ -169,24 +192,7 @@ public async Task ShutdownAsync()
_queueInProcess = false;
Trace.Info("All queue process task stopped.");

// Drain the queue
// ProcessWebConsoleLinesQueueAsync() will never throw exception, live console update is always best effort.
Trace.Verbose("Draining web console line queue.");
await ProcessWebConsoleLinesQueueAsync(runOnce: true);
Trace.Info("Web console line queue drained.");

// ProcessFilesUploadQueueAsync() will never throw exception, log file upload is always best effort.
Trace.Verbose("Draining file upload queue.");
await ProcessFilesUploadQueueAsync(runOnce: true);
Trace.Info("File upload queue drained.");

// ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown
// if there is any timeline records that failed to update contains output variabls.
Trace.Verbose("Draining timeline update queue.");
await ProcessTimelinesUpdateQueueAsync(runOnce: true);
Trace.Info("Timeline update queue drained.");

Trace.Info("All queue process tasks have been stopped, and all queues are drained.");
await DrainQueues();
}

public void QueueWebConsoleLine(Guid stepRecordId, string line, long lineNumber)
Expand Down