|
| 1 | +param( |
| 2 | + [Parameter(Mandatory = $true)] |
| 3 | + [ValidateNotNullOrEmpty()] |
| 4 | + [string] |
| 5 | + $ProcDumpPath, |
| 6 | + [Parameter(Mandatory = $true)] |
| 7 | + [ValidateNotNullOrEmpty()] |
| 8 | + [string] |
| 9 | + $ProcDumpOutputPath, |
| 10 | + [Parameter(Mandatory = $true)] |
| 11 | + [datetime] |
| 12 | + $WakeTime, |
| 13 | + [Parameter(Mandatory = $true)] |
| 14 | + [ValidateNotNullOrEmpty()] |
| 15 | + [string []] |
| 16 | + $CandidateProcessNames |
| 17 | +) |
| 18 | + |
| 19 | +Write-Output "Setting up a scheduled job to capture process dumps."; |
| 20 | + |
| 21 | +if ((-not (Test-Path $ProcDumpPath))) { |
| 22 | + Write-Warning "Can't find ProcDump at '$ProcDumpPath'."; |
| 23 | +} |
| 24 | +else { |
| 25 | + Write-Output "Using ProcDump from '$ProcDumpPath'."; |
| 26 | +} |
| 27 | + |
| 28 | +try { |
| 29 | + $previousJobs = Get-Job -Name CaptureDumps* -ErrorAction SilentlyContinue; |
| 30 | + $previousScheduledJobs = Get-ScheduledJob CaptureDumps* -ErrorAction SilentlyContinue; |
| 31 | + |
| 32 | + if ($previousJobs.Count -ne 0) { |
| 33 | + Write-Output "Found existing dump jobs."; |
| 34 | + } |
| 35 | + |
| 36 | + if ($previousScheduledJobs.Count -ne 0) { |
| 37 | + Write-Output "Found existing dump jobs."; |
| 38 | + } |
| 39 | + |
| 40 | + $previousJobs | Stop-Job -PassThru | Remove-Job; |
| 41 | + $previousScheduledJobs | Unregister-ScheduledJob; |
| 42 | +} |
| 43 | +catch { |
| 44 | + Write-Output "There was an error cleaning up previous jobs."; |
| 45 | + Write-Output $_.Exception.Message; |
| 46 | +} |
| 47 | + |
| 48 | +$repoRoot = Resolve-Path "$PSScriptRoot\..\.."; |
| 49 | +$ProcDumpOutputPath = Join-Path $repoRoot $ProcDumpOutputPath; |
| 50 | + |
| 51 | +Write-Output "Dumps will be placed at '$ProcDumpOutputPath'."; |
| 52 | +Write-Output "Watching processes $($CandidateProcessNames -join ', ')"; |
| 53 | + |
| 54 | +# This script registers as a scheduled job. This scheduled job executes after $WakeTime. |
| 55 | +# When the scheduled job executes, it runs procdump on all alive processes whose name matches $CandidateProcessNames. |
| 56 | +# The dumps are placed in $ProcDumpOutputPath |
| 57 | +# If the build completes sucessfully in less than $WakeTime, a final step unregisters the job. |
| 58 | + |
| 59 | +# Create a unique identifier for the job name |
| 60 | +$JobName = "CaptureDumps" + (New-Guid).ToString("N"); |
| 61 | + |
| 62 | +# Ensure that the dumps output path exists. |
| 63 | +if ((-not (Test-Path $ProcDumpOutputPath))) { |
| 64 | + New-Item -ItemType Directory $ProcDumpOutputPath | Out-Null; |
| 65 | +} |
| 66 | + |
| 67 | +# We write a sentinel file that we use at the end of the build to |
| 68 | +# find the job we started and to determine the results from the sheduled |
| 69 | +# job (Whether it ran or not and to display the outputs form the job) |
| 70 | +$sentinelFile = Join-Path $ProcDumpOutputPath "dump-sentinel.txt"; |
| 71 | +Out-File -FilePath $sentinelFile -InputObject $JobName | Out-Null; |
| 72 | + |
| 73 | +[scriptblock] $ScriptCode = { |
| 74 | + param( |
| 75 | + $ProcDumpPath, |
| 76 | + $ProcDumpOutputPath, |
| 77 | + $CandidateProcessNames) |
| 78 | + |
| 79 | + Write-Output "Waking up to capture process dumps. Determining hanging processes."; |
| 80 | + |
| 81 | + [System.Diagnostics.Process []]$AliveProcesses = @(); |
| 82 | + foreach ($candidate in $CandidateProcessNames) { |
| 83 | + try { |
| 84 | + $candidateProcesses = Get-Process $candidate; |
| 85 | + $candidateProcesses | ForEach-Object { Write-Output "Found candidate process $candidate with PID '$($_.Id)'." }; |
| 86 | + $AliveProcesses += $candidateProcesses; |
| 87 | + } |
| 88 | + catch { |
| 89 | + Write-Output "No process found for $candidate"; |
| 90 | + } |
| 91 | + } |
| 92 | + |
| 93 | + Write-Output "Starting process dump capture."; |
| 94 | + |
| 95 | + $dumpFullPath = [System.IO.Path]::Combine($ProcDumpOutputPath, "hung_PROCESSNAME_PID_YYMMDD_HHMMSS.dmp"); |
| 96 | + |
| 97 | + Write-Output "Capturing output for $($AliveProcesses.Length) processes."; |
| 98 | + |
| 99 | + foreach ($process in $AliveProcesses) { |
| 100 | + |
| 101 | + $procDumpArgs = @("-accepteula", "-ma", $process.Id, $dumpFullPath); |
| 102 | + try { |
| 103 | + Write-Output "Capturing dump for dump for '$($process.Name)' with PID '$($process.Id)'."; |
| 104 | + Start-Process -FilePath $ProcDumpPath -ArgumentList $procDumpArgs -NoNewWindow -Wait; |
| 105 | + } |
| 106 | + catch { |
| 107 | + Write-Output "There was an error capturing a process dump for '$($process.Name)' with PID '$($process.Id)'." |
| 108 | + Write-Warning $_.Exception.Message; |
| 109 | + } |
| 110 | + } |
| 111 | + |
| 112 | + Write-Output "Done capturing process dumps."; |
| 113 | +} |
| 114 | + |
| 115 | +$ScriptTrigger = New-JobTrigger -Once -At $WakeTime; |
| 116 | + |
| 117 | +try { |
| 118 | + Register-ScheduledJob -Name $JobName -ScriptBlock $ScriptCode -Trigger $ScriptTrigger -ArgumentList $ProcDumpPath, $ProcDumpOutputPath, $CandidateProcessNames; |
| 119 | +} |
| 120 | +catch { |
| 121 | + Write-Warning "Failed to register scheduled job '$JobName'. Dumps will not be captured for build hangs."; |
| 122 | + Write-Warning $_.Exception.Message; |
| 123 | +} |
0 commit comments