Closed
Description
Occurrences ?-8/24 (incl. PRs):
| Day | Run | Notes |
|---|---|---|
| 8/7 | Rolling run | OSX.1200.Amd64.Open |
| 8/4 | PR #72957 | Alpine.314.Arm32.Open -- likely unrelated arm32 problem |
| 8/2 | PR #73245 | Debian.10.Amd64.Open - Mono - release/6.0 branch |
| 7/22 | PR #72545 | OSX.1200.Amd64.Open |
| 7/22 | PR #72656 | Alpine.314.Arm32.Open -- likely unrelated arm32 problem |
| 7/20 | PR #72178 | Alpine.314.Arm32.Open -- likely unrelated arm32 problem |
| 7/14 | PR #72082 | Alpine.314.Arm32.Open -- likely unrelated arm32 problem |
| 7/7 | PR #71647 | OSX.1200.Amd64.Open |
| 6/27 | Rolling run | OSX.1200.Amd64.Open |
| Before 6/23 | -- | Most runs have 2-10 failures, logs are not available anymore |
let crashes = (friendlyNamePrefix : string, includePR : bool) {
cluster('engsrvprod.kusto.windows.net').database('engineeringdata').WorkItems
| where FriendlyName startswith friendlyNamePrefix
//| where Queued > ago(7d)
| where Status == "BadExit"
| where ExitCode == 139
| join kind=inner (cluster('engsrvprod.kusto.windows.net').database('engineeringdata').Jobs
| where ((Branch == 'refs/heads/main') or (Branch == 'refs/heads/master') or (includePR and (Source startswith "pr/")) or (Branch startswith 'refs/heads/release/6.0'))
| where Type startswith "test/functional/cli/"
and not(Properties contains "runtime-staging")
| summarize arg_max(Finished, Properties, Type, Branch, Source, Started, QueueName) by JobId
| project-rename JobType = Type) on JobId
| extend PropertiesJson = parse_json(Properties)
| extend OS = replace_regex(tostring(PropertiesJson.operatingSystem), @'\((.*)\).*|([^\(].*)', @'\1\2')
| extend Runtime = iif(PropertiesJson.runtimeFlavor == "mono", "Mono", iif(PropertiesJson.DefinitionName contains "coreclr", "CoreCLR", ""))
| extend TargetBranch = extractjson("$.['System.PullRequest.TargetBranch']", Properties)
| extend Architecture = PropertiesJson.architecture
| extend Scenario = iif(isempty(PropertiesJson.scenario), "--", PropertiesJson.scenario)
| project-away PropertiesJson
| order by Finished desc
};
crashes('System.Net.Security', true);Noticed it in one of the runs in PR: #72082
/root/helix/work/correlation/dotnet exec --runtimeconfig System.Net.Security.Tests.runtimeconfig.json --depsfile System.Net.Security.Tests.deps.json xunit.console.dll System.Net.Security.Tests.dll -xml testResults.xml -nologo -nocolor -notrait category=IgnoreForCI -notrait category=OuterLoop -notrait category=failing
popd
===========================================================================================================
/root/helix/work/workitem/e /root/helix/work/workitem/e
Discovering: System.Net.Security.Tests (method display = ClassAndMethod, method display options = None)
Discovered: System.Net.Security.Tests (found 296 of 485 test cases)
Starting: System.Net.Security.Tests (parallel test collections = on, max threads = 4)
System.Net.Security.Tests.SslStreamEKUTest.SslStream_SelfSignedClientEKUClientAuth_Ok [SKIP]
Condition(s) not met: "IsRootCertificateInstalled"
System.Net.Security.Tests.SslStreamEKUTest.SslStream_ServerEKUClientAuth_Fails [SKIP]
Condition(s) not met: "IsRootCertificateInstalled"
System.Net.Security.Tests.SslStreamEKUTest.SslStream_ClientEKUServerAuth_Fails [SKIP]
Condition(s) not met: "IsRootCertificateInstalled"
System.Net.Security.Tests.SslStreamEKUTest.SslStream_NoEKUServerAuth_Ok [SKIP]
Condition(s) not met: "IsRootCertificateInstalled"
System.Net.Security.Tests.SslStreamEKUTest.SslStream_NoEKUClientAuth_Ok [SKIP]
Condition(s) not met: "IsRootCertificateInstalled"
System.Net.Security.Tests.SslStreamCertificateTrustTest.SslStream_SendCertificateTrust_ThrowsOnUnsupportedPlatform [SKIP]
Condition(s) not met: "DoesNotSupportSendingCustomCANamesInTls"
./RunTests.sh: line 168: 21 Segmentation fault (core dumped) "$RUNTIME_PATH/dotnet" exec --runtimeconfig System.Net.Security.Tests.runtimeconfig.json --depsfile System.Net.Security.Tests.deps.json xunit.console.dll System.Net.Security.Tests.dll -xml testResults.xml -nologo -nocolor -notrait category=IgnoreForCI -notrait category=OuterLoop -notrait category=failing $RSP_FILE
/root/helix/work/workitem/e
----- end Thu Jul 14 01:05:06 UTC 2022 ----- exit code 139 ----------------------------------------------------------
exit code 139 means SIGSEGV Illegal memory access. Deref invalid pointer, overrunning buffer, stack overflow etc. Core dumped.There is a how-to-debug.md in DevOps attachment: https://helixre107v0xdeko0k025g8.blob.core.windows.net/dotnet-runtime-refs-pull-72082-merge-93051a65a35d46238a/System.Net.Security.Tests/1/how-to-debug-dump.md?helixlogtype=result
I am having some issues with my RPi device, and couldn't get the callstack from dump myself. Also, docker run arm32v7/ubuntu (which uses QEMU) failed to run dotnet7 tool install --global dotnet-sos command (with and without DOTNET_EnableWriteXorExecute=0).
FYI @janvorli