`System.IO.Tests` getting killed (OOM?) for `Libraries Test Run release mono Linux arm64 Debug`

[radical]

Build
Log

/root/helix/work/workitem /root/helix/work/workitem
  Discovering: System.IO.Tests (method display = ClassAndMethod, method display options = None)
  Discovered:  System.IO.Tests (found 678 of 681 test cases)
  Starting:    System.IO.Tests (parallel test collections = on, max threads = 6)
./RunTests.sh: line 162:    99 Killed                  "$RUNTIME_PATH/dotnet" exec --runtimeconfig System.IO.Tests.runtimeconfig.json --depsfile System.IO.Tests.deps.json xunit.console.dll System.IO.Tests.dll -xml testResults.xml -nologo -nocolor -notrait category=IgnoreForCI -notrait category=OuterLoop -notrait category=failing $RSP_FILE
/root/helix/work/workitem
----- end Thu Jul 22 01:29:17 UTC 2021 ----- exit code 137 ----------------------------------------------------------
exit code 137 means SIGKILL Killed eg by kill
ulimit -c value: unlimited
[463515.031418] [ 8294]  1000  8294    60030      559      19       5      498             0 gvfs-udisks2-vo
[463515.031421] [ 8295]  1000  8295    78109       93      19       4      292             0 gvfsd-trash
[463515.031424] [ 8326]  1000  8326    78002        0      19       5      787             0 gvfs-afc-volume
[463515.031428] [ 8337]  1000  8337    58449        2      15       4      219             0 gvfs-gphoto2-vo
[463515.031431] [ 8343]  1000  8343    96059        2      21       4      435             0 ibus-daemon
[463515.031434] [ 8348]  1000  8348   130049        2      43       3      900             0 evolution-addre
[463515.031437] [ 8351]  1000  8351    57949        0      14       4      185             0 gvfs-mtp-volume
[463515.031440] [ 8361]  1000  8361    57980        2      15       3      186             0 gvfs-goa-volume
[463515.031444] [ 8364]  1000  8364    58934        0      16       3      252             0 ibus-dconf
[463515.031447] [ 8369]  1000  8369    64870        0      28       4     1173             0 ibus-x11
[463515.031450] [ 8377]  1000  8377    58911        0      18       5      288             0 ibus-portal
[463515.031453] [ 8379]  1000  8379    40464        0      14       3      238             0 ibus-engine-sim
[463515.031456] [ 8403]  1000  8403   220173      181      58       5      802             0 evolution-addre
[463515.031459] [ 8515]  1000  8515      568        0       6       4       32             0 sh
[463515.031462] [ 8516]  1000  8516    58097        0      15       3      230             0 pxgsettings
[463515.031465] [ 8522]  1000  8522    39593        0      12       4      161             0 gvfsd-metadata
[463515.031468] [ 8545]  1000  8545    96589        0      22       4      853             0 gvfsd-network
[463515.031472] [ 8557]  1000  8557    77650       11      18       4      255             0 gvfsd-dnssd
[463515.031475] [ 8581]  1000  8581   137784        0      35       4     1070             0 zeitgeist-datah
[463515.031478] [ 8593]  1000  8593    95411      159      19       3      126             0 zeitgeist-daemo
[463515.031481] [ 8599]  1000  8599    66614        0      20       5      321             0 zeitgeist-fts
[463515.031484] [ 8715]  1000  8715   249740      772     152       5    49607             0 gnome-software
[463515.031487] [ 8764]  1000  8764   120880      118      36       5     1171             0 update-notifier
[463515.031490] [ 8831]  1000  8831      568        0       6       3       32             0 sh
[463515.031494] [ 8834]  1000  8834    98246       57      50       4     4887             0 check-new-relea
[463515.031497] [ 8997]     0  8997    96290       52      42       3     6192             0 fwupd
[463515.031500] [ 9009]  1000  9009      568        0       5       5       31             0 sh
[463515.031504] [ 9010]  1000  9010    58097        0      14       3      232             0 pxgsettings
[463515.031507] [ 9073]  1000  9073   174623     3572      94       4    10692             0 update-manager
[463515.031510] [ 9969]  1000  9969   128051      160      51       3     1378             0 deja-dup-monito
[463515.031513] [10442]  1000 10442   112670        0      57       3     2651             0 deja-dup
[463515.031516] [13810]  1000 13810    91547      383      36       3     1152             0 unity-panel-ser
[463515.031520] [28552]  1000 28552    98257       55      49       3     4894             0 check-new-relea
[463515.031526] [ 6171]     0  6171     2530      249      10       5        0          -999 containerd-shim
[463515.031529] [ 6230]  1001  6230      467        0       5       3       22             0 helix_docker_wo
[463515.031532] [ 6380]  1001  6380      467        0       4       3       24             0 execute.sh
[463515.031535] [ 6382]  1001  6382     1272        2       6       3       82             0 bash
[463515.031539] [ 6392]  1001  6392  2496741  1411294    2938      11    17607             0 dotnet
[463515.031542] [10040]     0 10040     1190       16       7       4        0             0 sleep
[463515.031545] [10169]     0 10169     3921      138       8       3      299             0 systemd-udevd
[463515.031548] [10170]     0 10170     3921      117       8       3      318             0 systemd-udevd
[463515.031552] [10171]     0 10171     3921      109       8       3      325             0 systemd-udevd
[463515.031554] [10172]     0 10172     3921      109       8       3      325             0 systemd-udevd
[463515.031557] [10173]     0 10173     3921      150       8       3      287             0 systemd-udevd
[463515.031561] [10174]     0 10174     3921      145       8       3      292             0 systemd-udevd
[463515.031564] [10198]     0 10198     3921      150       8       3      287             0 systemd-udevd
[463515.031567] [10201]     0 10201     3921      109       8       3      325             0 systemd-udevd
[463515.031570] [10202]     0 10202     3921      105       8       3      329         -1000 systemd-udevd
[463515.031573] Out of memory: Kill process 6392 (dotnet) score 474 or sacrifice child
[463515.054897] Killed process 6392 (dotnet) total-vm:9986964kB, anon-rss:5645176kB, file-rss:0kB, shmem-rss:0kB

I couldn't figure out the best area label to add to this issue. If you have write-permissions please help me learn by adding exactly one area label.

Tagging subscribers to this area: @dotnet/area-system-io
See info in area-owners.md if you want to be subscribed.

Issue Details

---

Build
Log

/root/helix/work/workitem /root/helix/work/workitem
  Discovering: System.IO.Tests (method display = ClassAndMethod, method display options = None)
  Discovered:  System.IO.Tests (found 678 of 681 test cases)
  Starting:    System.IO.Tests (parallel test collections = on, max threads = 6)
./RunTests.sh: line 162:    99 Killed                  "$RUNTIME_PATH/dotnet" exec --runtimeconfig System.IO.Tests.runtimeconfig.json --depsfile System.IO.Tests.deps.json xunit.console.dll System.IO.Tests.dll -xml testResults.xml -nologo -nocolor -notrait category=IgnoreForCI -notrait category=OuterLoop -notrait category=failing $RSP_FILE
/root/helix/work/workitem
----- end Thu Jul 22 01:29:17 UTC 2021 ----- exit code 137 ----------------------------------------------------------
exit code 137 means SIGKILL Killed eg by kill
ulimit -c value: unlimited
[463515.031418] [ 8294]  1000  8294    60030      559      19       5      498             0 gvfs-udisks2-vo
[463515.031421] [ 8295]  1000  8295    78109       93      19       4      292             0 gvfsd-trash
[463515.031424] [ 8326]  1000  8326    78002        0      19       5      787             0 gvfs-afc-volume
[463515.031428] [ 8337]  1000  8337    58449        2      15       4      219             0 gvfs-gphoto2-vo
[463515.031431] [ 8343]  1000  8343    96059        2      21       4      435             0 ibus-daemon
[463515.031434] [ 8348]  1000  8348   130049        2      43       3      900             0 evolution-addre
[463515.031437] [ 8351]  1000  8351    57949        0      14       4      185             0 gvfs-mtp-volume
[463515.031440] [ 8361]  1000  8361    57980        2      15       3      186             0 gvfs-goa-volume
[463515.031444] [ 8364]  1000  8364    58934        0      16       3      252             0 ibus-dconf
[463515.031447] [ 8369]  1000  8369    64870        0      28       4     1173             0 ibus-x11
[463515.031450] [ 8377]  1000  8377    58911        0      18       5      288             0 ibus-portal
[463515.031453] [ 8379]  1000  8379    40464        0      14       3      238             0 ibus-engine-sim
[463515.031456] [ 8403]  1000  8403   220173      181      58       5      802             0 evolution-addre
[463515.031459] [ 8515]  1000  8515      568        0       6       4       32             0 sh
[463515.031462] [ 8516]  1000  8516    58097        0      15       3      230             0 pxgsettings
[463515.031465] [ 8522]  1000  8522    39593        0      12       4      161             0 gvfsd-metadata
[463515.031468] [ 8545]  1000  8545    96589        0      22       4      853             0 gvfsd-network
[463515.031472] [ 8557]  1000  8557    77650       11      18       4      255             0 gvfsd-dnssd
[463515.031475] [ 8581]  1000  8581   137784        0      35       4     1070             0 zeitgeist-datah
[463515.031478] [ 8593]  1000  8593    95411      159      19       3      126             0 zeitgeist-daemo
[463515.031481] [ 8599]  1000  8599    66614        0      20       5      321             0 zeitgeist-fts
[463515.031484] [ 8715]  1000  8715   249740      772     152       5    49607             0 gnome-software
[463515.031487] [ 8764]  1000  8764   120880      118      36       5     1171             0 update-notifier
[463515.031490] [ 8831]  1000  8831      568        0       6       3       32             0 sh
[463515.031494] [ 8834]  1000  8834    98246       57      50       4     4887             0 check-new-relea
[463515.031497] [ 8997]     0  8997    96290       52      42       3     6192             0 fwupd
[463515.031500] [ 9009]  1000  9009      568        0       5       5       31             0 sh
[463515.031504] [ 9010]  1000  9010    58097        0      14       3      232             0 pxgsettings
[463515.031507] [ 9073]  1000  9073   174623     3572      94       4    10692             0 update-manager
[463515.031510] [ 9969]  1000  9969   128051      160      51       3     1378             0 deja-dup-monito
[463515.031513] [10442]  1000 10442   112670        0      57       3     2651             0 deja-dup
[463515.031516] [13810]  1000 13810    91547      383      36       3     1152             0 unity-panel-ser
[463515.031520] [28552]  1000 28552    98257       55      49       3     4894             0 check-new-relea
[463515.031526] [ 6171]     0  6171     2530      249      10       5        0          -999 containerd-shim
[463515.031529] [ 6230]  1001  6230      467        0       5       3       22             0 helix_docker_wo
[463515.031532] [ 6380]  1001  6380      467        0       4       3       24             0 execute.sh
[463515.031535] [ 6382]  1001  6382     1272        2       6       3       82             0 bash
[463515.031539] [ 6392]  1001  6392  2496741  1411294    2938      11    17607             0 dotnet
[463515.031542] [10040]     0 10040     1190       16       7       4        0             0 sleep
[463515.031545] [10169]     0 10169     3921      138       8       3      299             0 systemd-udevd
[463515.031548] [10170]     0 10170     3921      117       8       3      318             0 systemd-udevd
[463515.031552] [10171]     0 10171     3921      109       8       3      325             0 systemd-udevd
[463515.031554] [10172]     0 10172     3921      109       8       3      325             0 systemd-udevd
[463515.031557] [10173]     0 10173     3921      150       8       3      287             0 systemd-udevd
[463515.031561] [10174]     0 10174     3921      145       8       3      292             0 systemd-udevd
[463515.031564] [10198]     0 10198     3921      150       8       3      287             0 systemd-udevd
[463515.031567] [10201]     0 10201     3921      109       8       3      325             0 systemd-udevd
[463515.031570] [10202]     0 10202     3921      105       8       3      329         -1000 systemd-udevd
[463515.031573] Out of memory: Kill process 6392 (dotnet) score 474 or sacrifice child
[463515.054897] Killed process 6392 (dotnet) total-vm:9986964kB, anon-rss:5645176kB, file-rss:0kB, shmem-rss:0kB

Author:	radical
Assignees:	-
Labels:	area-System.IO, untriaged
Milestone:	-

[buyaa-n]

failed again in build, log:

C:\h\w\ACCB09AD\w\B19709CA\e>"C:\h\w\ACCB09AD\p\dotnet.exe" exec --runtimeconfig System.IO.Tests.runtimeconfig.json --depsfile System.IO.Tests.deps.json xunit.console.dll System.IO.Tests.dll -xml testResults.xml -nologo -nocolor -trait category=OuterLoop -notrait category=IgnoreForCI -notrait category=failing  
  Discovering: System.IO.Tests (method display = ClassAndMethod, method display options = None)
  Discovered:  System.IO.Tests (found 6 of 685 test cases)
  Starting:    System.IO.Tests (parallel test collections = on, max threads = 2)
   System.IO.Tests: [Long Running Test] 'System.IO.Tests.BufferedStream_StreamAsync.NoInt32OverflowInTheBufferingLogic', Elapsed: 00:02:11
   System.IO.Tests: [Long Running Test] 'System.IO.Tests.BufferedStream_StreamAsync.NoInt32OverflowInTheBufferingLogic', Elapsed: 00:04:11
   System.IO.Tests: [Long Running Test] 'System.IO.Tests.BufferedStream_StreamAsync.NoInt32OverflowInTheBufferingLogic', Elapsed: 00:06:12
   System.IO.Tests: [Long Running Test] 'System.IO.Tests.BufferedStream_StreamAsync.NoInt32OverflowInTheBufferingLogic', Elapsed: 00:08:12
   System.IO.Tests: [Long Running Test] 'System.IO.Tests.BufferedStream_StreamAsync.NoInt32OverflowInTheBufferingLogic', Elapsed: 00:10:12
   System.IO.Tests: [Long Running Test] 'System.IO.Tests.BufferedStream_StreamAsync.NoInt32OverflowInTheBufferingLogic', Elapsed: 00:12:12
   System.IO.Tests: [Long Running Test] 'System.IO.Tests.BufferedStream_StreamAsync.NoInt32OverflowInTheBufferingLogic', Elapsed: 00:14:12
['System.IO.Tests' END OF WORK ITEM LOG: Command timed out, and was killed]

[danmoseley]

I am seeing a failure which I suspect is another manifestation:

    <test name="System.IO.Tests.FileStream_Read.NoInt32OverflowInTheBufferingLogic" type="System.IO.Tests.FileStream_Read" method="NoInt32OverflowInTheBufferingLogic" time="1.4505152" result="Fail">
        <traits>
          <trait name="category" value="outerloop" />
        </traits>
        <failure exception-type="System.IO.IOException">
          <message><![CDATA[System.IO.IOException : There is not enough space on the disk. : 'C:\\Users\\danmose\\AppData\\Local\\Temp\\FileStream_Read_veaai11q.g5k\\NoInt32OverflowInTheBufferingLogic_26_5e4f963f']]></message>
          <stack-trace><![CDATA[   at System.IO.RandomAccess.WriteAtOffset(SafeFileHandle handle, ReadOnlySpan`1 buffer, Int64 fileOffset) in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\RandomAccess.Windows.cs:line 153
   at System.IO.Strategies.OSFileStreamStrategy.Write(ReadOnlySpan`1 buffer) in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\Strategies\OSFileStreamStrategy.cs:line 282
   at System.IO.Strategies.OSFileStreamStrategy.Write(Byte[] buffer, Int32 offset, Int32 count) in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\Strategies\OSFileStreamStrategy.cs:line 269
   at System.IO.Strategies.BufferedFileStreamStrategy.FlushWrite() in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\Strategies\BufferedFileStreamStrategy.cs:line 1020
   at System.IO.Strategies.BufferedFileStreamStrategy.Flush(Boolean flushToDisk) in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\Strategies\BufferedFileStreamStrategy.cs:line 804
   at System.IO.Strategies.BufferedFileStreamStrategy.Dispose(Boolean disposing) in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\Strategies\BufferedFileStreamStrategy.cs:line 150
   at System.IO.Stream.Dispose() in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\Stream.cs:line 158
   at System.IO.Tests.FileStream_Read.NoInt32OverflowInTheBufferingLogic() in C:\git\runtime\src\libraries\System.IO.FileSystem\tests\FileStream\Read.cs:line 38]]></stack-trace>

This test is reading and writing a few bytes at the start and end of a 4GB file. Is this actually reading and writing 4GB, so it's taking a very long time? Is this related to running out of disk space also? It's not clear as my disk seems to have 14GB free space.

cc @adamsitnik ?

[danmoseley]

Ah I see two other failures concurrently - both of these reading/writing multi gigabyte files. Still, not entirely clear why they all failed if (at least after tests were done) I had 14GB.

Perhaps these 3 tests should be segregated into a fixture where they are run serially?

      <test name="System.IO.Tests.File_ReadWriteAllBytesAsync.ReadFileOverMaxArrayLengthAsync" type="System.IO.Tests.File_ReadWriteAllBytesAsync" method="ReadFileOverMaxArrayLengthAsync" time="0.0078009" result="Fail">
        <traits>
          <trait name="category" value="outerloop" />
        </traits>
        <failure exception-type="System.IO.IOException">
          <message><![CDATA[System.IO.IOException : There is not enough space on the disk. : 'C:\\Users\\danmose\\AppData\\Local\\Temp\\File_ReadWriteAllBytesAsync_b3sf1ema.fvq\\ReadFileOverMaxArrayLengthAsync_93_c50ebc9e']]></message>
          <stack-trace><![CDATA[   at System.IO.Strategies.FileStreamHelpers.SetFileLength(SafeFileHandle handle, Int64 length) in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\Strategies\FileStreamHelpers.Windows.cs:line 128
   at System.IO.Strategies.OSFileStreamStrategy.SetLengthCore(Int64 value) in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\Strategies\OSFileStreamStrategy.cs:line 226
   at System.IO.Tests.File_ReadWriteAllBytesAsync.ReadFileOverMaxArrayLengthAsync() in C:\git\runtime\src\libraries\System.IO.FileSystem\tests\File\ReadWriteAllBytesAsync.cs:line 96
--- End of stack trace from previous location ---]]></stack-trace>
        </failure>
      </test>
      <test name="System.IO.Tests.File_ReadWriteAllBytesAsync.ReadFileOver2GBAsync" type="System.IO.Tests.File_ReadWriteAllBytesAsync" method="ReadFileOver2GBAsync" time="0.0081179" result="Fail">
        <traits>
          <trait name="category" value="outerloop" />
        </traits>
        <failure exception-type="System.IO.IOException">
          <message><![CDATA[System.IO.IOException : There is not enough space on the disk. : 'C:\\Users\\danmose\\AppData\\Local\\Temp\\File_ReadWriteAllBytesAsync_lvobmbw2.21p\\ReadFileOver2GBAsync_78_fb9b2321']]></message>
          <stack-trace><![CDATA[   at System.IO.Strategies.FileStreamHelpers.SetFileLength(SafeFileHandle handle, Int64 length) in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\Strategies\FileStreamHelpers.Windows.cs:line 128
   at System.IO.Strategies.OSFileStreamStrategy.SetLengthCore(Int64 value) in C:\git\runtime\src\libraries\System.Private.CoreLib\src\System\IO\Strategies\OSFileStreamStrategy.cs:line 226
   at System.IO.Tests.File_ReadWriteAllBytesAsync.ReadFileOver2GBAsync() in C:\git\runtime\src\libraries\System.IO.FileSystem\tests\File\ReadWriteAllBytesAsync.cs:line 81
--- End of stack trace from previous location ---]]></stack-trace>
        </failur