fix flaky sequential/test-child-process-pass-fd on fedora 24

[Trott]

Example failure:

https://ci.nodejs.org/job/node-test-commit-linux/7552/nodes=fedora24/console

  duration_ms: 1.113
  severity: fail
  stack: |-
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: spawn /home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/out/Release/node EAGAIN
        at exports._errnoException (util.js:1023:11)
        at Process.ChildProcess._handle.onexit (internal/child_process.js:193:32)
        at onErrorNT (internal/child_process.js:359:16)
        at _combinedTickCallback (internal/process/next_tick.js:74:11)
        at process._tickCallback (internal/process/next_tick.js:98:9)
        at Module.runMain (module.js:607:11)
        at run (bootstrap_node.js:418:7)
        at startup (bootstrap_node.js:139:9)
        at bootstrap_node.js:533:3
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
  ...

/cc @santigimeno

[Trott]

Another example failure:

https://ci.nodejs.org/job/node-test-commit-linux/7544/nodes=fedora24/console

not ok 1318 sequential/test-child-process-pass-fd
  ---
  duration_ms: 1.112
  severity: fail
  stack: |-
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: spawn /home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/out/Release/node EAGAIN
        at exports._errnoException (util.js:1023:11)
        at Process.ChildProcess._handle.onexit (internal/child_process.js:193:32)
        at onErrorNT (internal/child_process.js:359:16)
        at _combinedTickCallback (internal/process/next_tick.js:74:11)
        at process._tickCallback (internal/process/next_tick.js:98:9)
        at Module.runMain (module.js:607:11)
        at run (bootstrap_node.js:418:7)
        at startup (bootstrap_node.js:139:9)
        at bootstrap_node.js:533:3
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)

[Trott]

Small data set so far, but these failures are both on test-rackspace-fedora24-x64-1 while test-digitalocean-fedora24-x64-1 always seems to pass. @nodejs/build

[Trott]

Another one: https://ci.nodejs.org/job/node-test-commit-linux/7552/nodes=fedora24/console

And once again, it's on test-rackspace-fedora24-x64-1.

[santigimeno]

The EAGAIN error on spawn suggests that the problem could be that a limit on number of processes running is reached. See http://man7.org/linux/man-pages/man2/fork.2.html#ERRORS. So it could be a configuration problem in the CI bot.

[Trott]

The EAGAIN error on spawn suggests that the problem could be that a limit on number of processes running is reached.

I wonder if it's something like that, but not quite exactly that.

[root@test-rackspace-fedora24-x64-1 ~]# ulimit -u
31822
[root@test-rackspace-fedora24-x64-1 ~]# ps -ax | wc -l
145
[root@test-rackspace-fedora24-x64-1 ~]#

It's hard for me to imagine that we're actually topping out over 31822 processes.

[santigimeno]

@Trott have you checked the other limits defined in http://man7.org/linux/man-pages/man2/fork.2.html#ERRORS (max # of threads and max # of pids )?

[Trott]

@santigimeno

[root@test-rackspace-fedora24-x64-1 ~]# ulimit -a
core file size          (blocks, -c) unlimited
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 31822
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 1024
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 31822
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited
[root@test-rackspace-fedora24-x64-1 ~]# cat /proc/sys/kernel/threads-max
63645
[root@test-rackspace-fedora24-x64-1 ~]#

[Trott]

Also, for future reference: Still happening.

https://ci.nodejs.org/job/node-test-commit-linux/7574/nodes=fedora24/console

not ok 1326 sequential/test-child-process-pass-fd
  ---
  duration_ms: 1.112
  severity: fail
  stack: |-
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: spawn /home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/out/Release/node EAGAIN
        at exports._errnoException (util.js:1023:11)
        at Process.ChildProcess._handle.onexit (internal/child_process.js:193:32)
        at onErrorNT (internal/child_process.js:359:16)
        at _combinedTickCallback (internal/process/next_tick.js:74:11)
        at process._tickCallback (internal/process/next_tick.js:98:9)
        at Module.runMain (module.js:607:11)
        at run (bootstrap_node.js:418:7)
        at startup (bootstrap_node.js:139:9)
        at bootstrap_node.js:533:3
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)

[Trott]

Still going on: https://ci.nodejs.org/job/node-test-commit-linux/7585/nodes=fedora24/console

not ok 1329 sequential/test-child-process-pass-fd
  ---
  duration_ms: 1.112
  severity: fail
  stack: |-
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: spawn /home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/out/Release/node EAGAIN
        at exports._errnoException (util.js:1023:11)
        at Process.ChildProcess._handle.onexit (internal/child_process.js:193:32)
        at onErrorNT (internal/child_process.js:359:16)
        at _combinedTickCallback (internal/process/next_tick.js:74:11)
        at process._tickCallback (internal/process/next_tick.js:98:9)
        at Module.runMain (module.js:607:11)
        at run (bootstrap_node.js:418:7)
        at startup (bootstrap_node.js:139:9)
        at bootstrap_node.js:533:3
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)
    events.js:161
          throw er; // Unhandled 'error' event
          ^
    
    Error: channel closed
        at process.target.send (internal/child_process.js:553:16)
        at Socket.socketConnected (/home/iojs/build/workspace/node-test-commit-linux/nodes/fedora24/test/sequential/test-child-process-pass-fd.js:39:15)
        at Object.onceWrapper (events.js:291:19)
        at emitNone (events.js:86:13)
        at Socket.emit (events.js:186:7)
        at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1077:10)