Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 33 additions & 22 deletions gunicorn/arbiter.py
Original file line number Diff line number Diff line change
Expand Up @@ -523,36 +523,47 @@ def reap_workers(self):
# A worker was terminated. If the termination reason was
# that it could not boot, we'll shut it down to avoid
# infinite start/stop cycles.
exitcode = status >> 8
if exitcode != 0:
self.log.error('Worker (pid:%s) exited with code %s', wpid, exitcode)
if exitcode == self.WORKER_BOOT_ERROR:
reason = "Worker failed to boot."
raise HaltServer(reason, self.WORKER_BOOT_ERROR)
if exitcode == self.APP_LOAD_ERROR:
reason = "App failed to load."
raise HaltServer(reason, self.APP_LOAD_ERROR)

if exitcode > 0:
# If the exit code of the worker is greater than 0,
# let the user know.
self.log.error("Worker (pid:%s) exited with code %s.",
wpid, exitcode)
elif status > 0:
# If the exit code of the worker is 0 and the status
# is greater than 0, then it was most likely killed
# via a signal.

# Check if process was terminated by a signal
if os.WIFSIGNALED(status):
sig_number = os.WTERMSIG(status)
try:
sig_name = signal.Signals(status).name
sig_name = signal.Signals(sig_number).name
except ValueError:
sig_name = "code {}".format(status)
sig_name = "signal {}".format(sig_number)
msg = "Worker (pid:{}) was sent {}!".format(
wpid, sig_name)

# Additional hint for SIGKILL
if status == signal.SIGKILL:
if sig_number == signal.SIGKILL:
msg += " Perhaps out of memory?"
self.log.error(msg)
elif os.WIFEXITED(status):
exitcode = os.WEXITSTATUS(status)

# Check if exitcode indicates signal termination (shell convention: 128 + signal_number)
if exitcode > 128:
sig_number = exitcode - 128
try:
sig_name = signal.Signals(sig_number).name
except ValueError:
sig_name = "signal {}".format(sig_number)
msg = "Worker (pid:{}) was sent {}!".format(
wpid, sig_name)

# Additional hint for SIGKILL
if sig_number == signal.SIGKILL:
msg += " Perhaps out of memory?"
self.log.error(msg)
elif exitcode != 0:
self.log.error('Worker (pid:%s) exited with code %s', wpid, exitcode)

if exitcode == self.WORKER_BOOT_ERROR:
reason = "Worker failed to boot."
raise HaltServer(reason, self.WORKER_BOOT_ERROR)
if exitcode == self.APP_LOAD_ERROR:
reason = "App failed to load."
raise HaltServer(reason, self.APP_LOAD_ERROR)

worker = self.WORKERS.pop(wpid, None)
if not worker:
Expand Down
51 changes: 51 additions & 0 deletions tests/test_arbiter.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,3 +185,54 @@ def test_env_vars_available_during_preload():
# Note that we aren't making any assertions here, they are made in the
# dummy application object being loaded here instead.
gunicorn.arbiter.Arbiter(PreloadedAppWithEnvSettings())


@mock.patch('os.waitpid')
@mock.patch('os.WIFSIGNALED')
@mock.patch('os.WTERMSIG')
def test_arbiter_reap_workers_with_signal(mock_wtermsig, mock_wifsignaled, mock_os_waitpid):
"""Test that workers terminated by signals are logged with signal names."""
# Simulate SIGABRT (signal number 6)
mock_os_waitpid.side_effect = [(42, 6), (0, 0)]
mock_wifsignaled.return_value = True
mock_wtermsig.return_value = 6 # SIGABRT

arbiter = gunicorn.arbiter.Arbiter(DummyApplication())
arbiter.cfg.settings['child_exit'] = mock.Mock()
mock_worker = mock.Mock()
arbiter.WORKERS = {42: mock_worker}

with mock.patch.object(arbiter.log, 'error') as mock_log_error:
arbiter.reap_workers()
# Verify that the log message contains the signal name "SIGABRT"
assert mock_log_error.called
log_message = mock_log_error.call_args[0][0]
assert 'SIGABRT' in log_message
assert 'pid:42' in log_message or 'pid:{}'.format(42) in log_message


@mock.patch('os.waitpid')
@mock.patch('os.WIFSIGNALED')
@mock.patch('os.WIFEXITED')
@mock.patch('os.WEXITSTATUS')
def test_arbiter_reap_workers_with_exitcode_134(mock_wexitstatus, mock_wifexited,
mock_wifsignaled, mock_os_waitpid):
"""Test that workers exiting with code 134 (128+6) are logged as SIGABRT."""
# Simulate exit code 134 (128 + SIGABRT)
mock_os_waitpid.side_effect = [(42, 34304), (0, 0)] # 34304 = 134 << 8
mock_wifsignaled.return_value = False
mock_wifexited.return_value = True
mock_wexitstatus.return_value = 134

arbiter = gunicorn.arbiter.Arbiter(DummyApplication())
arbiter.cfg.settings['child_exit'] = mock.Mock()
mock_worker = mock.Mock()
arbiter.WORKERS = {42: mock_worker}

with mock.patch.object(arbiter.log, 'error') as mock_log_error:
arbiter.reap_workers()
# Verify that the log message contains the signal name "SIGABRT", not "code 134"
assert mock_log_error.called
log_message = mock_log_error.call_args[0][0]
assert 'SIGABRT' in log_message
assert 'code 134' not in log_message