procdog

#!/usr/bin/env python
r'''
Procdog is a simple command-line tool to start, stop, and check the health of
processes.

It is intended to be a simple, cross-platform alternative to standard tools
like System V service scripts, Upstart, and systemd, for use in development, in
build systems and test harnesses, etc.

It operates by starting a simple daemon that popen()s and monitors the process.
The daemon listens and accepts commands on a local Unix domain socket, making
it possible to check the process is running or terminate it, and to do basic
health checks. Every process is independent: There is a single Procdog daemon
for each monitored process. The daemon exits if the process terminates.

Many options can be set via a configuration file.

For further documentation, see: https://github.com/jlevy/procdog
'''

# Author: Joshua Levy
# Created: 2015-03-14

from __future__ import print_function
import socket, sys, os, errno, time, shlex, ConfigParser, argparse, re, traceback
from collections import namedtuple
from string import Template
from datetime import datetime
from threading import Thread, Lock
from Queue import Queue
# The subprocess module has known threading issues, so prefer subprocess32.
try:
    import subprocess32 as subprocess
except ImportError:
    import subprocess

NAME = "procdog"
VERSION = "0.1.13"
DESCRIPTION = "procdog: Lightweight command-line process control"
LONG_DESCRIPTION = __doc__

SOCKET_PATH_PAT = "/var/tmp/" + NAME + ".%s.sock"
LOCK_PATH_PAT = "/var/tmp/" + NAME + ".%s.lock"
LOG_PATH_PAT = "/var/tmp/" + NAME + ".%s.log"

CONFIG_FILE = "procdog.cfg"
CONFIG_DOTFILE = "." + CONFIG_FILE

# Commands.
CMD_START = "start"  # Not a command sent to monitor.
CMD_STATUS = "status"
CMD_WAIT = "wait"
CMD_STOP = "stop"

CMD_LIST = [CMD_START, CMD_STATUS, CMD_WAIT, CMD_STOP]

# Response codes.
RESP_STOPPED = "stopped"  # Not a command returned from monitor.
RESP_RUNNING = "running"
RESP_KILLED = "killed"
RESP_EXITED = "exited"
RESP_INVALID = "invalid"
RESP_ERROR = "error"

RESP_LIST = [RESP_STOPPED, RESP_RUNNING, RESP_KILLED, RESP_EXITED, RESP_INVALID, RESP_ERROR]

# Exit codes from this command-line tool.
EXIT_OK = 0
EXIT_ERROR = 2
EXIT_UNKNOWN = 3
EXIT_ALREADY_RUNNING = 4
EXIT_NOT_RUNNING = 5
EXIT_NOT_HEALTHY = 6

# Return code expected for a valid health check.
HEALTHY_CODE = 0

EOM = "\n"

# Timeouts to connect and send to local Unix domain socket.
SOCKET_TIMEOUT = 3
SEND_TIMEOUT = 10

DEBUG = False

# Log buffering. 0=unbuffered, 1=line buffered, otherwise the buffer size.
LOG_BUFFERING = 1

_UTC_TIMESTAMP = datetime.utcnow().isoformat() + 'Z'

# Workarounds for lack of Python 2 subprocess /dev/null support
_NULL_INPUT = open(os.devnull, "r")
_NULL_OUTPUT = open(os.devnull, "w")

# Lock is only local to process, but still helps serialize multiple monitor thread logs.
_LOG_LOCK = Lock()


# Exceptions
class StateException(Exception):
    pass


class NotRunningException(StateException):
    pass


class MonitorNotRunning(NotRunningException):
    pass


class AlreadyRunningException(StateException):
    pass


class OtherException(StateException):
    pass


def unlink_path(path, silent=False):
    '''Slightly more robust unlink with a silent fail option.'''
    if silent:
        try:
            os.unlink(path)
        except:
            pass
    else:
        try:
            os.unlink(path)
        except OSError:
            if os.path.exists(path):
                raise


def file_lock_acquire(path, contents=""):
    '''Try to acquire a file-based lock, creating a file only if it does not exist. This is a simple approach
     that should be relatively cross platform, without flock() or fcntl(), and simply uses POSIX O_EXCL.
     No timeouts. Returns with True if the lock is acquired or False if it is not available. More
     discussion on related topics:
     http://stackoverflow.com/questions/489861/locking-a-file-in-python
     http://stackoverflow.com/questions/688343/reference-for-proper-handling-of-pid-file-on-unix
  '''
    try:
        fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o644)
        with os.fdopen(fd, 'a') as f:
            f.write(contents)
        return True
    except OSError as e:
        if e.errno == errno.EEXIST:
            return False
        else:
            raise OSError("unexpected error locking file '%s': %s" % (path, e))


def file_lock_release(path):
    unlink_path(path, silent=False)


# FIXME use this
class Process:
    '''Name and tracking data for the process to be managed.'''

    def __init__(self, name):
        self.name = name
        self.socket_path = SOCKET_PATH_PAT % name
        self.lock_path = LOCK_PATH_PAT % name
        self.log_path = LOG_PATH_PAT % name

    def acquire_lock(self):
        return file_lock_acquire(self.lock_path)

    def release_lock(self):
        return file_lock_release(self.lock_path)


class Response:
    '''Represents responses sent by the monitor daemon. Responses are errors like
     "error: some message" or status codes followed by key=value pairs, like
     "running, health=xxx, pid=yyy".'''

    def __init__(self, raw=None, code=None, error=None, **kwargs):
        if raw is not None:
            if raw.startswith(RESP_ERROR + ": "):
                (self.code, self.error) = raw.split(": ", 1)
                self.values = None
            else:
                bits = raw.split(", ")
                self.code = bits[0]
                self.values = {key: val for (key, val) in [pair.split("=") for pair in bits[1:]]}
                self.error = None
        else:
            self.code = RESP_ERROR if error is not None else code
            self.error = error
            self.values = kwargs
        self._validate()

    def _validate(self):
        if self.code not in RESP_LIST:
            raise ValueError("invalid response code: '%s'" % self.code)

    def is_running(self):
        return self.code == RESP_RUNNING

    def is_healthy(self):
        return self.is_running() and int(self.values.get("health", -1)) == 0

    def exit_code(self, control_cmd, strict=False, ensure_healthy=False):
        if control_cmd == CMD_WAIT:
            ensure_healthy = True  # The point to the wait command is to get health.
        if self.code == RESP_INVALID:
            return EXIT_UNKNOWN
        elif self.code == RESP_ERROR:
            return EXIT_ERROR
        elif control_cmd == CMD_START or control_cmd == CMD_STATUS or control_cmd == CMD_WAIT:
            if ensure_healthy:
                return EXIT_OK if self.is_healthy() else EXIT_NOT_HEALTHY
            else:
                return EXIT_OK if self.is_running() else EXIT_NOT_RUNNING
        elif control_cmd == CMD_STOP:
            if self.code == RESP_KILLED:
                return EXIT_OK
            elif self.code == RESP_STOPPED or self.code == RESP_EXITED:
                return EXIT_NOT_RUNNING if strict else EXIT_OK
            elif self.is_running():  # For completeness, but should be an exception.
                return EXIT_ALREADY_RUNNING
            else:
                return EXIT_UNKNOWN  # Shouldn't happen.
        else:
            assert False

    def encode(self):
        if self.error is not None:
            return "%s: %s" % (RESP_ERROR, self.error)
        else:
            out = [self.code] + ["%s=%s" % (k, v) for (k, v) in sorted(self.values.items()) if v is not None]
            return ", ".join(out)

    def __str__(self):
        return self.encode()

    def __repr__(self):
        return "Response(raw=%s)" % repr(self.encode())


def _debug(msg):
    global DEBUG
    if DEBUG:
        with _LOG_LOCK:
            print("%s (%s): debug: %s" % (NAME, os.getpid(), msg), file=sys.stderr)


def _info(msg, exc_info=None):
    with _LOG_LOCK:
        exc_str = "\n%s" % "".join(traceback.format_exception(*exc_info)) if exc_info else ""
        print("%s (%s): %s%s" % (NAME, os.getpid(), msg, exc_str), file=sys.stderr)


def _die(msg, code=1, exceptional=False):
    print("%s: error: %s" % (NAME, msg), file=sys.stderr)
    global DEBUG
    if DEBUG:
        traceback.print_exc(file=sys.stderr)
    sys.exit(code)


def _recv_msg(sock):
    '''Receive a (short) newline-delimited command. Trims whitespace.'''
    data = ""
    while True:
        more_data = sock.recv(1024)
        data += more_data
        if not more_data or more_data.find(EOM) >= 0:
            break
    return data.split(EOM)[0].strip()


class _ListenThread(Thread):
    '''Listen on Unix domain socket, adding all connections to the given queue.'''

    def __init__(self, socket_path, queue):
        Thread.__init__(self)
        self.socket_path = socket_path
        self.queue = queue
        self.daemon = True  # Do not prevent process termination on shutdown.

    def run(self):
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        sock.bind(self.socket_path)
        sock.listen(16)
        _info("monitor: listening on %s" % self.socket_path)

        while True:
            connection, client_address = sock.accept()
            _debug("monitor: listen accept")
            self.queue.put(connection)


class _HandleThread(Thread):
    '''We use a single thread (not a pool) since this is thread is only to monitor a single process.
     All request handling, including the wait with health check, happen serially.'''

    def __init__(self, queue, handler):
        Thread.__init__(self)
        self.queue = queue
        self.handler = handler
        self.daemon = True  # Do not prevent process termination on shutdown.

    def run(self):
        failure_count = 0
        while True:
            connection = self.queue.get(True)
            try:
                cmd = _recv_msg(connection)
                _debug("monitor: received command '%s'" % cmd)
                response = self.handler(cmd)
                connection.sendall(response + EOM)
            except:
                # Catch anything not already caught by the handler (e.g. a broken pipe on send).
                failure_count += 1
                if failure_count < 5:
                    _info("monitor: handler exception", exc_info=sys.exc_info())
                else:
                    raise
            finally:
                connection.close()


def _health_check(health_cmd, count=None, shell=False):
    _debug("monitor: running health check: command: %s" % health_cmd)
    args = health_cmd if shell else shlex.split(health_cmd)
    health = subprocess.call(args, stdin=_NULL_INPUT, stdout=_NULL_OUTPUT, stderr=_NULL_OUTPUT, shell=shell)
    _debug("monitor: health check%s: result: %s" % ("" if count is None else " %s" % count, health))
    return health


def _wait_for_health(popen, options):
    '''Wait for the given process to exit, the health command to return true, or the total health
     checks to expire. Return final health code, or None if process already exited.'''
    health = None
    for count in range(0, options.health_count):
        if popen.poll() is not None:
            return None
        health = _health_check(options.health_command, count=count)
        if health == HEALTHY_CODE:
            return health
        time.sleep(options.health_delay)
    return health


def _handle_command(cmd, popen, options):
    '''Handle the given command and return a result, using the given popen to get info
     about the process.'''
    try:
        health_command = options.health_command
        health_count = options.health_count
        health_delay = options.health_delay
        # Treat an empty command like a status command for convenience. This way you can cat the
        # socket and see status.
        if cmd == "" or cmd == CMD_STATUS:
            health = None
            if health_command:
                health = _health_check(health_command)
            if popen.poll() is None:
                resp = Response(code=RESP_RUNNING, pid=popen.pid, health=health)
            else:
                resp = Response(code=RESP_EXITED, returncode=popen.returncode, health=health)
        elif cmd == CMD_WAIT:
            if health_command:
                health = _wait_for_health(popen, options)
            if popen.poll() is None:
                resp = Response(code=RESP_RUNNING, pid=popen.pid, health=health)
            else:
                resp = Response(code=RESP_EXITED, returncode=popen.returncode)
        elif cmd == CMD_STOP:
            if popen.poll() is None:
                # TODO: Handle additional signals besides 15
                os.killpg(os.getpgid(popen.pid), 15)
                popen.wait()
                resp = Response(code=RESP_KILLED, signal=15)
            else:
                resp = Response(code=RESP_EXITED, returncode=popen.returncode)
        else:
            resp = Response(code=RESP_INVALID)
    except (OSError, IOError) as e:
        _info("exception handling command '%s': %s" % (cmd, e))
        resp = Response(error=e)

    _info("monitor: command '%s': response '%s'" % (cmd, resp))
    return resp


def _open_logfile(filename, write_mode, proc_name, log_options=None):
    f = open(filename, write_mode, LOG_BUFFERING)
    if log_options:
        f.write("%s: %s Starting process '%s' with options: %s\n" % (NAME, _UTC_TIMESTAMP, proc_name, log_options))
        f.flush()
    return f


def daemonize(home_dir="/", umask=077, do_redirect=True, stdout=None, stderr=None):
    '''Do a standard double fork to daemonize. Based on portions of:
     http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/www.boxedice.com
     https://github.com/serverdensity/python-daemon
  '''

    # Do first fork and return control to calling process.
    try:
        pid = os.fork()
        if pid > 0:
            return False
    except OSError, e:
        sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
        sys.exit(1)

    # Decouple from parent environment.
    os.chdir(home_dir)
    os.setsid()
    os.umask(umask)

    # Do second fork.
    try:
        pid = os.fork()
        if pid > 0:
            sys.exit(0)  # Exit second parent.
    except OSError, e:
        sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
        sys.exit(1)

    # Redirect standard file descriptors.
    if do_redirect:
        sys.stdout.flush()
        sys.stderr.flush()

        so = file(stdout, 'a+') if stdout else _NULL_OUTPUT
        se = file(stderr, 'a+', 0) if stderr else so

        os.dup2(_NULL_INPUT.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

    # TODO: Change from Python signal handler?
    return True


def daemonize_and_monitor(proc_name, shell_cmd, options):
    '''Start a monitor daemon that forks the given shell command, listens, and handles commands via a
     control socket.'''

    # Remove the socket path now, before forking, so that client can't connect until
    # daemon is running.
    socket_path = SOCKET_PATH_PAT % proc_name
    unlink_path(socket_path, silent=False)

    # Deamonize. If we're debugging, keep all debug output on the console.
    global DEBUG
    is_daemon = daemonize(home_dir=options.dir, do_redirect=not DEBUG, stdout=LOG_PATH_PAT % proc_name)

    if is_daemon:
        _info("monitor: starting with %s" % (options, ))

        try:
            # Start listening immediately so any client needn't wait long to know we're already running.
            queue = Queue()
            listen_thread = _ListenThread(socket_path, queue)
            listen_thread.start()

            popen = None
            error = None

            # Open file descriptors for stdout/stderr if needed. We always set stdin to /dev/null.
            # TODO: Consider closing of these on process exit.
            write_mode = 'a' if options.append else 'w'
            fin = _NULL_INPUT
            fout = None
            ferr = None
            # Note paths here are relative to options.dir.
            if options.stdin:
                fin = open(options.stdin, 'r')
            if options.stdout:
                fout = _open_logfile(options.stdout, write_mode, proc_name, log_options=options)
            if options.stderr:
                if options.stderr == options.stdout:
                    ferr = fout
                else:
                    ferr = _open_logfile(options.stderr, write_mode, proc_name, log_options=options)

            # Start process, tracking any immediate errors.
            try:
                args = shell_cmd if options.shell else shlex.split(shell_cmd)
                popen = subprocess.Popen(args,
                                         stdin=fin,
                                         stdout=fout,
                                         stderr=ferr,
                                         shell=options.shell,
                                         preexec_fn=os.setsid)
            except OSError as e:
                error = "Failed to start: %s: %s" % (e, shell_cmd)
                _info("monitor: %s" % error)
            if popen and popen.poll() is not None:
                error = "Failure starting (code %s): %s" % (popen.poll(), shell_cmd)
                _info("monitor: %s" % error)

            # If there was an error starting the process, communicate that to original client.
            if popen:
                handler = lambda cmd: _handle_command(cmd, popen, options).encode()
            else:
                handler = lambda cmd: Response(error=error).encode()

            handle_thread = _HandleThread(queue, handler)
            handle_thread.start()

            if popen:
                exit_code = popen.wait()
                _info("monitor: process done (code %s), exiting" % exit_code)

        finally:
            # We release the lock for standard situations but don't install a signal handler, since it's
            # probably best for clarity if the lock file is in place if something strange happened
            # (like a kill directly on procdog).
            file_lock_release(LOCK_PATH_PAT % proc_name)

        time.sleep(options.linger)
        # Just for cleanliness, best effort at removing the path when we're done; though if it remains
        # there we will just remove and re-bind on next startup.
        unlink_path(socket_path, silent=True)
        sys.exit(0)

    else:
        _wait_for_connect(proc_name)


def send_command(proc_name, cmd):
    '''Send a command to the monitor process with the given name via its control socket.'''

    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    try:
        # Set a short timeout on connect, in case process not running. Once we connect, disable the timeout.
        sock.settimeout(SOCKET_TIMEOUT)
        sock.connect(SOCKET_PATH_PAT % proc_name)
        _debug("client: connected")
        sock.settimeout(SEND_TIMEOUT)
        sock.sendall(cmd + EOM)
        data = _recv_msg(sock)
        if not data:
            # A rare race or a bug.
            raise NotRunningException("no data from monitor")
        _debug("client: %s: %s" % (cmd, data))
        return Response(raw=data)
    except socket.error as e:
        # Detect problematic corner cases where monitor has been killed, crashed, tampered with, etc.
        # If we get connection refused (61) process can't be running.
        # If we get file not found (2), the socket file is missing.
        if e.errno == 61 or e.errno == 2:
            raise MonitorNotRunning("couldn't connect: %s" % e)
        else:
            raise NotRunningException("socket error: %s" % e)
    finally:
        sock.close()


def _wait_for_connect(proc_name):
    _debug("client: wait: checking for connect...")
    timeout = time.time() + SOCKET_TIMEOUT
    exc = None
    while time.time() < timeout:
        try:
            send_command(proc_name, CMD_STATUS)
            _debug("client: wait: connect success")
            return
        except NotRunningException as e:
            exc = e
            time.sleep(.1)
    raise OtherException("timeout connecting to monitor: %s" % exc)


def _start_monitor(proc_name, shell_cmd, options):
    daemonize_and_monitor(proc_name, shell_cmd, options)
    if options.ensure_healthy:
        _debug("monitor: ensure_healthy is set, waiting for health")
        resp = send_command(proc_name, CMD_WAIT)
        if resp.is_running() and not resp.is_healthy():
            _debug("monitor: ensure_healthy is set and process is still not healthy (%s); stopping" % resp)
            resp = send_command(proc_name, CMD_STOP)
        elif not resp.is_running():
            _debug("monitor: ensure_healthy is set but process is not running (%s)" % resp)
        else:
            _debug("monitor: ensure_healthy is set and process is still healthy (%s)" % resp)
    else:
        resp = send_command(proc_name, CMD_STATUS)
    return resp


def start(proc_name, shell_cmd, options, strict=False):
    '''Start a process, assigning it the given id. If strict is true, return existing process status;
     otherwise raise exception if process already is running.'''

    lock_path = LOCK_PATH_PAT % proc_name
    lock_acquired = file_lock_acquire(lock_path)
    _debug("client: acquired lock: %s: %s" % (lock_acquired, lock_path))

    if lock_acquired:
        resp = _start_monitor(proc_name, shell_cmd, options)
    else:
        if strict:
            raise AlreadyRunningException("process already running (to override, remove lock at %s)" % lock_path)
        else:
            try:
                resp = send_command(proc_name, CMD_STATUS)
            except MonitorNotRunning as e:
                _debug("client: stale lock file present but monitor not listening; restarting new monitor")
                resp = _start_monitor(proc_name, shell_cmd, options)
    return resp


def status(proc_name):
    '''Return status of the process with the given id.'''
    try:
        return send_command(proc_name, CMD_STATUS)
    except NotRunningException:
        return Response(code=RESP_STOPPED)


def wait(proc_name):
    '''Same as status, but if process is running, try to wait until it is healthy.'''
    try:
        return send_command(proc_name, CMD_WAIT)
    except NotRunningException:
        return Response(code=RESP_STOPPED)


def stop(proc_name, strict=False):
    '''Stop the command with the given id. If strict is true, raise exception if process already is stopped.'''
    try:
        return send_command(proc_name, CMD_STOP)
    except NotRunningException as e:
        if strict:
            raise
        else:
            return Response(code=RESP_STOPPED)


def _locate_config_file():
    '''Look in common locations for config file.'''
    base_dir = os.path.dirname(sys.argv[0])
    paths = [os.path.join(base_dir, CONFIG_FILE), os.path.join(os.getenv("HOME"), CONFIG_DOTFILE)]
    for path in paths:
        if os.path.isfile(path):
            _debug("using config file: %s" % path)
            return path
    return None


def _parse_config(config, section, option, target_type):
    if target_type == str:
        return config.get(section, option)
    elif target_type == bool:
        return config.getboolean(section, option)
    elif target_type == int:
        return config.getint(section, option)
    elif target_type == float:
        return config.getfloat(section, option)


def _read_config_file(proc_name, config_file, target_types):
    out = {}
    try:
        if config_file:
            config = ConfigParser.RawConfigParser()
            config.read(config_file)
            for (key, str_value) in config.items(proc_name):
                if key not in Options._fields:
                    raise ValueError("unrecognized config key: '%s' in '%s'" % (key, config_file))
                out[key] = _parse_config(config, proc_name, key, target_types[key])
    except ConfigParser.NoSectionError:
        pass
    return out


def _expand_variables(template_str):
    '''Expand a string like "$HOME/foo" using environment variables.'''
    if template_str is None:
        return None
    else:
        try:
            return Template(template_str).substitute(os.environ)
        except Exception as e:
            raise ValueError("could not expand environment variable names in command '%s': %s" % (template_str, e))


def _execute(proc_name, control_cmd, options):
    if control_cmd == CMD_START:
        response = start(proc_name, options.command, options=options, strict=options.strict)
    elif control_cmd == CMD_STOP:
        response = stop(proc_name, strict=options.strict)
    elif control_cmd == CMD_STATUS:
        response = status(proc_name)
    elif control_cmd == CMD_WAIT:
        response = wait(proc_name)
    return response

# These are the options configurable by file or command line.
Options = namedtuple(
    "Options",
    "command health_command health_count health_delay ensure_healthy dir stdin stdout stderr append linger strict shell")

OPTION_DEFAULTS = Options(command=None,
                          health_command=None,
                          health_count=5,
                          health_delay=1.,
                          ensure_healthy=False,
                          dir=None,
                          stdin=None,
                          stdout=None,
                          stderr=None,
                          append=False,
                          linger=3.,
                          strict=False,
                          shell=False)


def main():
    parser = argparse.ArgumentParser(description=DESCRIPTION, version=VERSION, epilog="\n" + __doc__)
    parser.add_argument("control_cmd", help="control command", choices=CMD_LIST)
    parser.add_argument("proc_name", help="unique alphanumeric name for this process")
    parser.add_argument("-c", "--command",
                        help="with start: full command to start process, with all arguments, in shell syntax")
    parser.add_argument("--health-command", help="with start: full command to perform health check")
    parser.add_argument("--health-count", help="with start: max tries for health check", type=int)
    parser.add_argument("--health-delay", help="with start: delays for health check", type=float)
    parser.add_argument("--ensure-healthy",
                        help="with start: wait for health, then shut down if not healhty",
                        action="store_true")
    parser.add_argument("--dir", help="with start: alternate directory to start from")
    parser.add_argument("--stdin", help="with start: filename for standard input (defaults to /dev/null)")
    parser.add_argument("--stdout", help="with start: filename for standard input")
    parser.add_argument("--stderr", help="with start: filename for standard error (may be same as --stdout)")
    parser.add_argument("--append",
                        help="with start: append to files specified in --stdout and --stderr",
                        action="store_true")
    parser.add_argument("--linger", help="with start: seconds for monitor to linger, preserving exit code", type=float)
    parser.add_argument("--strict",
                        help="abort with error if asked to start a started process or stop a stopped one",
                        action="store_true")
    parser.add_argument("--shell", help="use shell to fully parse/execute the command string", action="store_true")
    parser.add_argument("--config", help="non-default location on config file to read")
    parser.add_argument("--debug", help="enable debugging output", action="store_true")
    args = parser.parse_args()

    global DEBUG
    DEBUG = args.debug

    try:
        config_file = args.config if args.config else _locate_config_file()

        # XXX Hack to reuse argparse's internal actions list, to avoid repeating ourselves when parsing out data types from the config file.
        actions = {action.dest: action for action in parser._actions if action.dest in Options._fields}
        target_types = {action.dest: (action.type or type(action.const or "")) for action in actions.values()}

        file_options = _read_config_file(args.proc_name, config_file, target_types)

        def was_explicitly_set(action, val):
            return (type(action) != argparse._StoreTrueAction and type(action) != argparse._StoreFalseAction
                    ) or val != action.default

        # These are only the options that were explicitly set.
        command_line_options = {
            key: val
            for (key, val) in vars(args).items()
            if key in Options._fields and val is not None and was_explicitly_set(actions[key], val)
        }

        # Precedence on options is command line, then file, then defaults.
        default_options = vars(OPTION_DEFAULTS)

        combined = default_options.copy()
        combined.update(file_options)
        combined.update(command_line_options)
        options = Options(**combined)

        if config_file:
            _debug("config file options from file %s: %s" % (config_file, file_options))
        _debug("command-line options: %s" % command_line_options)
        _debug("combined options: %s" % (options, ))

        # Final preparation of options.
        # Expand environment variables now, while they are available.
        # TODO: Could and no-expand option to disable variable expansion (perhaps useful with --shell).
        options = options._replace(command=_expand_variables(options.command),
                                   health_command=_expand_variables(options.health_command),
                                   stdout=_expand_variables(options.stdout),
                                   stderr=_expand_variables(options.stderr),
                                   dir=_expand_variables(options.dir) or os.getcwd())

        if not re.compile("^[\\w-]+$").match(args.proc_name):
            parser.error("Invalid process name '%s'; must be alphanumeric, with only underscores or dashes" %
                         args.proc_name)
        if args.control_cmd == "start" and not options.command:
            parser.error(
                "Must specify --command with 'start' or supply a command in the config file for this id ('%s')" %
                args.proc_name)
        if not os.path.isdir(options.dir):
            parser.error("Not a directory: %s" % options.dir)

        response = _execute(args.proc_name, args.control_cmd, options)
        print(response.encode())

        exit_code = response.exit_code(args.control_cmd, strict=options.strict, ensure_healthy=options.ensure_healthy)
        _debug("client: exit code %s (based on '%s' with strict=%s and ensure_healthy=%s)" %
               (exit_code, response, options.strict, options.ensure_healthy))
        sys.exit(exit_code)

    except AlreadyRunningException as e:
        _die("process '%s' is already running" % args.proc_name, code=EXIT_ALREADY_RUNNING, exceptional=True)
    except NotRunningException as e:
        _die("process '%s' is not running" % args.proc_name, code=EXIT_NOT_RUNNING, exceptional=True)
    except (ValueError, OSError) as e:
        _die(str(e), code=EXIT_UNKNOWN, exceptional=True)


if __name__ == '__main__':
    main()

# TODO:
# - better logic around where procdog.cfg can be when procdog is installed globally via pip
# - handle corner case scenario:
#     process already started externally without procdog, so process started with procdog fails,
#     but health check passes immediately since process is listening
# - handle bad stdout/stderr output file/directory gracefully
# - in non-strict mode expose via a second variable whether process was already started/stopped
# - remove wait, or replace wait command with clearer-named alternative (wait for start vs wait for health)
# - add restart command
# - service support
#     add a symlink target for /etc/init.d to auto-convert to a service script (like /lib/init/upstart-job)
# - add config loading from /etc/procdog.conf and/or /etc/procdog.d/*
# - pipe log output to process (like logrotate) if target starts with "|"
# - auto restart n times within k seconds on process dying
# - figure out/clarify commands for wait-for-health vs wait-for-exit
# - on Linux, exit code doesn't show kill signal (-15): stop gives 'exited, returncode=0' vs 'exited, returncode=-15'
# - kill -15 followed by kill -9 if no response; custom kill signals
# - list command to show status of all running processes
# - customizable shutdown command (e.g. another script to execute that does something besides just kill)
# - restart (kill + start) command
# - retry policies to rerun if process dies
# - tests for more corner cases such as shlex failures and config file errors
# - more testing on Mac+Linux
# - keyboard interrupt handler with best effort to kill process if user does control-C while health check is waiting?