Add reload timeout for old process

Dockerfile

@@ -21,7 +21,7 @@ ENV RSYSLOG_DESTINATION=127.0.0.1 \
     STATS_PORT=1936 \
     STATS_AUTH="stats:stats" \
     SSL_BIND_OPTIONS=no-sslv3 \
-    SSL_BIND_CIPHERS="ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:DHE-DSS-AES128-SHA:DES-CBC3-SHA" \
+    SSL_BIND_CIPHERS="ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:DHE-DSS-AES128-SHA:DES-CBC3-SHA" \
     HEALTH_CHECK="check inter 2000 rise 2 fall 3" \
     NBPROC=1
 

Dockerfile-dev

@@ -0,0 +1,44 @@
+FROM alpine:3.4
+
+RUN echo PyYAML==3.11 > requirements.txt && \
+    echo cached-property==1.2.0 >> requirements.txt && \
+    echo docker-py==1.10.3 >> requirements.txt && \
+    echo dockerpty==0.4.1 >> requirements.txt && \
+    echo docopt==0.6.1 >> requirements.txt && \
+    venum34==1.0.4 >> requirements.txt && \
+    echo jsonschema==2.5.1 >> requirements.txt && \
+    vtexttable==0.8.4 >> requirements.txt && \
+    echo future==0.15.0 >> requirements.txt && \
+    echo requests==2.7.0 >> requirements.txt && \
+    echo six==1.9.0 >> requirements.txt && \
+    echo websocket-client==0.37.0 >> requirements.txt && \
+    echo docker-compose==1.6.0 >> requirements.txt && \
+    echo python-dockercloud==1.0.5 >> requirements.txt && \
+    echo gevent==1.1.1 >> requirements.txt
+
+RUN apk update && \
+    apk --no-cache add tini haproxy py-pip build-base python-dev ca-certificates && \
+    pip install -r requirements.txt
+
+ENV RSYSLOG_DESTINATION=127.0.0.1 \
+    MODE=http \
+    BALANCE=roundrobin \
+    MAXCONN=4096 \
+    OPTION="redispatch, httplog, dontlognull, forwardfor" \
+    TIMEOUT="connect 5000, client 50000, server 50000" \
+    STATS_PORT=1936 \
+    STATS_AUTH="stats:stats" \
+    SSL_BIND_OPTIONS=no-sslv3 \
+    SSL_BIND_CIPHERS="ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES128-GCM-SHA256:AES128-SHA256:AES128-SHA:AES256-GCM-SHA384:AES256-SHA256:AES256-SHA:DHE-DSS-AES128-SHA:DES-CBC3-SHA" \
+    HEALTH_CHECK="check inter 2000 rise 2 fall 3" \
+    NBPROC=1
+
+COPY . /haproxy-src
+RUN cp /haproxy-src/reload.sh /reload.sh && \
+    cd /haproxy-src && \
+    pip install .
+
+
+EXPOSE 80 443 1936
+ENTRYPOINT ["/sbin/tini", "--"]
+CMD ["dockercloud-haproxy"]

README.md

@@ -156,8 +156,8 @@ Similar to using legacy links, here list some differences that you need to notic
 Once the stack is up, you can scale the web service using `docker-compose scale web=3`. dockercloud/haproxy will automatically reload its configuration.
 
 #### Running with Docker Compose v2 and Swarm (using envvar)
-When using links like previous section, the Docker Swarm scheduler can be too restrictive. 
-Even with overlay network, swarm (As of 1.1.0) will attempt to schedule haproxy on the same node as the linked service due to legacy links behavior. 
+When using links like previous section, the Docker Swarm scheduler can be too restrictive.
+Even with overlay network, swarm (As of 1.1.0) will attempt to schedule haproxy on the same node as the linked service due to legacy links behavior.
 This can cause unwanted scheduling patterns or errors such as "Unable to find a node fulfilling all dependencies..."
 
 Since Compose V2 allows discovery through the service names, Dockercloud haproxy only needs the links to indentify which service should be load balanced.
@@ -214,11 +214,13 @@ Settings in this part is immutable, you have to redeploy HAProxy service to make
 |FORCE_DEFAULT_BACKEND| True | set the default_service as a default backend. This is useful when you have more than one backend and you don't want your default_service as a default backend    
 |HEALTH_CHECK|check|set health check on each backend route, possible value: "check inter 2000 rise 2 fall 3". See:[HAProxy:check](https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#5.2-check)|
 |HTTP_BASIC_AUTH| |a comma-separated list of credentials(`<user>:<pass>`) for HTTP basic auth, which applies to all the backend routes. To escape comma, use `\,`. *Attention:* DO NOT rely on this for authentication in production|
+|HTTP_BASIC_AUTH_SECURE| |a comma-separated list of credentials(`<user>:<encrypted-pass>`) for HTTP basic auth, which applies to all the backend routes. To escape comma, use `\,`. See:[HAProxy:user](https://cbonte.github.io/haproxy-dconv/1.5/configuration.html#3.4-user) *Attention:* DO NOT rely on this for authentication in production|
 |MAXCONN|4096|sets the maximum per-process number of concurrent connections.|
 |MODE|http|mode of load balancing for HAProxy. Possible values include: `http`, `tcp`, `health`|
 |MONITOR_PORT| |the port number where monitor_uri should be added to. Use together with `MONTIOR_URI`. Possible value: `80`|
 |MONITOR_URI| |the exact URI which we want to intercept to return HAProxy's health status instead of forwarding the request.See: http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-monitor-uri. Possible value: `/ping`|
 |OPTION|redispatch|comma-separated list of HAProxy `option` entries to the `default` section.|
+|RELOAD_TIMEOUT|0| When haproxy is reconfigured, a new process starts and attaches to the TCP socket for new connections, leaving the old process to handle existing connections.  This timeout specifies how long the old process is permitted to continue running before being killed. <br/>  `-1`: Old process is killed immediately<br/>  `0`: No timeout, old process will run as long as TCP connections last.  This could potentially be quite a while as `http-keep-alives` are enabled which will keep TCP connections open.<br/>  `>0`: Timeout in secs after which the process will be killed.
 |RSYSLOG_DESTINATION|127.0.0.1|the rsyslog destination to where HAProxy logs are sent|
 |SKIP_FORWARDED_PROTO||If set to any value, HAProxy will not add an X-Forwarded- headers. This can be used when combining HAProxy with another load balancer|
 |SSL_BIND_CIPHERS| |explicitly set which SSL ciphers will be used for the SSL server. This sets the HAProxy `ssl-default-bind-ciphers` configuration setting.|
@@ -238,9 +240,11 @@ Settings here can overwrite the settings in HAProxy, which are only applied to t
 |BALANCE|load balancing algorithm to use. Possible values include: `roundrobin`, `static-rr`, `source`, `leastconn`. See:[HAProxy:balance](https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-balance)|
 |COOKIE|sticky session option. Possible value `SRV insert indirect nocache`. See:[HAProxy:cookie](http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-cookie)|
 |DEFAULT_SSL_CERT|similar to SSL_CERT, but stores the pem file at `/certs/cert0.pem` as the default ssl certs. If multiple `DEFAULT_SSL_CERT` are specified in linked services and HAProxy, the behavior is undefined|
-|EXCLUDE_PORTS|comma separated port numbers(e.g. 3306, 3307). By default, HAProxy will add all the ports exposed by the application services to the backend routes. You can exclude the ports that you don't want to be routed, like database port|
+|EXCLUDE_BASIC_AUTH|if set, the application by the application services to the backend routes. You can exclude the ports that you don't want to be routed, like database port|
+|EXCLUDE_PORTS|if set(any value) and `HTTP_BASIC_AUTH` global setting is set, no basic auth will be applied to this service.|
 |EXTRA_ROUTE_SETTINGS|a string which is append to the each backend route after the health check,possible value: "send-proxy"|
 |EXTRA_SETTINGS|comma-separated string of extra settings, and each part will be appended to either related backend section or listen session in the configuration file. To escape comma, use `\,`. Possible value: `balance source`|
+|FAILOVER|if set(any value), it configures this service to be run as HAProxy `backup` for other configured service(s) in this backend|
 |FORCE_SSL|if set(any value) together with ssl termination enabled. HAProxy will redirect HTTP request to HTTPS request.
 |GZIP_COMPRESSION_TYPE|enable gzip compression. The value of this envvar is a list of MIME types that will be compressed. Some possible values: `text/html text/plain text/css application/javascript`. See:[HAProxy:compression](http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#4-compression)|
 |HEALTH_CHECK|set health check on each backend route, possible value: "check inter 2000 rise 2 fall 3". See:[HAProxy:check](https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#5.2-check)|
@@ -337,7 +341,7 @@ To set SSL certificate, you can either:
 1. set `DEFAULT_SSL_CERT` in `dockercloud/haproxy`, or
 2. set `SSL_CERT` and/or `DEFAULT_SSL_CERT` in the application services linked to HAProxy
 
-The difference between `SSL_CERT` and `DEFAULT_SSL_CERT` is that, the multiple certificates specified by `SSL_CERT` are stored in as cert1.pem, cert2.pem, ..., whereas the one specified by `DEFAULT_SSL_CERT` is always stored as cert0.pem. In that case, HAProxy will use cert0.pem as the default certificate when there is no SNI match. However, when multiple `DEFAULT_SSL_CERTIFICATE` is provided, only one of the certificates can be stored as cert0.pem, others are discarded.
+The difference between `SSL_CERT` and `DEFAULT_SSL_CERT` is that, the multiple certificates specified by `SSL_CERT` are stored in as cert1.pem, cert2.pem, ..., whereas the one specified by `DEFAULT_SSL_CERT` is always stored as cert0.pem. In that case, HAProxy will use cert0.pem as the default certificate when there is no SNI match. However, when multiple `DEFAULT_SSL_CERT` is provided, only one of the certificates can be stored as cert0.pem, others are discarded.
 
 #### PEM Files
 The certificate specified in `dockercloud/haproxy` or in the linked application services is a pem file, containing a private key followed by a public certificate(private key must be put before the public certificate and any extra Authority certificates, order matters). You can run the following script to generate a self-signed certificate:

haproxy/__init__.py

@@ -1 +1 @@
-__version__ = "1.6.2"
+__version__ = "1.6.3"

haproxy/config.py

@@ -70,6 +70,7 @@ def parse_additional_backend_settings(envvars):
 HAPROXY_SERVICE_URI = os.getenv("DOCKERCLOUD_SERVICE_API_URI")
 HEALTH_CHECK = os.getenv("HEALTH_CHECK", "check inter 2000 rise 2 fall 3")
 HTTP_BASIC_AUTH = os.getenv("HTTP_BASIC_AUTH")
+HTTP_BASIC_AUTH_SECURE = os.getenv("HTTP_BASIC_AUTH_SECURE")
 MAXCONN = os.getenv("MAXCONN", "4096")
 MODE = os.getenv("MODE", "http")
 MONITOR_PORT = os.getenv("MONITOR_PORT")
@@ -84,6 +85,7 @@ def parse_additional_backend_settings(envvars):
 TIMEOUT = os.getenv("TIMEOUT", "connect 5000, client 50000, server 50000")
 NBPROC = int(os.getenv("NBPROC", 1))
 SWARM_MODE_POLLING_INTERVAL = int(os.getenv("SWARM_MODE_POLLING_INTERVAL", 5))
+RELOAD_TIMEOUT = os.getenv("RELOAD_TIMEOUT", "0")
 
 # global
 RUNNING_MODE = None

haproxy/haproxycfg.py

@@ -163,7 +163,7 @@ def update(self):
             cfg_dict.update(self._config_global_section())
             cfg_dict.update(self._config_defaults_section())
             cfg_dict.update(self._config_stats_section())
-            cfg_dict.update(self._config_userlist_section(HTTP_BASIC_AUTH))
+            cfg_dict.update(self._config_userlist_section(HTTP_BASIC_AUTH, HTTP_BASIC_AUTH_SECURE))
             cfg_dict.update(self._config_tcp_sections())
             cfg_dict.update(self._config_frontend_sections())
             cfg_dict.update(self._config_backend_sections())
@@ -290,21 +290,27 @@ def _config_defaults_section():
         return cfg
 
     @staticmethod
-    def _config_userlist_section(basic_auth):
-        cfg = OrderedDict()
-        if basic_auth:
-            auth_list = re.split(r'(?<!\\),', basic_auth)
+    def _parse_userlist(auth_section, type):
+        userlist = []
+        if auth_section:
+            auth_list = re.split(r'(?<!\\),', auth_section)
             userlist = []
             for auth in auth_list:
                 if auth.strip():
                     terms = auth.strip().split(":", 1)
                     if len(terms) == 2:
                         username = terms[0].replace("\,", ",")
                         password = terms[1].replace("\,", ",")
-                        userlist.append("user %s insecure-password %s" % (username, password))
+                        userlist.append("user %s %s %s" % (username, type, password))
+        return userlist
 
-            if userlist:
-                cfg["userlist haproxy_userlist"] = userlist
+    @staticmethod
+    def _config_userlist_section(basic_auth, basic_auth_secure):
+        cfg = OrderedDict()
+        userlist = Haproxy._parse_userlist(basic_auth, "insecure-password") + \
+                   Haproxy._parse_userlist(basic_auth_secure, "password")
+        if userlist:
+            cfg["userlist haproxy_userlist"] = userlist
         return cfg
 
     def _config_tcp_sections(self):

haproxy/helper/backend_helper.py

@@ -16,13 +16,13 @@ def get_backend_section(details, routes, vhosts, service_alias, routes_added):
     route_health_check = get_route_health_check(details, service_alias, HEALTH_CHECK)
     extra_route_settings = get_extra_route_settings(details, service_alias, EXTRA_ROUTE_SETTINGS)
     route_setting = " ".join([route_health_check, extra_route_settings]).strip()
-    backend_routes = get_backend_routes(route_setting, is_sticky, routes, routes_added, service_alias)
+    backend_routes = get_backend_routes(route_setting, is_sticky, routes, routes_added, service_alias, details)
     backend.extend(backend_routes)
 
     return backend
 
 
-def get_backend_routes(route_setting, is_sticky, routes, routes_added, service_alias):
+def get_backend_routes(route_setting, is_sticky, routes, routes_added, service_alias, details):
     backend_routes = []
     for _service_alias, routes in routes.iteritems():
         if not service_alias or _service_alias == service_alias:
@@ -41,6 +41,9 @@ def get_backend_routes(route_setting, is_sticky, routes, routes_added, service_a
                     if route_setting:
                         backend_route.append(route_setting)
 
+                    if details.get(service_alias, {}).get('failover', False):
+                        backend_route.append("backup")
+
                     backend_routes.append(" ".join(backend_route))
 
     return sorted(backend_routes)
@@ -80,7 +83,7 @@ def get_backend_settings(details, service_alias, basic_auth):
     backend_settings.extend(get_hsts_max_age_setting(details, service_alias))
     backend_settings.extend(get_options_setting(details, service_alias))
     backend_settings.extend(get_extra_settings_setting(details, service_alias))
-    backend_settings.extend(get_basic_auth_setting(basic_auth))
+    backend_settings.extend(get_basic_auth_setting(details, basic_auth, service_alias))
 
     return backend_settings, is_sticky
 
@@ -163,9 +166,11 @@ def get_extra_settings_setting(details, service_alias):
     return setting
 
 
-def get_basic_auth_setting(basic_auth):
+def get_basic_auth_setting(details, basic_auth, service_alias):
     setting = []
-    if basic_auth:
+    exclude_basic_auth = get_service_attribute(details, "exclude_basic_auth", service_alias)
+
+    if basic_auth and not exclude_basic_auth:
         setting.append("acl need_auth http_auth(haproxy_userlist)")
         setting.append("http-request auth realm haproxy_basic_auth if !need_auth")
     return setting

haproxy/helper/swarm_mode_link_helper.py

@@ -18,8 +18,9 @@ def get_swarm_mode_haproxy_id_nets(docker, haproxy_container_short_id):
         logger.info("Dockercloud haproxy is not running in a service in SwarmMode")
         return "", set()
 
-    haproxy_nets = set([network.get("NetworkID", "") for network in
-                        haproxy_container.get("NetworkSettings", {}).get("Networks", {}).values()])
+    haproxy_nets = set([network.get("NetworkID", "") for name, network in
+                        haproxy_container.get("NetworkSettings", {}).get("Networks", {}).iteritems()
+                        if name != "ingress"])
 
     return haproxy_service_id, haproxy_nets
 

haproxy/helper/update_helper.py

@@ -1,19 +1,47 @@
 import logging
 import subprocess
 import thread
+import time
+import threading
 
-from haproxy.config import HAPROXY_RUN_COMMAND
+from haproxy.config import HAPROXY_RUN_COMMAND, RELOAD_TIMEOUT
 
 logger = logging.getLogger("haproxy")
 
+# RELOAD_TIMEOUT has the following values and effect:
+# -1 : Reload haproxy with "-st" which will immediately kill the previous process
+# 0 : Reload haproxy with "-sf" and no timeout.  This can potentially leave 
+#     "broken" processes (where the backends have changed) hanging around 
+#     with existing connections.
+# > 0 : Reload haproxy with "-sf" but if it takes longer than RELOAD_TIMEOUT then kill it
+#       This gives existing connections a chance to finish.  RELOAD_TIMEOUT should be set to 
+#       the approximate time it takes docker to finish updating services.  By this point the
+#       existing configuration will be invalid, and any connections still using it will 
+#       have invalid backends.
+#
+def run_reload(old_process, timeout = int(RELOAD_TIMEOUT)):
 
-def run_reload(old_process):
     if old_process:
         # Reload haproxy
         logger.info("Reloading HAProxy")
-        new_process = subprocess.Popen(HAPROXY_RUN_COMMAND + ["-sf", str(old_process.pid)])
-        thread.start_new_thread(wait_pid, (old_process,))
-        logger.info("HAProxy has been reloaded(PID: %s)", str(new_process.pid))
+        if timeout == -1:
+          flag = "-st"
+          logger.info("Restarting HAProxy immediately")
+        else:
+          flag = "-sf"
+          logger.info("Restarting HAProxy gracefully")
+
+        new_process = subprocess.Popen(HAPROXY_RUN_COMMAND + [flag, str(old_process.pid)])
+        logger.info("HAProxy is reloading (new PID: %s)", str(new_process.pid))
+
+        thread = threading.Thread(target = wait_pid, args=[old_process,timeout])
+        thread.start()
+
+        # Block only if we have a timeout.  If we don't it could take forever, and so
+        # returning immediately maintains the original behaviour of no timeout.
+        if timeout > 0:
+          thread.join()
+
     else:
         # Launch haproxy
         logger.info("Launching HAProxy")
@@ -23,6 +51,30 @@ def run_reload(old_process):
     return new_process
 
 
-def wait_pid(process):
+def wait_pid(process, timeout):
+    start = time.time()
+
+    timer = None
+
+    if timeout > 0:
+      timer = threading.Timer(timeout, timeout_handler, [process] )
+      timer.start()
+
     process.wait()
-    logger.info("HAProxy(PID:%s) has been terminated" % str(process.pid))
+
+    if timer is not None:
+      timer.cancel();
+
+    duration = time.time() - start
+    logger.info("Old HAProxy(PID: %s) ended after %s sec", str(process.pid), str(duration))
+
+
+def timeout_handler( processs ):
+  if processs.poll() is None:
+    try:
+      processs.terminate()
+      logger.info("Old HAProxy process taking too long to complete - terminating")
+    except OSError as e:
+      if e.errno != errno.ESRCH:
+        raise
+

haproxy/parser/base_parser.py

@@ -188,3 +188,11 @@ def parse_extra_settings(value):
     @staticmethod
     def parse_extra_route_settings(value):
         return value
+
+    @staticmethod
+    def parse_failover(value):
+        return value
+
+    @staticmethod
+    def parse_exclude_basic_auth(value):
+        return value