Live Nginx (re)configuration without reloading (kubernetes#2174)

Makefile

@@ -50,7 +50,7 @@ IMAGE = $(REGISTRY)/$(IMGNAME)
 MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 
 # Set default base image dynamically for each arch
-BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.34
+BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.37
 
 ifeq ($(ARCH),arm)
 	QEMUARCH=arm

cmd/nginx/flags.go

@@ -131,6 +131,9 @@ func parseFlags() (bool, *controller.Configuration, error) {
 		publishStatusAddress = flags.String("publish-status-address", "",
 			`User customized address to be set in the status of ingress resources. The controller will set the
 		endpoint records on the ingress using this address.`)
+
+		dynamicConfigurationEnabled = flags.Bool("enable-dynamic-configuration", false,
+			`When enabled controller will try to avoid Nginx reloads as much as possible by using Lua. Disabled by default.`)
 	)
 
 	flag.Set("logtostderr", "true")
@@ -192,28 +195,29 @@ func parseFlags() (bool, *controller.Configuration, error) {
 	}
 
 	config := &controller.Configuration{
-		APIServerHost:            *apiserverHost,
-		KubeConfigFile:           *kubeConfigFile,
-		UpdateStatus:             *updateStatus,
-		ElectionID:               *electionID,
-		EnableProfiling:          *profiling,
-		EnableSSLPassthrough:     *enableSSLPassthrough,
-		EnableSSLChainCompletion: *enableSSLChainCompletion,
-		ResyncPeriod:             *resyncPeriod,
-		DefaultService:           *defaultSvc,
-		Namespace:                *watchNamespace,
-		ConfigMapName:            *configMap,
-		TCPConfigMapName:         *tcpConfigMapName,
-		UDPConfigMapName:         *udpConfigMapName,
-		DefaultSSLCertificate:    *defSSLCertificate,
-		DefaultHealthzURL:        *defHealthzURL,
-		PublishService:           *publishSvc,
-		PublishStatusAddress:     *publishStatusAddress,
-		ForceNamespaceIsolation:  *forceIsolation,
-		UpdateStatusOnShutdown:   *updateStatusOnShutdown,
-		SortBackends:             *sortBackends,
-		UseNodeInternalIP:        *useNodeInternalIP,
-		SyncRateLimit:            *syncRateLimit,
+		APIServerHost:               *apiserverHost,
+		KubeConfigFile:              *kubeConfigFile,
+		UpdateStatus:                *updateStatus,
+		ElectionID:                  *electionID,
+		EnableProfiling:             *profiling,
+		EnableSSLPassthrough:        *enableSSLPassthrough,
+		EnableSSLChainCompletion:    *enableSSLChainCompletion,
+		ResyncPeriod:                *resyncPeriod,
+		DefaultService:              *defaultSvc,
+		Namespace:                   *watchNamespace,
+		ConfigMapName:               *configMap,
+		TCPConfigMapName:            *tcpConfigMapName,
+		UDPConfigMapName:            *udpConfigMapName,
+		DefaultSSLCertificate:       *defSSLCertificate,
+		DefaultHealthzURL:           *defHealthzURL,
+		PublishService:              *publishSvc,
+		PublishStatusAddress:        *publishStatusAddress,
+		ForceNamespaceIsolation:     *forceIsolation,
+		UpdateStatusOnShutdown:      *updateStatusOnShutdown,
+		SortBackends:                *sortBackends,
+		UseNodeInternalIP:           *useNodeInternalIP,
+		SyncRateLimit:               *syncRateLimit,
+		DynamicConfigurationEnabled: *dynamicConfigurationEnabled,
 		ListenPorts: &ngx_config.ListenPorts{
 			Default:  *defServerPort,
 			Health:   *healthzPort,

internal/ingress/controller/config/config.go

@@ -604,23 +604,24 @@ func (cfg Configuration) BuildLogFormatUpstream() string {
 
 // TemplateConfig contains the nginx configuration to render the file nginx.conf
 type TemplateConfig struct {
-	ProxySetHeaders         map[string]string
-	AddHeaders              map[string]string
-	MaxOpenFiles            int
-	BacklogSize             int
-	Backends                []*ingress.Backend
-	PassthroughBackends     []*ingress.SSLPassthroughBackend
-	Servers                 []*ingress.Server
-	TCPBackends             []ingress.L4Service
-	UDPBackends             []ingress.L4Service
-	HealthzURI              string
-	CustomErrors            bool
-	Cfg                     Configuration
-	IsIPV6Enabled           bool
-	IsSSLPassthroughEnabled bool
-	RedirectServers         map[string]string
-	ListenPorts             *ListenPorts
-	PublishService          *apiv1.Service
+	ProxySetHeaders             map[string]string
+	AddHeaders                  map[string]string
+	MaxOpenFiles                int
+	BacklogSize                 int
+	Backends                    []*ingress.Backend
+	PassthroughBackends         []*ingress.SSLPassthroughBackend
+	Servers                     []*ingress.Server
+	TCPBackends                 []ingress.L4Service
+	UDPBackends                 []ingress.L4Service
+	HealthzURI                  string
+	CustomErrors                bool
+	Cfg                         Configuration
+	IsIPV6Enabled               bool
+	IsSSLPassthroughEnabled     bool
+	RedirectServers             map[string]string
+	ListenPorts                 *ListenPorts
+	PublishService              *apiv1.Service
+	DynamicConfigurationEnabled bool
 }
 
 // ListenPorts describe the ports required to run the

internal/ingress/controller/controller.go

@@ -95,6 +95,8 @@ type Configuration struct {
 	FakeCertificateSHA  string
 
 	SyncRateLimit float32
+
+	DynamicConfigurationEnabled bool
 }
 
 // GetPublishService returns the configured service used to set ingress status
@@ -167,6 +169,15 @@ func (n *NGINXController) syncIngress(item interface{}) error {
 	if !n.isForceReload() && n.runningConfig.Equal(&pcfg) {
 		glog.V(3).Infof("skipping backend reload (no changes detected)")
 		return nil
+	} else if !n.isForceReload() && n.cfg.DynamicConfigurationEnabled && n.IsDynamicallyConfigurable(&pcfg) {
+		err := n.ConfigureDynamically(&pcfg)
+		if err == nil {
+			glog.Infof("dynamic reconfiguration succeeded, skipping reload")
+			n.runningConfig = &pcfg
+			return nil
+		}
+
+		glog.Warningf("falling back to reload, could not dynamically reconfigure: %v", err)
 	}
 
 	glog.Infof("backend reload required")
@@ -182,6 +193,19 @@ func (n *NGINXController) syncIngress(item interface{}) error {
 	incReloadCount()
 	setSSLExpireTime(servers)
 
+	if n.isForceReload() && n.cfg.DynamicConfigurationEnabled {
+		go func() {
+			// it takes time for Nginx to start listening on the port
+			time.Sleep(1 * time.Second)
+			err := n.ConfigureDynamically(&pcfg)
+			if err == nil {
+				glog.Infof("dynamic reconfiguration succeeded")
+			} else {
+				glog.Warningf("could not dynamically reconfigure: %v", err)
+			}
+		}()
+	}
+
 	n.runningConfig = &pcfg
 	n.SetForceReload(false)
 

internal/ingress/controller/nginx.go

@@ -18,10 +18,12 @@ package controller
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"strconv"
@@ -606,23 +608,24 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
 	cfg.SSLDHParam = sslDHParam
 
 	tc := ngx_config.TemplateConfig{
-		ProxySetHeaders:         setHeaders,
-		AddHeaders:              addHeaders,
-		MaxOpenFiles:            maxOpenFiles,
-		BacklogSize:             sysctlSomaxconn(),
-		Backends:                ingressCfg.Backends,
-		PassthroughBackends:     ingressCfg.PassthroughBackends,
-		Servers:                 ingressCfg.Servers,
-		TCPBackends:             ingressCfg.TCPEndpoints,
-		UDPBackends:             ingressCfg.UDPEndpoints,
-		HealthzURI:              ngxHealthPath,
-		CustomErrors:            len(cfg.CustomHTTPErrors) > 0,
-		Cfg:                     cfg,
-		IsIPV6Enabled:           n.isIPV6Enabled && !cfg.DisableIpv6,
-		RedirectServers:         redirectServers,
-		IsSSLPassthroughEnabled: n.cfg.EnableSSLPassthrough,
-		ListenPorts:             n.cfg.ListenPorts,
-		PublishService:          n.GetPublishService(),
+		ProxySetHeaders:             setHeaders,
+		AddHeaders:                  addHeaders,
+		MaxOpenFiles:                maxOpenFiles,
+		BacklogSize:                 sysctlSomaxconn(),
+		Backends:                    ingressCfg.Backends,
+		PassthroughBackends:         ingressCfg.PassthroughBackends,
+		Servers:                     ingressCfg.Servers,
+		TCPBackends:                 ingressCfg.TCPEndpoints,
+		UDPBackends:                 ingressCfg.UDPEndpoints,
+		HealthzURI:                  ngxHealthPath,
+		CustomErrors:                len(cfg.CustomHTTPErrors) > 0,
+		Cfg:                         cfg,
+		IsIPV6Enabled:               n.isIPV6Enabled && !cfg.DisableIpv6,
+		RedirectServers:             redirectServers,
+		IsSSLPassthroughEnabled:     n.cfg.EnableSSLPassthrough,
+		ListenPorts:                 n.cfg.ListenPorts,
+		PublishService:              n.GetPublishService(),
+		DynamicConfigurationEnabled: n.cfg.DynamicConfigurationEnabled,
 	}
 
 	content, err := n.t.Write(tc)
@@ -745,3 +748,43 @@ func (n *NGINXController) setupSSLProxy() {
 		}
 	}()
 }
+
+// IsDynamicallyConfigurable decides if the new configuration can be dynamically configured without reloading
+func (n *NGINXController) IsDynamicallyConfigurable(pcfg *ingress.Configuration) bool {
+	var copyOfRunningConfig ingress.Configuration = *n.runningConfig
+	var copyOfPcfg ingress.Configuration = *pcfg
+
+	copyOfRunningConfig.Backends = []*ingress.Backend{}
+	copyOfPcfg.Backends = []*ingress.Backend{}
+
+	return copyOfRunningConfig.Equal(&copyOfPcfg)
+}
+
+// ConfigureDynamically JSON encodes new Backends and POSTs it to an internal HTTP endpoint
+// that is handled by Lua
+func (n *NGINXController) ConfigureDynamically(pcfg *ingress.Configuration) error {
+	buf, err := json.Marshal(pcfg.Backends)
+	if err != nil {
+		return err
+	}
+
+	glog.V(2).Infof("posting backends configuration: %s", buf)
+
+	url := fmt.Sprintf("http://localhost:%d/configuration/backends", n.cfg.ListenPorts.Status)
+	resp, err := http.Post(url, "application/json", bytes.NewReader(buf))
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			glog.Warningf("error while closing response body: \n%v", err)
+		}
+	}()
+
+	if resp.StatusCode != http.StatusCreated {
+		return fmt.Errorf("Unexpected error code: %d", resp.StatusCode)
+	}
+
+	return nil
+}

internal/ingress/controller/nginx_test.go

@@ -16,7 +16,78 @@ limitations under the License.
 
 package controller
 
-import "testing"
+import (
+	"testing"
+
+	"k8s.io/ingress-nginx/internal/ingress"
+)
+
+func TestIsDynamicallyConfigurable(t *testing.T) {
+	backends := []*ingress.Backend{{
+		Name: "fakenamespace-myapp-80",
+		Endpoints: []ingress.Endpoint{
+			{
+				Address: "10.0.0.1",
+				Port:    "8080",
+			},
+			{
+				Address: "10.0.0.2",
+				Port:    "8080",
+			},
+		},
+	}}
+
+	servers := []*ingress.Server{{
+		Hostname: "myapp.fake",
+		Locations: []*ingress.Location{
+			{
+				Path:    "/",
+				Backend: "fakenamespace-myapp-80",
+			},
+		},
+	}}
+
+	commonConfig := &ingress.Configuration{
+		Backends: backends,
+		Servers:  servers,
+	}
+
+	n := &NGINXController{
+		runningConfig: &ingress.Configuration{
+			Backends: backends,
+			Servers:  servers,
+		},
+	}
+
+	newConfig := commonConfig
+	if !n.IsDynamicallyConfigurable(newConfig) {
+		t.Errorf("When new config is same as the running config it should be deemed as dynamically configurable")
+	}
+
+	newConfig = &ingress.Configuration{
+		Backends: []*ingress.Backend{{Name: "another-backend-8081"}},
+		Servers:  []*ingress.Server{{Hostname: "myapp1.fake"}},
+	}
+	if n.IsDynamicallyConfigurable(newConfig) {
+		t.Errorf("Expected to not be dynamically configurable when there's more than just backends change")
+	}
+
+	newConfig = &ingress.Configuration{
+		Backends: []*ingress.Backend{{Name: "a-backend-8080"}},
+		Servers:  servers,
+	}
+	if !n.IsDynamicallyConfigurable(newConfig) {
+		t.Errorf("Expected to be dynamically configurable when only backends change")
+	}
+
+	if !n.runningConfig.Equal(commonConfig) {
+		t.Errorf("Expected running config to not change")
+	}
+
+	if !newConfig.Equal(&ingress.Configuration{Backends: []*ingress.Backend{{Name: "a-backend-8080"}}, Servers: servers}) {
+		t.Errorf("Expected new config to not change")
+	}
+}
 
 func TestNginxHashBucketSize(t *testing.T) {
 	tests := []struct {

internal/ingress/controller/template/template.go

@@ -307,7 +307,7 @@ func buildLoadBalancingConfig(b interface{}, fallbackLoadBalancing string) strin
 // (specified through the nginx.ingress.kubernetes.io/rewrite-to annotation)
 // If the annotation nginx.ingress.kubernetes.io/add-base-url:"true" is specified it will
 // add a base tag in the head of the response from the service
-func buildProxyPass(host string, b interface{}, loc interface{}) string {
+func buildProxyPass(host string, b interface{}, loc interface{}, dynamicConfigurationEnabled bool) string {
 	backends, ok := b.([]*ingress.Backend)
 	if !ok {
 		glog.Errorf("expected an '[]*ingress.Backend' type but %T was returned", b)
@@ -323,14 +323,19 @@ func buildProxyPass(host string, b interface{}, loc interface{}) string {
 	path := location.Path
 	proto := "http"
 
-	upstreamName := location.Backend
+	upstreamName := "upstream_balancer"
+
+	if !dynamicConfigurationEnabled {
+		upstreamName = location.Backend
+	}
+
 	for _, backend := range backends {
 		if backend.Name == location.Backend {
 			if backend.Secure || backend.SSLPassthrough {
 				proto = "https"
 			}
 
-			if isSticky(host, location, backend.SessionAffinity.CookieSessionAffinity.Locations) {
+			if !dynamicConfigurationEnabled && isSticky(host, location, backend.SessionAffinity.CookieSessionAffinity.Locations) {
 				upstreamName = fmt.Sprintf("sticky-%v", upstreamName)
 			}
 
@@ -340,6 +345,7 @@ func buildProxyPass(host string, b interface{}, loc interface{}) string {
 
 	// defProxyPass returns the default proxy_pass, just the name of the upstream
 	defProxyPass := fmt.Sprintf("proxy_pass %s://%s;", proto, upstreamName)
+
 	// if the path in the ingress rule is equals to the target: no special rewrite
 	if path == location.Rewrite.Target {
 		return defProxyPass