Add worker-cpu-affinity nginx option (kubernetes#2201)

worker_cpu_affinity is a common optimization method for improving nginx performance, adding this as a custom configuration. Also fix some format issues found during editing.

Makefile

@@ -97,7 +97,7 @@ container: .container-$(ARCH)
 .PHONY: .container-$(ARCH)
 .container-$(ARCH):
 	cp -RP ./* $(TEMP_DIR)
-	$(SED_I) 's|BASEIMAGE|$(BASEIMAGE)|g' $(DOCKERFILE)
+	$(SED_I) "s|BASEIMAGE|$(BASEIMAGE)|g" $(DOCKERFILE)
 	$(SED_I) "s|QEMUARCH|$(QEMUARCH)|g" $(DOCKERFILE)
 	$(SED_I) "s|DUMB_ARCH|$(DUMB_ARCH)|g" $(DOCKERFILE)
 

docs/user-guide/configmap.md

@@ -88,6 +88,7 @@ The following table shows a configuration option's name, type, and the default v
 |[use-http2](#use-http2)|bool|"true"|
 |[gzip-types](#gzip-types)|string|"application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component"|
 |[worker-processes](#worker-processes)|string|`<Number of CPUs>`|
+|[worker-cpu-affinity](#worker-cpu-affinity)|string|""|
 |[worker-shutdown-timeout](#worker-shutdown-timeout)|string|"10s"|
 |[load-balance](#load-balance)|string|"least_conn"|
 |[variables-hash-bucket-size](#variables-hash-bucket-size)|int|128|
@@ -489,6 +490,15 @@ Sets the MIME types in addition to "text/html" to compress. The special value "\
 Sets the number of [worker processes](http://nginx.org/en/docs/ngx_core_module.html#worker_processes).
 The default of "auto" means number of available CPU cores.
 
+## worker-cpu-affinity
+
+Binds worker processes to the sets of CPUs. [worker_cpu_affinity](http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity).
+By default worker processes are not bound to any specific CPUs. The value can be:
+
+- "": empty string indicate no affinity is applied.
+- cpumask: e.g. `0001 0010 0100 1000` to bind processes to specific cpus.
+- auto: binding worker processes automatically to available CPUs.
+
 ## worker-shutdown-timeout
 
 Sets a timeout for Nginx to [wait for worker to gracefully shutdown](http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout). The default is "10s".

internal/ingress/controller/config/config.go

@@ -101,6 +101,10 @@ type Configuration struct {
 	// By default access logs go to /var/log/nginx/access.log
 	AccessLogPath string `json:"access-log-path,omitempty"`
 
+	// WorkerCpuAffinity bind nginx worker processes to CPUs this will improve response latency
+	// http://nginx.org/en/docs/ngx_core_module.html#worker_cpu_affinity
+	// By default this is disabled
+	WorkerCpuAffinity string `json:"worker-cpu-affinity,omitempty"`
 	// ErrorLogPath sets the path of the error logs
 	// http://nginx.org/en/docs/ngx_core_module.html#error_log
 	// By default error logs go to /var/log/nginx/error.log
@@ -492,6 +496,7 @@ func NewDefault() Configuration {
 	cfg := Configuration{
 		AllowBackendServerHeader:   false,
 		AccessLogPath:              "/var/log/nginx/access.log",
+		WorkerCpuAffinity:          "",
 		ErrorLogPath:               "/var/log/nginx/error.log",
 		BrotliLevel:                4,
 		BrotliTypes:                brotliTypes,

internal/ingress/controller/template/configmap.go

@@ -56,9 +56,9 @@ func ReadConfig(src map[string]string) config.Configuration {
 
 	errors := make([]int, 0)
 	skipUrls := make([]string, 0)
-	whitelist := make([]string, 0)
-	proxylist := make([]string, 0)
-	hideHeaderslist := make([]string, 0)
+	whiteList := make([]string, 0)
+	proxyList := make([]string, 0)
+	hideHeadersList := make([]string, 0)
 
 	bindAddressIpv4List := make([]string, 0)
 	bindAddressIpv6List := make([]string, 0)
@@ -77,21 +77,21 @@ func ReadConfig(src map[string]string) config.Configuration {
 	}
 	if val, ok := conf[hideHeaders]; ok {
 		delete(conf, hideHeaders)
-		hideHeaderslist = strings.Split(val, ",")
+		hideHeadersList = strings.Split(val, ",")
 	}
 	if val, ok := conf[skipAccessLogUrls]; ok {
 		delete(conf, skipAccessLogUrls)
 		skipUrls = strings.Split(val, ",")
 	}
 	if val, ok := conf[whitelistSourceRange]; ok {
 		delete(conf, whitelistSourceRange)
-		whitelist = append(whitelist, strings.Split(val, ",")...)
+		whiteList = append(whiteList, strings.Split(val, ",")...)
 	}
 	if val, ok := conf[proxyRealIPCIDR]; ok {
 		delete(conf, proxyRealIPCIDR)
-		proxylist = append(proxylist, strings.Split(val, ",")...)
+		proxyList = append(proxyList, strings.Split(val, ",")...)
 	} else {
-		proxylist = append(proxylist, "0.0.0.0/0")
+		proxyList = append(proxyList, "0.0.0.0/0")
 	}
 	if val, ok := conf[bindAddress]; ok {
 		delete(conf, bindAddress)
@@ -137,11 +137,11 @@ func ReadConfig(src map[string]string) config.Configuration {
 	to := config.NewDefault()
 	to.CustomHTTPErrors = filterErrors(errors)
 	to.SkipAccessLogURLs = skipUrls
-	to.WhitelistSourceRange = whitelist
-	to.ProxyRealIPCIDR = proxylist
+	to.WhitelistSourceRange = whiteList
+	to.ProxyRealIPCIDR = proxyList
 	to.BindAddressIpv4 = bindAddressIpv4List
 	to.BindAddressIpv6 = bindAddressIpv6List
-	to.HideHeaders = hideHeaderslist
+	to.HideHeaders = hideHeadersList
 	to.HTTPRedirectCode = redirectCode
 	to.ProxyStreamResponses = streamResponses
 	to.DisableIpv6DNS = !ing_net.IsIPv6Enabled()

rootfs/etc/nginx/template/nginx.tmpl

@@ -16,10 +16,14 @@ load_module /etc/nginx/modules/ngx_http_modsecurity_module.so;
 daemon off;
 
 worker_processes {{ $cfg.WorkerProcesses }};
+{{ if gt (len $cfg.WorkerCpuAffinity) 0 }}
+worker_cpu_affinity {{ $cfg.WorkerCpuAffinity }};
+{{ end }}
+
 pid /run/nginx.pid;
 {{ if ne .MaxOpenFiles 0 }}
 worker_rlimit_nofile {{ .MaxOpenFiles }};
-{{ end}}
+{{ end }}
 
 {{/* http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout */}}
 {{/* avoid waiting too long during a reload */}}