GRPCRoute Support

Makefile

@@ -16,6 +16,7 @@ TELEMETRY_ENDPOINT=# if empty, NGF will report telemetry in its logs at debug le
 TELEMETRY_ENDPOINT_INSECURE = false
 
 GW_API_VERSION = 1.0.0
+ENABLE_EXPERIMENTAL = false
 NODE_VERSION = $(shell cat .nvmrc)
 
 # go build flags - should not be overridden by the user
@@ -192,13 +193,13 @@ install-ngf-local-build-with-plus: build-images-with-plus load-images-with-plus
 
 .PHONY: helm-install-local
 helm-install-local: ## Helm install NGF on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build.
-	./conformance/scripts/install-gateway.sh $(GW_API_VERSION)
-	helm install dev $(CHART_DIR) --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never -n nginx-gateway
+	./conformance/scripts/install-gateway.sh $(GW_API_VERSION) $(ENABLE_EXPERIMENTAL)
+	helm install dev $(CHART_DIR) --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway
 
 .PHONY: helm-install-local-with-plus
 helm-install-local-with-plus: ## Helm install NGF with NGINX Plus on configured kind cluster with local images. To build, load, and install with helm run make install-ngf-local-build-with-plus.
-	./conformance/scripts/install-gateway.sh $(GW_API_VERSION)
-	helm install dev $(CHART_DIR) --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginx.plus=true -n nginx-gateway
+	./conformance/scripts/install-gateway.sh $(GW_API_VERSION) $(ENABLE_EXPERIMENTAL)
+	helm install dev $(CHART_DIR) --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=$(PREFIX) --set nginxGateway.image.tag=$(TAG) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=$(NGINX_PLUS_PREFIX) --set nginx.image.tag=$(TAG) --set nginx.image.pullPolicy=Never --set nginx.plus=true --set nginxGateway.gwAPIExperimentalFeatures.enable=$(ENABLE_EXPERIMENTAL) -n nginx-gateway
 
 # Debug Targets
 .PHONY: debug-build

README.md

@@ -11,7 +11,7 @@
 
 NGINX Gateway Fabric is an open-source project that provides an implementation of
 the [Gateway API](https://gateway-api.sigs.k8s.io/) using [NGINX](https://nginx.org/) as the data plane. The goal of
-this project is to implement the core Gateway APIs -- `Gateway`, `GatewayClass`, `HTTPRoute`, `TCPRoute`, `TLSRoute`,
+this project is to implement the core Gateway APIs -- `Gateway`, `GatewayClass`, `HTTPRoute`, `GRPCRoute`, `TCPRoute`, `TLSRoute`,
 and `UDPRoute` -- to configure an HTTP or TCP/UDP load balancer, reverse-proxy, or API gateway for applications running
 on Kubernetes. NGINX Gateway Fabric supports a subset of the Gateway API.
 

build/Dockerfile.nginx

@@ -13,6 +13,8 @@ RUN apk add --no-cache libcap \
 
 COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
 COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
+COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
+COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf
 
 RUN chown -R 101:1001 /etc/nginx /var/cache/nginx /var/lib/nginx
 

build/Dockerfile.nginxplus

@@ -29,6 +29,8 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
 
 COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
 COPY ${NGINX_CONF_DIR}/nginx-plus.conf /etc/nginx/nginx.conf
+COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf
+COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf
 
 RUN chown -R 101:1001 /etc/nginx /var/cache/nginx /var/lib/nginx
 

charts/nginx-gateway-fabric/templates/rbac.yaml

@@ -92,6 +92,7 @@ rules:
   - referencegrants
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - backendtlspolicies
+  - grpcroutes
 {{- end }}
   verbs:
   - list
@@ -104,6 +105,7 @@ rules:
   - gatewayclasses/status
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - backendtlspolicies/status
+  - grpcroutes/status
 {{- end }}
   verbs:
   - update

conformance/Makefile

@@ -17,6 +17,10 @@ PROVISIONER_MANIFEST=provisioner/provisioner.yaml
 ENABLE_EXPERIMENTAL ?= false
 .DEFAULT_GOAL := help
 
+ifeq ($(ENABLE_EXPERIMENTAL),true)
+    SUPPORTED_FEATURES +=,GRPCExactMethodMatching,GRPCRouteListenerHostnameMatching,GRPCRouteHeaderMatching
+endif
+
 .PHONY: help
 help: Makefile ## Display this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n    make \033[36m<target>\033[0m\n\nTargets:\n\n"}; {printf "    \033[36m%-30s\033[0m %s\n", $$1, $$2}'

deploy/manifests/nginx-gateway-experimental.yaml

@@ -77,6 +77,7 @@ rules:
   - httproutes
   - referencegrants
   - backendtlspolicies
+  - grpcroutes
   verbs:
   - list
   - watch
@@ -87,6 +88,7 @@ rules:
   - gateways/status
   - gatewayclasses/status
   - backendtlspolicies/status
+  - grpcroutes/status
   verbs:
   - update
 - apiGroups:

deploy/manifests/nginx-plus-gateway-experimental.yaml

@@ -83,6 +83,7 @@ rules:
   - httproutes
   - referencegrants
   - backendtlspolicies
+  - grpcroutes
   verbs:
   - list
   - watch
@@ -93,6 +94,7 @@ rules:
   - gateways/status
   - gatewayclasses/status
   - backendtlspolicies/status
+  - grpcroutes/status
   verbs:
   - update
 - apiGroups:

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.2 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect

go.sum

@@ -11,8 +11,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
-github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/emicklei/go-restful/v3 v3.11.2 h1:1onLa9DcsMYO9P+CXaL0dStDqQ2EHHXLiz+BtnqkLAU=
+github.com/emicklei/go-restful/v3 v3.11.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI=
 github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=

internal/framework/gatewayclass/validate.go

@@ -22,6 +22,7 @@ var gatewayCRDs = map[string]apiVersion{
 	"httproutes.gateway.networking.k8s.io":         {},
 	"referencegrants.gateway.networking.k8s.io":    {},
 	"backendtlspolicies.gateway.networking.k8s.io": {},
+	"grpcroutes.gateway.networking.k8s.io":         {},
 }
 
 type apiVersion struct {

internal/mode/static/handler.go

@@ -245,7 +245,13 @@ func (h *eventHandlerImpl) updateStatuses(ctx context.Context, logger logr.Logge
 	if h.cfg.updateGatewayClassStatus {
 		gcReqs = status.PrepareGatewayClassRequests(graph.GatewayClass, graph.IgnoredGatewayClasses, transitionTime)
 	}
-	routeReqs := status.PrepareRouteRequests(graph.Routes, transitionTime, h.latestReloadResult, h.cfg.gatewayCtlrName)
+	routeReqs := status.PrepareRouteRequests(
+		graph.Routes,
+		transitionTime,
+		h.latestReloadResult,
+		h.cfg.gatewayCtlrName,
+	)
+
 	polReqs := status.PrepareBackendTLSPolicyRequests(graph.BackendTLSPolicies, transitionTime, h.cfg.gatewayCtlrName)
 
 	reqs := make([]frameworkStatus.UpdateRequest, 0, len(gcReqs)+len(routeReqs)+len(polReqs))

internal/mode/static/manager.go

@@ -424,7 +424,7 @@
 	}
 
 	if cfg.ExperimentalFeatures {
-		backendTLSObjs := []ctlrCfg{
+		gwExpFeatures := []ctlrCfg{
 			{
 				objectType: &gatewayv1alpha2.BackendTLSPolicy{},
 				options: []controller.Option{
@@ -436,8 +436,14 @@
 				// https://github.com/nginxinc/nginx-gateway-fabric/issues/1545
 				objectType: &apiv1.ConfigMap{},
 			},
+			{
+				objectType: &gatewayv1alpha2.GRPCRoute{},
+				options: []controller.Option{
+					controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
+				},
+			},
 		}
-		controllerRegCfgs = append(controllerRegCfgs, backendTLSObjs...)
+		controllerRegCfgs = append(controllerRegCfgs, gwExpFeatures...)
 	}
 
 	if cfg.ConfigName != "" {
@@ -604,7 +610,12 @@
 	}
 
 	if enableExperimentalFeatures {
-		objectLists = append(objectLists, &gatewayv1alpha2.BackendTLSPolicyList{}, &apiv1.ConfigMapList{})
+		objectLists = append(
+			objectLists,
+			&gatewayv1alpha2.BackendTLSPolicyList{},
+			&apiv1.ConfigMapList{},
+			&gatewayv1alpha2.GRPCRouteList{},
+		)
 	}
 
 	if gwNsName == nil {

internal/mode/static/manager_test.go

@@ -99,6 +99,7 @@ func TestPrepareFirstEventBatchPreparerArgs(t *testing.T) {
 				&ngfAPI.NginxProxyList{},
 				partialObjectMetadataList,
 				&gatewayv1alpha2.BackendTLSPolicyList{},
+				&gatewayv1alpha2.GRPCRouteList{},
 			},
 			experimentalEnabled: true,
 		},

internal/mode/static/nginx/conf/grpc-error-locations.conf

@@ -0,0 +1,55 @@
+location @grpc_deadline_exceeded {
+    default_type application/grpc;
+    add_header content-type application/grpc;
+    add_header grpc-status 4;
+    add_header grpc-message 'deadline exceeded';
+    return 204;
+}
+
+location @grpc_permission_denied {
+    default_type application/grpc;
+    add_header content-type application/grpc;
+    add_header grpc-status 7;
+    add_header grpc-message 'permission denied';
+    return 204;
+}
+
+location @grpc_resource_exhausted {
+    default_type application/grpc;
+    add_header content-type application/grpc;
+    add_header grpc-status 8;
+    add_header grpc-message 'resource exhausted';
+    return 204;
+}
+
+location @grpc_unimplemented {
+    default_type application/grpc;
+    add_header content-type application/grpc;
+    add_header grpc-status 12;
+    add_header grpc-message unimplemented;
+    return 204;
+}
+
+location @grpc_internal {
+    default_type application/grpc;
+    add_header content-type application/grpc;
+    add_header grpc-status 13;
+    add_header grpc-message 'internal error';
+    return 204;
+}
+
+location @grpc_unavailable {
+    default_type application/grpc;
+    add_header content-type application/grpc;
+    add_header grpc-status 14;
+    add_header grpc-message unavailable;
+    return 204;
+}
+
+location @grpc_unauthenticated {
+    default_type application/grpc;
+    add_header content-type application/grpc;
+    add_header grpc-status 16;
+    add_header grpc-message unauthenticated;
+    return 204;
+}

internal/mode/static/nginx/conf/grpc-error-pages.conf

@@ -0,0 +1,19 @@
+error_page 400 = @grpc_internal;
+error_page 401 = @grpc_unauthenticated;
+error_page 403 = @grpc_permission_denied;
+error_page 404 = @grpc_unimplemented;
+error_page 429 = @grpc_unavailable;
+error_page 502 = @grpc_unavailable;
+error_page 503 = @grpc_unavailable;
+error_page 504 = @grpc_unavailable;
+error_page 405 = @grpc_internal;
+error_page 408 = @grpc_deadline_exceeded;
+error_page 413 = @grpc_resource_exhausted;
+error_page 414 = @grpc_resource_exhausted;
+error_page 415 = @grpc_internal;
+error_page 426 = @grpc_internal;
+error_page 495 = @grpc_unauthenticated;
+error_page 496 = @grpc_unauthenticated;
+error_page 497 = @grpc_internal;
+error_page 500 = @grpc_internal;
+error_page 501 = @grpc_internal;

internal/mode/static/nginx/conf/nginx-plus.conf

@@ -27,6 +27,8 @@ http {
   sendfile on;
   tcp_nopush on;
 
+  http2 on;
+
   server {
     listen 127.0.0.1:8765;
     root /usr/share/nginx/html;

internal/mode/static/nginx/conf/nginx.conf

@@ -27,6 +27,8 @@ http {
   sendfile on;
   tcp_nopush on;
 
+  http2 on;
+
   server {
     listen unix:/var/run/nginx/nginx-status.sock;
     access_log off;

internal/mode/static/nginx/config/http/config.go

@@ -7,6 +7,7 @@ type Server struct {
 	Locations     []Location
 	IsDefaultHTTP bool
 	IsDefaultSSL  bool
+	GRPC          bool
 	Port          int32
 }
 
@@ -19,9 +20,10 @@ type Location struct {
 	ProxySSLVerify  *ProxySSLVerify
 	Return          *Return
 	Rewrites        []string
+	GRPC            bool
 }
 
-// Header defines a HTTP header to be passed to the proxied server.
+// Header defines an HTTP header to be passed to the proxied server.
 type Header struct {
 	Name  string
 	Value string