feat: add optional Gateway API resource definition to expose endpoints

charts/hedera-network/config-files/envoy.yaml

@@ -51,5 +51,5 @@ static_resources:
               - endpoint:
                   address:
                     socket_address:
-                      address: network-{{ .nodeConfig.name }}-service
-                      port_value: 50211
+                      address: network-{{ .nodeConfig.name }}-svc
+                      port_value: 50211

charts/hedera-network/config-files/haproxy.cfg

@@ -11,17 +11,16 @@ defaults
     timeout client 30s
     timeout server 30s
 frontend fe_proxy
-    mode http
-    option httplog
-    option http-use-htx
+    mode tcp
+    option tcplog
     option logasap
     # Read the blacklist of OFAC Sanctioned Country IP List
     # Log & Reject the Traffic
     acl ofac_ip src -f /var/opt/ofac/ofacBlock.acl
     tcp-request connection reject if ofac_ip
     acl ofac_ip_xff hdr_ip(x-forwarded-for) -f /var/opt/ofac/ofacBlock.acl
     http-request deny if ofac_ip_xff
-    bind *:50211 proto h2
+    bind *:50211
     default_backend be_servers
 frontend fe_proxy_tls
     mode tcp
@@ -33,13 +32,12 @@ frontend fe_proxy_tls
     bind *:50212
     default_backend be_servers_tls
 backend be_servers
-    mode http
-    option http-use-htx
+    mode tcp
     # Health Checks: Poll the server with regular health checks every 10 seconds
     # Health Checks: Observe all Layer 4 TCP connections for problems.
     # Health Checks: 5 connections must fail before on-error parameter is invoked and marks server down.
     # Health Checks: Revive the server with regular health checks by polling the server every 10 seconds.
-    server server1 network-{{ .nodeConfig.name }}-svc:50211 proto h2 check inter 10s downinter 10s observe layer4 error-limit 5 on-error mark-down
+    server server1 network-{{ .nodeConfig.name }}-svc:50211 check inter 10s downinter 10s observe layer4 error-limit 5 on-error mark-down
 backend be_servers_tls
     mode tcp
     option ssl-hello-chk

charts/hedera-network/templates/gateway-api/envoy-routes.yaml

@@ -0,0 +1,24 @@
+{{- range $index, $node := ($.Values.hedera.nodes) }}
+{{- $envoyProxy := $node.envoyProxy | default dict -}}
+{{- $defaults := $.Values.defaults.envoyProxy }}
+{{- if default $defaults.enable $envoyProxy.enable | eq "true" }}
+---
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: HTTPRoute
+metadata:
+  name: envoy-routes-{{ $node.name }}
+  namespace: default
+  labels:
+    fullstack.hedera.com/type: http-route
+spec:
+  parentRefs:
+    - name: fst
+      sectionName: http-{{ $node.name }}
+  hostnames:
+    - {{ tpl $.Values.gatewayApi.route.hostname (dict "node" $node "Template" $.Template) }}
+  rules:
+    - backendRefs:
+      - name: envoy-proxy-{{ $node.name }}-svc
+        port: 8080
+{{- end }}
+{{- end }}

charts/hedera-network/templates/gateway-api/gateway.yaml

@@ -0,0 +1,52 @@
+{{- if $.Values.gatewayApi.gatewayClass.enable | eq "true" }}
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: GatewayClass
+metadata:
+  name: {{ $.Values.gatewayApi.gatewayClass.name }}
+  namespace: default
+  labels:
+    fullstack.hedera.com/type: gateway-class
+spec:
+  controllerName: {{ $.Values.gatewayApi.gatewayClass.controllerName }}
+{{- end }}
+{{- if $.Values.gatewayApi.gateway.enable | eq "true" }}
+---
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: Gateway
+metadata:
+  name: {{ $.Values.gatewayApi.gateway.name }}
+  namespace: default
+  labels:
+    fullstack.hedera.com/type: gateway
+spec:
+  gatewayClassName: {{ $.Values.gatewayApi.gatewayClass.name }}
+  listeners:
+    - name: http-debug
+      protocol: HTTP
+      port: 80
+    - name: tcp-debug
+      protocol: TCP
+      port: 9000
+      allowedRoutes:
+        kinds:
+          - kind: TCPRoute
+    - name: grpc-debug
+      protocol: TCP
+      port: 9090
+      allowedRoutes:
+        kinds:
+          - kind: TCPRoute # we use TCPRoute to for GRPC
+    {{- range $index, $node := $.Values.hedera.nodes }}
+    {{- $tcp_port := mul $index 1000 | add 50211 }}
+    {{- $http_port := mul $index 100 | add 8080 }}
+    - name: tcp-{{ $node.name }} # for haproxy or network-node TCPRoute
+      protocol: TCP
+      port: {{ $tcp_port }}
+      allowedRoutes:
+        kinds:
+          - kind: TCPRoute
+    - name: http-{{ $node.name }} # for envoy-proxy HTTPRoute
+      protocol: HTTP
+      port: {{ $http_port }}
+    {{- end }}
+{{- end }}

charts/hedera-network/templates/gateway-api/haproxy-routes.yaml

@@ -0,0 +1,23 @@
+{{- range $index, $node := ($.Values.hedera.nodes) }}
+{{- $haproxy := $node.haproxy | default dict -}}
+{{- $defaults := $.Values.defaults.haproxy }}
+{{- if default $defaults.enable $haproxy.enable | eq "true" }}
+---
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: TCPRoute
+metadata:
+  name: haproxy-grpc-route-{{ $node.name }}
+  namespace: default
+  labels:
+    fullstack.hedera.com/type: tcp-route
+    fullstack.hedera.com/nodeName: {{ $node.name }}
+spec:
+  parentRefs:
+    - name: fst
+      sectionName: tcp-{{ $node.name }}
+  rules:
+    - backendRefs:
+      - name: haproxy-{{ $node.name }}-svc
+        port: 50211
+{{- end }}
+{{- end }}

charts/hedera-network/templates/gateway-api/network-node-routes.yaml

@@ -0,0 +1,23 @@
+{{- range $index, $node := ($.Values.hedera.nodes) }}
+{{- $haproxy := $node.haproxy | default dict -}}
+{{- $defaults := $.Values.defaults.haproxy }}
+{{- if default $defaults.enable $haproxy.enable | eq "false" }}
+---
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: TCPRoute
+metadata:
+  name: node-grpc-route-{{ $node.name }}
+  namespace: default
+  labels:
+    fullstack.hedera.com/type: tcp-route
+    fullstack.hedera.com/nodeName: {{ $node.name }}
+spec:
+  parentRefs:
+    - name: fst
+      sectionName: tcp-{{ $node.name }}
+  rules:
+    - backendRefs:
+      - name: network-{{ $node.name }}-svc
+        port: 50211
+{{- end }}
+{{- end }}

charts/hedera-network/templates/services/envoy-svc.yaml

@@ -1,6 +1,6 @@
 {{ range $index, $node := ($.Values.hedera.nodes) }}
 {{- $envoyProxy := $node.envoyProxy | default dict -}}
-{{- $defaults := $.Values.defaults.haproxy }}
+{{- $defaults := $.Values.defaults.envoyProxy }}
 {{- if default $defaults.enable $envoyProxy.enable | eq "true" }}
 ---
 apiVersion: v1

charts/hedera-network/values.yaml

@@ -30,6 +30,19 @@ tester:
     pullPolicy: "IfNotPresent"
   resources: {}
 
+# gateway-api configuration
+gatewayApi:
+  gatewayClass:
+    name: "fst"
+    enable: "true"
+    controllerName: "gateway.envoyproxy.io/gatewayclass-controller"
+#    controllerName: "haproxy-ingress.github.io/controller"
+  gateway:
+    name: "fst"
+    enable: "true"
+  route:
+    hostname: "{{ .node.name }}.fst.local"
+
 # default settings for a single node
 # This default configurations can be overridden for each node in the hedera.nodes section.
 defaults:

dev/Makefile

@@ -23,7 +23,10 @@ SCRIPTS_DIR=$(PWD)/scripts
 CHART_DIR=$(PWD)/../charts/hedera-network
 SCRIPT_NAME=direct-install.sh
 TMP_DIR=${SCRIPTS_DIR}/../temp
+
+# scripts
 TELEMETRY_SCRIPT="telemetry.sh"
+GATEWAY_API_SCRIPT="gateway.sh"
 
 .PHONY: all
 all: setup setup-cluster reset
@@ -42,7 +45,7 @@ update-helm-dependencies:
 	helm dependency update ../charts/hedera-network
 
 .PHONY: deploy-chart
-deploy-chart: deploy-minio-operator-if-required deploy-prometheus-operator
+deploy-chart: deploy-minio-operator-if-required deploy-prometheus-operator deploy-gateway-api
 	echo ">> Deploying helm chart..." && \
 	echo "" && \
 	if [ "${SCRIPT_NAME}" = "nmt-install.sh" ]; then \
@@ -143,6 +146,17 @@ restart: stop-nodes start-nodes
 .PHONY: reset
 reset: destroy-network start
 
+######################################### Gateway API #################################
+.PHONY: deploy-gateway-api
+deploy-gateway-api:
+	#source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && deploy_haproxy_ingress
+	source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && deploy_envoy_gateway_api
+
+.PHONY: destroy-gateway-api
+destroy-gateway-api:
+	#source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_haproxy_ingress
+	source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_envoy_gateway_api
+
 ######################################### Prometheus #################################
 .PHONY: fetch-prometheus-operator-bundle
 fetch-prometheus-operator-bundle:

dev/gateway-api/Makefile

@@ -0,0 +1,43 @@
+SHELLOPTS:=$(if $(SHELLOPTS),$(SHELLOPTS):)pipefail:errexit
+.ONESHELL:
+
+SCRIPTS_DIR=$(PWD)/../scripts
+GATEWAY_API_SCRIPT="gateway.sh"
+
+.PHONY: deploy-gateway-api
+deploy-gateway-api:
+	#source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && deploy_haproxy_ingress
+	source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && deploy_envoy_gateway_api
+
+.PHONY: destroy-gateway-api
+destroy-gateway-api:
+	#source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_haproxy_ingress
+	source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_envoy_gateway_api
+
+.PHONY: deploy-fst-gateway
+deploy-fst-gateway: deploy-gateway-api
+	kubectl apply -f ./fst-gateway.yaml
+
+.PHONY: destroy-fst-gateway
+destroy-fst-gateway:
+	kubectl delete -f fst-gateway.yaml
+
+.PHONY: expose-envoy-gateway-svc
+expose-envoy-gateway-svc:
+	source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && expose_envoy_gateway_svc ${PORT} ${GATEWAY_PORT}
+
+.PHONY: unexpose-envoy-gateway-svc
+unexpose-envoy-gateway-svc:
+	source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && unexpose_envoy_gateway_svc
+
+.PHONY: test-gateway-http-route
+test-gateway-http-route:
+	source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && test_http_route
+
+.PHONY: test-gateway-grpc-route
+test-gateway-grpc-route:
+	source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && test_grpc_route
+
+.PHONY: test-gateway-tcp-route
+test-gateway-tcp-route:
+	source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && test_tcp_route

dev/gateway-api/README.md

@@ -0,0 +1,29 @@
+# Gateway API
+This folder includes scripts and files to debug Gateway API.
+
+## Manual Test
+- Deploy `fst` Gateway
+    - `make deploy-fst-gateway`
+- Test HTTPRoute
+    - ` make test-gateway-http-route` and check for below line
+  ```
+  ********************************************************
+  SUCCESS: HTTPRoute debug.fst.local:8080
+  ********************************************************
+  ```
+- Test TCPRoute
+    - ` make test-gateway-tcp-route` and check for below line
+  ```
+  ********************************************************
+  SUCCESS: TCPRoute localhost:9000
+  ********************************************************
+  ```
+- Test GRPCRoute
+    - ` make test-gateway-grpc-route` and check for below line
+  ```
+  ********************************************************
+  SUCCESS: GRPCRoute debug.fst.local:9090
+  ********************************************************
+  ```
+- Delete `fst` Gateway
+    - `make destroy-fst-gateway`

dev/gateway-api/fst-gateway.yaml

@@ -0,0 +1,31 @@
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: GatewayClass
+metadata:
+  name: fst
+spec:
+  controllerName: gateway.envoyproxy.io/gatewayclass-controller
+#  controllerName: haproxy-ingress.github.io/controller
+---
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: Gateway
+metadata:
+  name: fst
+spec:
+  gatewayClassName: fst
+  listeners:
+    - name: http-debug
+      protocol: HTTP
+      port: 80
+    - name: tcp-debug
+      protocol: TCP
+      port: 9000
+      allowedRoutes:
+        kinds:
+          - kind: TCPRoute
+    - name: grpc-debug
+      protocol: TCP
+      port: 9090
+      allowedRoutes:
+        kinds:
+          - kind: TCPRoute # we use TCPRoute to for GRPC
+---