Merge pull request kubernetes-sigs#50 from salrashid123/update-gke-gr…

…pc-api Update to networkingv1; omit TLS HC
jaison-tiu · Jun 30, 2021 · 733a844 · 733a844
2 parents 8ebcac3 + 1ce9449
commit 733a844
Show file tree

Hide file tree

Showing 6 changed files with 120 additions and 39 deletions.
diff --git a/ingress/ingress-custom-grpc-health-check/example/README.md b/ingress/ingress-custom-grpc-health-check/example/README.md
@@ -10,6 +10,14 @@ $ gcloud container  clusters create cluster-1 \
   --zone us-central1-a  --num-nodes 2 --enable-ip-alias  -q
 ```
 
+Configure a custom SSL Policy (this is optional and simply added to demonstrate custom TLS policies using `networking.gke.io/v1beta1.FrontEndConfig`)
+
+```
+gcloud compute ssl-policies create gke-ingress-ssl-policy \
+    --profile MODERN \
+    --min-tls-version 1.2 
+```
+
 Deploy application
 
 ```bash
@@ -34,9 +42,14 @@ ingress.networking.k8s.io/fe-ingress       <none>   *       34.120.140.72   80,
 
 export XLB_IP=`kubectl get ingress.extensions/fe-ingress -o jsonpath='{.status.loadBalancer.ingress[].ip}'`
 export ILB_IP=`kubectl get ingress.extensions/fe-ilb-ingress -o jsonpath='{.status.loadBalancer.ingress[].ip}'`
-```
 
+echo $XLB_IP
+echo $ILB_IP
+```
 
+NOTE:
+- [Configuring Ingress for external load balancing](https://cloud.google.com/kubernetes-engine/docs/how-to/load-balance-ingress#creating_an_ingress)
+ >> The only supported path types for the pathType field is ImplementationSpecific.
 #### Test External
 
 Verify external loadbalancing by transmitting 10 RPCs over one channel.  The responses will show different pods that handled each request
@@ -45,9 +58,8 @@ Verify external loadbalancing by transmitting 10 RPCs over one channel.  The res
 $ docker run --add-host grpc.domain.com:$XLB_IP  \
   -t gcr.io/cloud-solutions-images/grpc_app /grpc_client \
    --host=grpc.domain.com:443 --tlsCert /certs/CA_crt.pem \
-   --servername grpc.domain.com --repeat 10
+   --servername grpc.domain.com --repeat 10 -skipHealthCheck
 
-2021/01/27 12:53:08 RPC HealthChekStatus:SERVING
 2021/01/27 12:53:08 RPC Response: message:"Hello unary RPC msg   from hostname fe-deployment-6c96c9648-sztpp"
 2021/01/27 12:53:08 RPC Response: message:"Hello unary RPC msg   from hostname fe-deployment-6c96c9648-zj659"
 2021/01/27 12:53:08 RPC Response: message:"Hello unary RPC msg   from hostname fe-deployment-6c96c9648-zj659"
@@ -86,9 +98,8 @@ Rerun the test.  Notice the new pods in the response
 $ docker run --add-host grpc.domain.com:$XLB_IP \
    -t gcr.io/cloud-solutions-images/grpc_app /grpc_client \
    --host=grpc.domain.com:443 --tlsCert /certs/CA_crt.pem  \
-   --servername grpc.domain.com --repeat 10
+   --servername grpc.domain.com --repeat 10 -skipHealthCheck
 
-2021/01/27 12:54:18 RPC HealthChekStatus:SERVING
 2021/01/27 12:54:19 RPC Response: message:"Hello unary RPC msg   from hostname fe-deployment-6c96c9648-sztpp"
 2021/01/27 12:54:19 RPC Response: message:"Hello unary RPC msg   from hostname fe-deployment-6c96c9648-xc7d7"
 2021/01/27 12:54:19 RPC Response: message:"Hello unary RPC msg   from hostname fe-deployment-6c96c9648-d4rx2"
@@ -109,9 +120,8 @@ To test the internal loadbalancer, you must configure a VM from within an [alloc
  $ docker run --add-host grpc.domain.com:$XLB_IP \
      -t gcr.io/cloud-solutions-images/grpc_app /grpc_client \
      --host=grpc.domain.com:443 --tlsCert /certs/CA_crt.pem  \
-     --servername grpc.domain.com --repeat 10
+     --servername grpc.domain.com --repeat 10 -skipHealthCheck
 
-2021/01/27 12:51:45 RPC HealthChekStatus:SERVING
 2021/01/27 12:51:45 RPC Response: message:"Hello unary RPC msg   from hostname fe-deployment-6c96c9648-sztpp"
 2021/01/27 12:51:45 RPC Response: message:"Hello unary RPC msg   from hostname fe-deployment-6c96c9648-zj659"
 2021/01/27 12:51:45 RPC Response: message:"Hello unary RPC msg   from hostname fe-deployment-6c96c9648-sztpp"
@@ -124,6 +134,73 @@ To test the internal loadbalancer, you must configure a VM from within an [alloc
 2021/01/27 12:51:45 RPC Response: message:"Hello unary RPC msg   from hostname fe-deployment-6c96c9648-zj659"
 ```
 
+### HTTP/2 HealthChecks
+
+If you want the healthCheck proxy to use HTTP/2, you need to enable TLS termination on the proxy.  To do that, mount TLS certificates to the pod and configure the proxy to use them:
+
+e.g.
+
+```yaml
+apiVersion: v1
+data:
+  http_server_crt.pem: LS0tLS1CRUdJTiBDRVJ-redacted
+  http_server_key.pem: LS0tLS1CRUdJTiBQUkl-redacted
+kind: Secret
+metadata:
+  name: hc-secret
+  namespace: default
+type: Opaque
+---
+```
+
+- `fe-deployment.yaml`:
+
+```yaml
+    spec:
+      containers:
+      - name: hc-proxy
+        image: gcr.io/cloud-solutions-images/grpc_health_proxy:1.0.0
+        args: [
+          "--http-listen-addr=0.0.0.0:8443",
+          "--grpcaddr=localhost:50051",
+          "--service-name=echo.EchoServer",
+          "--https-listen-ca=/config/CA_crt.pem",
+          "--https-listen-cert=/certs/http_server_crt.pem",
+          "--https-listen-key=/certs/http_server_key.pem",
+          "--grpctls",        
+          "--grpc-sni-server-name=grpc.domain.com",
+          "--grpc-ca-cert=/config/CA_crt.pem",
+          "--logtostderr=1",
+          "-v=1"
+        ]
+        ports:
+        - containerPort: 8443
+        volumeMounts:
+        - name: certs-vol
+          mountPath: /certs
+          readOnly: true
+      volumes:
+      - name: certs-vol
+        secret:
+          secretName: hc-secret          
+```
+
+You will also need to configure the service to use HTTP/2:
+
+- `fe-srv-ingress.yaml`
+
+```yaml
+apiVersion: cloud.google.com/v1
+kind: BackendConfig
+metadata:
+  name: fe-grpc-backendconfig
+spec:
+  healthCheck:
+    type: HTTP2
+    requestPath: /
+    port: 8443
+```
+
 Source images used in this example can be found here:
   - [gcr.io/cloud-solutions-images/grpc_health_proxy](https://github.com/salrashid123/grpc_health_proxy)
   - [gcr.io/cloud-solutions-images/grpc_app](https://github.com/salrashid123/grpc_health_proxy/tree/master/example)
diff --git a/ingress/ingress-custom-grpc-health-check/example/fe-deployment.yaml b/ingress/ingress-custom-grpc-health-check/example/fe-deployment.yaml
@@ -35,9 +35,6 @@ spec:
           "--http-listen-addr=0.0.0.0:8443",
           "--grpcaddr=localhost:50051",
           "--service-name=echo.EchoServer",
-          "--https-listen-ca=/config/CA_crt.pem",
-          "--https-listen-cert=/certs/http_server_crt.pem",
-          "--https-listen-key=/certs/http_server_key.pem",
           "--grpctls",        
           "--grpc-sni-server-name=grpc.domain.com",
           "--grpc-ca-cert=/config/CA_crt.pem",
@@ -49,10 +46,7 @@ spec:
         volumeMounts:
         - name: config-vol
           mountPath: /config
-          readOnly: true
-        - name: certs-vol
-          mountPath: /certs
-          readOnly: true          
+          readOnly: true       
       - name: grpc-app
         image: gcr.io/cloud-solutions-images/grpc_app
         args: [
@@ -74,9 +68,6 @@ spec:
           items:
           - key: CA_crt.pem
             path: CA_crt.pem
-      - name: certs-vol
-        secret:
-          secretName: hc-secret
       - name: fe-certs-vol
         secret:
           secretName: fe-secret
diff --git a/ingress/ingress-custom-grpc-health-check/example/fe-ilb-ingress.yaml b/ingress/ingress-custom-grpc-health-check/example/fe-ilb-ingress.yaml
@@ -12,17 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-apiVersion: networking.k8s.io/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: fe-ilb-ingress
   annotations:
     kubernetes.io/ingress.allow-http: "false"
-    kubernetes.io/ingress.class: "gce-internal"
-
+    kubernetes.io/ingress.class: "gce-internal"    
 spec:
   tls:
   - secretName: fe-secret
-  backend:
-    serviceName: fe-srv-ingress
-    servicePort: 50051
+  rules:
+  - host: "grpc.domain.com"
+    http:
+      paths:
+      - path: "/echo.EchoServer/*"
+        pathType: ImplementationSpecific
+        backend:
+          service:
+            name: fe-srv-ingress
+            port:
+              number: 50051
diff --git a/ingress/ingress-custom-grpc-health-check/example/fe-ingress.yaml b/ingress/ingress-custom-grpc-health-check/example/fe-ingress.yaml
@@ -12,18 +12,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-apiVersion: networking.k8s.io/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: fe-ingress
   annotations:
     kubernetes.io/ingress.class: "gce"
-    kubernetes.io/ingress.allow-http: "false"   
+    kubernetes.io/ingress.allow-http: "false"
+    networking.gke.io/v1beta1.FrontendConfig: "fe-frontend-config"   
 spec:
   tls:
   - hosts:
     - grpc.domain.com
     secretName: fe-secret
-  backend:
-    serviceName: fe-srv-ingress
-    servicePort: 50051
+  rules:
+  - host: "grpc.domain.com"
+    http:
+      paths:
+      - path: "/echo.EchoServer/*"
+        pathType: ImplementationSpecific        
+        backend:
+          service:
+            name: fe-srv-ingress
+            port:
+              number: 50051
+---
+apiVersion: networking.gke.io/v1beta1
+kind: FrontendConfig
+metadata:
+  name: fe-frontend-config
+spec:
+  sslPolicy: gke-ingress-ssl-policy
diff --git a/ingress/ingress-custom-grpc-health-check/example/fe-secret.yaml b/ingress/ingress-custom-grpc-health-check/example/fe-secret.yaml
@@ -22,15 +22,5 @@ metadata:
   namespace: default
 type: Opaque
 ---
-apiVersion: v1
-data:
-  http_server_crt.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVNakNDQXhxZ0F3SUJBZ0lCSURBTkJna3Foa2lHOXcwQkFRVUZBREJYTVFzd0NRWURWUVFHRXdKVlV6RVAKTUEwR0ExVUVDZ3dHUjI5dloyeGxNUk13RVFZRFZRUUxEQXBGYm5SbGNuQnlhWE5sTVNJd0lBWURWUVFEREJsRgpiblJsY25CeWFYTmxJRk4xWW05eVpHbHVZWFJsSUVOQk1CNFhEVEl4TURFeU56RXlNRFl6TlZvWERUSTJNRGN5Ck1ERXlNRFl6TlZvd1RURUxNQWtHQTFVRUJoTUNWVk14RHpBTkJnTlZCQW9NQmtkdmIyZHNaVEVUTUJFR0ExVUUKQ3d3S1JXNTBaWEp3Y21selpURVlNQllHQTFVRUF3d1BhSFIwY0M1a2IyMWhhVzR1WTI5dE1JSUJJakFOQmdrcQpoa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQW1KRHoxbGFaNER0QTdOQnpqOVcxV0NNdk5rR09LblU0CnBEZndmd3FrQXc1T1k2SDdqY1BVcit6VnpHNVMrNDFaa0MvSVMwakpzZDZKcGVjS1V3VUpDRVVSQzVkUWJTdjYKN1JyczZXUFRQK1V4ZGg4Q1A3QWxLZTl0ZFltOUFrbkFXVWlCcTRLeFhxRW1KOHI3MEx5VW1FNUd0eUhGRUo0cgppSXhnSG55MVNWaTF4V0dGelBJUS9PckJGd3BRb0lMek9DbnRCc0VVSTJQaUxZTkVSVGM4bmlxNk5HZUFOaEFvCkwxTUZtUG5BQmhRaHBFY1FQQjNBUkhKTzRUb2FpUjJHakRJcnY4Qm1uaXFkZ214RHd3QlF4VGNMWU8wL2RlV1IKTllzVTZ3OVgwcXNlTHJpNXFMN3hVeFRGTjhGU0crMTNzQXc3VkpxUUJGaTZKdGY0dk5NSVF3SURBUUFCbzRJQgpFVENDQVEwd0RnWURWUjBQQVFIL0JBUURBZ2VBTUFrR0ExVWRFd1FDTUFBd0V3WURWUjBsQkF3d0NnWUlLd1lCCkJRVUhBd0V3SFFZRFZSME9CQllFRkJsY3hIaDlQQWRXRjkzckRQSGhqN000TFRRdE1COEdBMVVkSXdRWU1CYUEKRkwvaEhQQWlTSS84Tzg5ZDJlMnVpSEFoMzkyR01FUUdDQ3NHQVFVRkJ3RUJCRGd3TmpBMEJnZ3JCZ0VGQlFjdwpBb1lvYUhSMGNEb3ZMM0JyYVM1bGMyOWtaVzF2WVhCd01pNWpiMjB2WTJFdmRHeHpMV05oTG1ObGNqQTVCZ05WCkhSOEVNakF3TUM2Z0xLQXFoaWhvZEhSd09pOHZjR3RwTG1WemIyUmxiVzloY0hBeUxtTnZiUzlqWVM5MGJITXQKWTJFdVkzSnNNQm9HQTFVZEVRUVRNQkdDRDJoMGRIQXVaRzl0WVdsdUxtTnZiVEFOQmdrcWhraUc5dzBCQVFVRgpBQU9DQVFFQU9DVzBuaDMxbGViOW9Qd082MFNXRWFPMmNiNlZJdE9OOE5idHhLM3Q0WXB5M0QwSTNzLytabE4xCnhnaEo3eXJUQ2NlNFUybmxMOU1HNktrRGp1a2lQNDUxSjhiWUdpQ1BlVkhCQWFRa3ZVaVpWM2tQUjF3WGZJTlgKSWZGYVV6WjlqT3duUkpqTThQbEx6R2VpcDdBZG1ubU8vU25ENG1LWUZ6TlZFNmhjNW91eWhRYkFZcTJqTHA2NApJOFdpV0FWMk00NXpKQ3hmbTExUWo5enkyNDJlT2tCUjJmaFhiZ0YzQmZNRHhyUkc5UTFMMDg1bXVnNVA1ZTNjCldmd1JOa1d6QVlHSjVpTmE1bHVRbHp6NFpibWJnZk9yZGFuSG5SaU5PM1ZjMjhPWUdCUXhOY2VjdGttdjFSZksKelJ3VlduWEtiM1Nyc2pvTENiUWwwWWR0RUJxdXN3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==
-  http_server_key.pem: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2QUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktZd2dnU2lBZ0VBQW9JQkFRQ1lrUFBXVnBuZ08wRHMKMEhPUDFiVllJeTgyUVk0cWRUaWtOL0IvQ3FRRERrNWpvZnVOdzlTdjdOWE1ibEw3alZtUUw4aExTTW14M29tbAo1d3BUQlFrSVJSRUxsMUJ0Sy9ydEd1enBZOU0vNVRGMkh3SS9zQ1VwNzIxMWliMENTY0JaU0lHcmdyRmVvU1luCnl2dlF2SlNZVGthM0ljVVFuaXVJakdBZWZMVkpXTFhGWVlYTThoRDg2c0VYQ2xDZ2d2TTRLZTBHd1JRalkrSXQKZzBSRk56eWVLcm8wWjRBMkVDZ3ZVd1dZK2NBR0ZDR2tSeEE4SGNCRWNrN2hPaHFKSFlhTU1pdS93R2FlS3AyQwpiRVBEQUZERk53dGc3VDkxNVpFMWl4VHJEMWZTcXg0dXVMbW92dkZURk1VM3dWSWI3WGV3RER0VW1wQUVXTG9tCjEvaTgwd2hEQWdNQkFBRUNnZ0VBRnlNc2g1MzhtaUFDV2FmSFR0SDNQWldnYkZjR3kzT1prbllWVi83eUhyQ0YKdk9CcytQUHhCbmtoZDI1bjBWUi9kN2wzWTh0M0l5MS9ySlJjWVhqTEJPRU1rN2h2N3c2VGNHLzc2KzFDdFZiNQo2Rk1Oa1VFU2NjaW96Z0dFazV0QnU0aWVlYmRKVlBPVmlNVms2U1FnV1BUT0RxbmxhLzdBMVdXTWtqTVJmenpqCjZxaHR5dWMyRTgzbndZNGU5TlM1bW9KUVRHM3RzY044UHJxU0x0MTFzc1NwNU41Mjc0czhMaVRlQy9SQkJzWGUKMzUzcDhIL3ZtQUxnNnZQc2tjTzVxRTNLQTRDdDh6QkJaWnVxbk5sWkcyUnVJTXlXMzN5N3Q0WGdGNGo5SXB4YgppcGl2SkIxcTdvNjhjTXhWSTNiNFlBM1dwbElaaVNnQk9tbytvUVFKNFFLQmdRREdFNlQwdXJ0SEpSS1E0NzBoCnV4MWZRbkVCV3JyOCt1UWFvdU9TV3FxdHB5VnU3Vi9tQ3Q4UUJtbWNKTVlGOXBoQzh0WmRmUmRybUFUdHlFVzEKalFsbWhwdjRTeVRwMWVMUHBLMTBLcU5HTjBTQ1FyK0taSVJEQ0FtY0dpTVpyeEkyUUJNS2hrVU9IalRQeHQ3SgpyVEEvKzY5N3FoVWhCV2xtRGN4NzArVTgwUUtCZ1FERkxsQ2JaQ1cyUkcvYmhsOFNPc296S0poRCtWYy9lZHg2CnBxYjhqSklPVS9nREY2Q2F3c3MwMFZZYUlHZ0tQZUFGemozK2I0cWQ3WnM2a2JWR3JUUG1CQnRvbkZVdlFiN3EKYnpHMUxrdndpK1lVM3I3OFl5dEpsaDJzUzBSZHgzT040dkthQk4zbXRjdzZsY1poelZlY0lZUDFhc2RnSUp4NwoxSXRsaXAxbzB3S0JnSEJQb3pEQTBIKytuWkJhRVZ1VzVVaW1PNzB3M09Xa3ZNSDlxbHZKM1EwcHdTZnNvSHVuCm5tTlB2NEl4dDdhWjVDdlIxWXFjNnpLdXQ5eTd1Z3IvTlYxVTVnc2JVSXJtSS96b3V4RXJPL2tNVFdRdXhMeUwKMGhMUDgrTDNUdis1TFBKbDBtWWRTL3A5VmlTWjlxa1Q4ay9WUGVNNHhWSTc1MDllYzV1aXdsQ0JBb0dBU3hFawowdU8vajdKaU8xcmVtdHdTMm9NYjVOVHFLRnBHVzlOU1ExZG95MWVnVmQwSzRhQkRLR3FCc3hTZlJ6YmpNSktoCisxbVBsaXc5S29FS1dFdmNORGRnRCtWa2NNZEEzWk5UZ3p5SzRKc0NEdlAyRmJQVGFRSmpiWktDQm9uR0xrQTIKUi9pT1dpVGdDWFczdnNna0VHYWFERGFJak1vZGlPOFQycVo0NHMwQ2dZQXZ3NGdFWktSY0hJaVBKRHJHMU54RgpDeUhjVEY5Ti80SFlJa0MwaTE4SGNVRytaS2FWN1ZwMUVDb2JQUEZwakhib09qQWNHbWdyQWcwTTBZdnprRnRUCjJIOXpsYkx3UElxblJqTmxjbzl5bWphSnpEaEpadnNoVE5ZbHg1b0YyNzIzblBTZXpldkR3dmI5NExBM3hVVnQKVytoTWkveUVId2x1QjZORkVldjlHUT09Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0=
-kind: Secret
-metadata:
-  name: hc-secret
-  namespace: default
-type: Opaque
----
 
 
diff --git a/ingress/ingress-custom-grpc-health-check/example/fe-srv-ingress.yaml b/ingress/ingress-custom-grpc-health-check/example/fe-srv-ingress.yaml
@@ -39,6 +39,6 @@ metadata:
   name: fe-grpc-backendconfig
 spec:
   healthCheck:
-    type: HTTP2
+    type: HTTP
     requestPath: /
     port: 8443