Migrate K8s HTTPRoutes to GRPCRoutes (#868)

This way we can make use of pure grpc routing.

Includes some hardening for newer docker daemons which might use
dual-stack ip subnets.

Fixes https://github.com/TraceMachina/nativelink/security/dependabot/31

deployment-examples/chromium/routes.yaml

@@ -1,27 +1,25 @@
 ---
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute  # TODO(aaronmondal): Use GRPCRoute after resolution of
-                 # https://github.com/TraceMachina/nativelink/issues/481
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: GRPCRoute
 metadata:
   name: cache-route
 spec:
   parentRefs:
-    - sectionName: cache
-      name: cache
+    - name: cache
+      sectionName: cache
   rules:
     - backendRefs:
         - name: nativelink-cas
           port: 50051
 ---
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute # TODO(aaronmondal): Pure GRPC is unstable here. Find out why
-                #                    and migrate to a GRPCRoute.
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: GRPCRoute
 metadata:
   name: scheduler-route
 spec:
   parentRefs:
-    - sectionName: scheduler
-      name: scheduler
+    - name: scheduler
+      sectionName: scheduler
   rules:
     - backendRefs:
         - name: nativelink-scheduler

deployment-examples/kubernetes/routes.yaml

@@ -1,27 +1,25 @@
 ---
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute  # TODO(aaronmondal): Use GRPCRoute after resolution of
-                 # https://github.com/TraceMachina/nativelink/issues/481
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: GRPCRoute
 metadata:
   name: cache-route
 spec:
   parentRefs:
-    - sectionName: cache
-      name: cache
+    - name: cache
+      sectionName: cache
   rules:
     - backendRefs:
         - name: nativelink-cas
           port: 50051
 ---
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute # TODO(aaronmondal): Pure GRPC is unstable here. Find out why
-                #                    and migrate to a GRPCRoute.
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: GRPCRoute
 metadata:
   name: scheduler-route
 spec:
   parentRefs:
-    - sectionName: scheduler
-      name: scheduler
+    - name: scheduler
+      sectionName: scheduler
   rules:
     - backendRefs:
         - name: nativelink-scheduler

native-cli/components/cilium.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"regexp"
 	"slices"
+	"strings"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
@@ -48,7 +49,7 @@ func (component *Cilium) Install(
 			"k8sServicePort": pulumi.String("6443"),
 
 			// Required for proper Cilium operation.
-			"kubeProxyReplacement": pulumi.String("strict"),
+			"kubeProxyReplacement": pulumi.Bool(true),
 
 			// Use the Gateway API instead of the older Ingress resource.
 			"gatewayAPI": pulumi.Map{"enabled": pulumi.Bool(true)},
@@ -103,6 +104,10 @@ func l2Announcements(
 
 			OtherFields: map[string]interface{}{
 				"spec": pulumi.Map{
+					"interfaces": pulumi.StringArray{
+						pulumi.String("^eth[0-9]+"),
+						pulumi.String("^enp[0-9]+"),
+					},
 					"externalIPs":     pulumi.Bool(true),
 					"loadBalancerIPs": pulumi.Bool(true),
 				},
@@ -117,8 +122,8 @@ func l2Announcements(
 	return []pulumi.Resource{l2Announcements}, nil
 }
 
-// kindCIDRs returns the container id range of the kind network.
-func kindCIDRs() (string, error) {
+// kindIPv4Subnet gets the IPv4 subnet from `docker network inspect kind`.
+func kindIPv4Subnet() (string, error) {
 	dockerCtx := context.Background()
 
 	cli, err := client.NewClientWithOpts(
@@ -136,15 +141,21 @@ func kindCIDRs() (string, error) {
 
 	for _, network := range networks {
 		if network.Name == "kind" {
-			if len(network.IPAM.Config) > 0 {
-				kindNetCIDR := network.IPAM.Config[0].Subnet
-
-				return kindNetCIDR, nil
+			for _, config := range network.IPAM.Config {
+				if strings.Contains(config.Subnet, ":") {
+					// Ignore IPv6 subnets.
+					continue
+				}
+
+				if strings.Contains(config.Subnet, ".") {
+					// The IPv4 subnet.
+					return config.Subnet, nil
+				}
 			}
 		}
 	}
 
-	return "", fmt.Errorf("%w: %s", errPulumi, "no kind network found")
+	return "", fmt.Errorf("%w: %s", errPulumi, "no kind IPv4 subnet found")
 }
 
 // defaultPool creates a CiliumLoadBalancerIPPool which allocates IPs on the
@@ -154,7 +165,7 @@ func defaultPool(
 	ctx *pulumi.Context,
 	ciliumResources []pulumi.Resource,
 ) ([]pulumi.Resource, error) {
-	kindNetCIDR, err := kindCIDRs()
+	kindNetCIDR, err := kindIPv4Subnet()
 	if err != nil {
 		return nil, fmt.Errorf("%w: %w", errPulumi, err)
 	}

native-cli/default.nix

@@ -3,7 +3,7 @@ pkgs.buildGoModule {
   pname = "native-cli";
   version = "0.3.0";
   src = ./.;
-  vendorHash = "sha256-HL407aegfvZ8UcziWNgmAxPveHXYf4KcBTolYGVBd4w=";
+  vendorHash = "sha256-yekdKWG1DdMr8/BzzGrcO0hkIjSNnV80LoEWZcZ1khQ=";
   buildInputs = [pkgs.makeWrapper];
   installPhase = ''
     runHook preInstall

native-cli/go.mod

@@ -3,11 +3,11 @@ module github.com/TraceMachina/nativelink/native-cli
 go 1.22.1
 
 require (
-	github.com/docker/docker v26.0.1+incompatible
+	github.com/docker/docker v26.0.2+incompatible
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1
-	github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.10.0
-	github.com/pulumi/pulumi/sdk/v3 v3.113.0
+	github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.11.0
+	github.com/pulumi/pulumi/sdk/v3 v3.113.2
 	github.com/spf13/cobra v1.8.0
 	sigs.k8s.io/kind v0.22.0
 )
@@ -104,7 +104,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.25.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/crypto v0.22.0 // indirect
-	golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect
+	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect

native-cli/go.sum

@@ -62,8 +62,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
 github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
-github.com/docker/docker v26.0.1+incompatible h1:t39Hm6lpXuXtgkF0dm1t9a5HkbUfdGy6XbWexmGr+hA=
-github.com/docker/docker v26.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v26.0.2+incompatible h1:yGVmKUFGgcxA6PXWAokO0sQL22BrQ67cgVjko8tGdXE=
+github.com/docker/docker v26.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -215,10 +215,10 @@ github.com/pulumi/esc v0.8.3 h1:myeDL6dD/mz34zZjCL8s7d/tWHBJYxfMxDCL11MHoqc=
 github.com/pulumi/esc v0.8.3/go.mod h1:v5VAPxYDa9DRwvubbzKt4ZYf5y0esWC2ccSp/AT923I=
 github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1 h1:plWLn9O6u80Vr37LoCsckyobBfcrdTU9cERor72QjqA=
 github.com/pulumi/pulumi-docker/sdk/v3 v3.6.1/go.mod h1:N4Yu4c49QErfucPt9Y/fGmpTryRqc0VfhyKHsGR9/g8=
-github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.10.0 h1:xHEFQ/k2fzFp3TADpE/US28Ri4WZfzEAcT99fiDZ1+U=
-github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.10.0/go.mod h1:9SKR5gTWY4FP9XnSNWd+HSeQt9lffrNCe+zbKvezI/o=
-github.com/pulumi/pulumi/sdk/v3 v3.113.0 h1:CIlmxJZdjxpPPoFe/rrP1dWTwh3CB7ahs/dA6SHcbuE=
-github.com/pulumi/pulumi/sdk/v3 v3.113.0/go.mod h1:JWSzKBoHd8rlncC1DhXLf7YdV+Bk/Qf+hSZOOQh0WwQ=
+github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.11.0 h1:Y/Zat+RiwDoYfttzzmUc8eoB4kRwAf4zX2uoKJ7eKys=
+github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.11.0/go.mod h1:9SKR5gTWY4FP9XnSNWd+HSeQt9lffrNCe+zbKvezI/o=
+github.com/pulumi/pulumi/sdk/v3 v3.113.2 h1:uieqCVqUqg1PNZ255JPCeNy3XZNs1dMAyIozP4acy4s=
+github.com/pulumi/pulumi/sdk/v3 v3.113.2/go.mod h1:JWSzKBoHd8rlncC1DhXLf7YdV+Bk/Qf+hSZOOQh0WwQ=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
@@ -295,8 +295,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
 golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
-golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc=
-golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
+golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
+golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=

native-cli/programs/local.go

@@ -25,7 +25,7 @@ func ProgramForLocalCluster(ctx *pulumi.Context) error {
 	cilium, err := components.AddComponent(
 		ctx,
 		"cilium",
-		&components.Cilium{Version: "1.15.3"},
+		&components.Cilium{Version: "1.16.0-pre.1"},
 	)
 	if err != nil {
 		log.Println(err)