✨ support proxy between hub cluster and managed cluster

Signed-off-by: Yang Le <yangle@redhat.com>

pkg/operator/operators/klusterlet/controllers/bootstrapcontroller/bootstrapcontroller.go

@@ -154,6 +154,7 @@ func (k *bootstrapController) sync(ctx context.Context, controllerContext factor
 	}
 
 	if bootstrapKubeconfig.Server != hubKubeconfig.Server ||
+		bootstrapKubeconfig.ProxyURL != hubKubeconfig.ProxyURL ||
 		!bytes.Equal(bootstrapKubeconfig.CertificateAuthorityData, hubKubeconfig.CertificateAuthorityData) {
 		// the bootstrap kubeconfig secret is changed, reload the klusterlet agents
 		reloadReason := fmt.Sprintf("the bootstrap secret %s/%s is changed", agentNamespace, helpers.BootstrapHubKubeConfig)

pkg/operator/operators/klusterlet/controllers/bootstrapcontroller/bootstrapcontroller_test.go

@@ -57,7 +57,7 @@ func TestSync(t *testing.T) {
 			name:     "client certificate expired",
 			queueKey: "test/test",
 			objects: []runtime.Object{
-				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443")),
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443", "")),
 				newHubKubeConfigSecret("test", time.Now().Add(-60*time.Second).UTC()),
 			},
 			expectedRebootstrapping: true,
@@ -70,7 +70,7 @@ func TestSync(t *testing.T) {
 		{
 			name:     "the bootstrap is not started",
 			queueKey: "test/test",
-			objects:  []runtime.Object{newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443"))},
+			objects:  []runtime.Object{newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443", ""))},
 			validateActions: func(t *testing.T, actions []clienttesting.Action) {
 				if len(actions) != 0 {
 					t.Errorf("expected no actions happens, but got %#v", actions)
@@ -81,7 +81,7 @@ func TestSync(t *testing.T) {
 			name:     "the bootstrap secret is not changed",
 			queueKey: "test/test",
 			objects: []runtime.Object{
-				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443")),
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443", "")),
 				newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
 			},
 			validateActions: func(t *testing.T, actions []clienttesting.Action) {
@@ -91,10 +91,24 @@ func TestSync(t *testing.T) {
 			},
 		},
 		{
-			name:     "the bootstrap secret is changed",
+			name:     "hub server url is changed",
 			queueKey: "test/test",
 			objects: []runtime.Object{
-				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443")),
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "")),
+				newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
+			},
+			expectedRebootstrapping: true,
+			validateActions: func(t *testing.T, actions []clienttesting.Action) {
+				if len(actions) != 0 {
+					t.Errorf("expected no actions happens, but got %#v", actions)
+				}
+			},
+		},
+		{
+			name:     "proxy url is changed",
+			queueKey: "test/test",
+			objects: []runtime.Object{
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "https://10.0.118.10:3129")),
 				newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
 			},
 			expectedRebootstrapping: true,
@@ -109,7 +123,7 @@ func TestSync(t *testing.T) {
 			queueKey:            "test/test",
 			initRebootstrapping: true,
 			objects: []runtime.Object{
-				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443")),
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "")),
 				newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
 				newDeploymentWithAvailableReplicas("test-registration-agent", "test", 1),
 			},
@@ -123,7 +137,7 @@ func TestSync(t *testing.T) {
 			queueKey:            "test/test",
 			initRebootstrapping: true,
 			objects: []runtime.Object{
-				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443")),
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "")),
 				newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
 				newDeployment("test-registration-agent", "test"),
 			},
@@ -284,11 +298,12 @@ func newSecret(name, namespace string, kubeConfig []byte) *corev1.Secret {
 	return secret
 }
 
-func newKubeConfig(host string) []byte {
+func newKubeConfig(host, proxyURL string) []byte {
 	configData, _ := runtime.Encode(clientcmdlatest.Codec, &clientcmdapi.Config{
 		Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": {
 			Server:                host,
 			InsecureSkipTLSVerify: true,
+			ProxyURL:              proxyURL,
 		}},
 		Contexts: map[string]*clientcmdapi.Context{"default-context": {
 			Cluster: "default-cluster",
@@ -345,7 +360,7 @@ func newHubKubeConfigSecret(namespace string, notAfter time.Time) *corev1.Secret
 			Namespace: namespace,
 		},
 		Data: map[string][]byte{
-			"kubeconfig": newKubeConfig("https://10.0.118.47:6443"),
+			"kubeconfig": newKubeConfig("https://10.0.118.47:6443", ""),
 			"tls.crt": pem.EncodeToMemory(&pem.Block{
 				Type:  certutil.CertificateBlockType,
 				Bytes: cert.Raw,

pkg/registration/clientcert/certificate.go

@@ -147,13 +147,19 @@ func getCertValidityPeriod(secret *corev1.Secret) (*time.Time, *time.Time, error
 
 // BuildKubeconfig builds a kubeconfig based on a rest config template with a cert/key pair
 func BuildKubeconfig(clientConfig *restclient.Config, certPath, keyPath string) clientcmdapi.Config {
+	return BuildKubeconfigWithProxyURL(clientConfig, "", certPath, keyPath)
+}
+
+// BuildKubeconfigWithProxyURL builds a kubeconfig based on a rest config as template with proxy url and a cert/key pair
+func BuildKubeconfigWithProxyURL(clientConfig *restclient.Config, proxyURL, certPath, keyPath string) clientcmdapi.Config {
 	// Build kubeconfig.
 	kubeconfig := clientcmdapi.Config{
 		// Define a cluster stanza based on the bootstrap kubeconfig.
 		Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": {
 			Server:                   clientConfig.Host,
 			InsecureSkipTLSVerify:    false,
 			CertificateAuthorityData: clientConfig.CAData,
+			ProxyURL:                 proxyURL,
 		}},
 		// Define auth based on the obtained client cert.
 		AuthInfos: map[string]*clientcmdapi.AuthInfo{"default-auth": {

pkg/registration/spoke/spokeagent.go

@@ -213,7 +213,11 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context,
 			managementKubeClient, 10*time.Minute, informers.WithNamespace(o.agentOptions.ComponentNamespace))
 
 		// create a kubeconfig with references to the key/cert files in the same secret
-		kubeconfig := clientcert.BuildKubeconfig(bootstrapClientConfig, clientcert.TLSCertFile, clientcert.TLSKeyFile)
+		proxyURL, err := getProxyURLFromKubeconfig(o.registrationOption.BootstrapKubeconfig)
+		if err != nil {
+			return err
+		}
+		kubeconfig := clientcert.BuildKubeconfigWithProxyURL(bootstrapClientConfig, proxyURL, clientcert.TLSCertFile, clientcert.TLSKeyFile)
 		kubeconfigData, err := clientcmd.Write(kubeconfig)
 		if err != nil {
 			return err
@@ -303,7 +307,11 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context,
 	recorder.Event("HubClientConfigReady", "Client config for hub is ready.")
 
 	// create a kubeconfig with references to the key/cert files in the same secret
-	kubeconfig := clientcert.BuildKubeconfig(hubClientConfig, clientcert.TLSCertFile, clientcert.TLSKeyFile)
+	proxyURL, err := getProxyURLFromKubeconfig(o.agentOptions.HubKubeconfigFile)
+	if err != nil {
+		return err
+	}
+	kubeconfig := clientcert.BuildKubeconfigWithProxyURL(hubClientConfig, proxyURL, clientcert.TLSCertFile, clientcert.TLSKeyFile)
 	kubeconfigData, err := clientcmd.Write(kubeconfig)
 	if err != nil {
 		return err
@@ -465,3 +473,22 @@ func (o *SpokeAgentConfig) getSpokeClusterCABundle(kubeConfig *rest.Config) ([]b
 	}
 	return data, nil
 }
+
+func getProxyURLFromKubeconfig(filename string) (string, error) {
+	config, err := clientcmd.LoadFromFile(filename)
+	if err != nil {
+		return "", err
+	}
+
+	currentContext, ok := config.Contexts[config.CurrentContext]
+	if !ok {
+		return "", nil
+	}
+
+	cluster, ok := config.Clusters[currentContext.Cluster]
+	if !ok {
+		return "", nil
+	}
+
+	return cluster.ProxyURL, nil
+}

pkg/registration/spoke/spokeagent_test.go

@@ -9,9 +9,12 @@ import (
 	"time"
 
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 
 	commonoptions "open-cluster-management.io/ocm/pkg/common/options"
 	testingcommon "open-cluster-management.io/ocm/pkg/common/testing"
+	"open-cluster-management.io/ocm/pkg/registration/clientcert"
 	testinghelpers "open-cluster-management.io/ocm/pkg/registration/helpers/testing"
 )
 
@@ -231,3 +234,48 @@ func TestGetSpokeClusterCABundle(t *testing.T) {
 		})
 	}
 }
+
+func TestGetProxyURLFromKubeconfig(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "testvalidhubclientconfig")
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	kubeconfigWithoutProxy := clientcert.BuildKubeconfig(&rest.Config{}, "tls.crt", "tls.key")
+	kubeconfigWithProxy := clientcert.BuildKubeconfigWithProxyURL(&rest.Config{}, "https://127.0.0.1:3129", "tls.crt", "tls.key")
+
+	cases := []struct {
+		name             string
+		kubeconfig       clientcmdapi.Config
+		expectedProxyURL string
+	}{
+		{
+			name:             "without proxy url",
+			kubeconfig:       kubeconfigWithoutProxy,
+			expectedProxyURL: "",
+		},
+		{
+			name:             "with proxy url",
+			kubeconfig:       kubeconfigWithProxy,
+			expectedProxyURL: "https://127.0.0.1:3129",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			filename := path.Join(tempDir, "kubeconfig")
+			if err := clientcmd.WriteToFile(c.kubeconfig, filename); err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+
+			proxyURL, err := getProxyURLFromKubeconfig(filename)
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+
+			if c.expectedProxyURL != proxyURL {
+				t.Errorf("expect %s, but %s", c.expectedProxyURL, proxyURL)
+			}
+		})
+	}
+}