support run spoke agent outside of managed cluster

Signed-off-by: zhujian <jiazhu@redhat.com>

pkg/clientcert/cert_controller.go

@@ -166,7 +166,7 @@ func (c *clientCertificateController) sync(ctx context.Context, syncCtx factory.
 
 	// reconcile pending csr if exists
 	if len(c.csrName) > 0 {
-		newSecretConfig, err := c.syncCSR(secret)
+		newSecretConfig, err := c.syncCSR()
 		if err != nil {
 			c.reset()
 			return err
@@ -231,7 +231,7 @@ func (c *clientCertificateController) sync(ctx context.Context, syncCtx factory.
 	return nil
 }
 
-func (c *clientCertificateController) syncCSR(secret *corev1.Secret) (map[string][]byte, error) {
+func (c *clientCertificateController) syncCSR() (map[string][]byte, error) {
 	// skip if there is no ongoing csr
 	if len(c.csrName) == 0 {
 		return nil, fmt.Errorf("no ongoing csr")

pkg/spoke/addon/configuration.go

@@ -49,7 +49,7 @@ func (c *registrationConfig) x509Subject(clusterName, agentName string) *pkix.Na
 }
 
 // getAddOnInstallationNamespace returns addon installation namespace from addon spec.
-// If the installation namespace is not specficed on addon spec, the addon defaul
+// If the installation namespace is not specified on addon spec, the addon default
 // installation namespace open-cluster-management-agent-addon will be returned.
 func getAddOnInstallationNamespace(addOn *addonv1alpha1.ManagedClusterAddOn) string {
 	installationNamespace := addOn.Spec.InstallNamespace

pkg/spoke/addon/registration_controller.go

@@ -132,7 +132,7 @@ func (c *addOnRegistrationController) syncAddOn(ctx context.Context, syncCtx fac
 		return err
 	}
 
-	// stop registraton for the stale registration configs
+	// stop registration for the stale registration configs
 	errs := []error{}
 	for hash, cachedConfig := range cachedConfigs {
 		if _, ok := configs[hash]; ok {

pkg/spoke/managedcluster/registration.go

@@ -30,9 +30,9 @@ func NewClientCertForHubController(
 	clientCertSecretName string,
 	kubeconfigData []byte,
 	spokeCoreClient corev1client.CoreV1Interface,
+	spokeSecretInformer corev1informers.SecretInformer,
 	hubCSRClient csrclient.CertificateSigningRequestInterface,
 	hubCSRInformer certificatesinformers.CertificateSigningRequestInformer,
-	spokeSecretInformer corev1informers.SecretInformer,
 	recorder events.Recorder,
 	controllerName string,
 ) factory.Controller {

pkg/spoke/spokeagent.go

@@ -60,6 +60,7 @@ type SpokeAgentOptions struct {
 	SpokeExternalServerURLs  []string
 	ClusterHealthCheckPeriod time.Duration
 	MaxCustomClusterClaims   int
+	SpokeKubeconfig          string
 }
 
 // NewSpokeAgentOptions returns a SpokeAgentOptions
@@ -74,9 +75,10 @@ func NewSpokeAgentOptions() *SpokeAgentOptions {
 
 // RunSpokeAgent starts the controllers on spoke agent to register to the hub.
 //
-// The spoke agent uses three kubeconfigs for different concerns:
-// - The 'spoke' kubeconfig: used to communicate with the spoke cluster where
-//   the agent is running.
+// The spoke agent uses four kubeconfigs for different concerns:
+// - The 'agent' kubeconfig: used to communicate with the cluster where the agent is running.
+// - The 'spoke' kubeconfig: used to communicate with the spoke/managed cluster which will
+//   be registered to the hub.
 // - The 'bootstrap' kubeconfig: used to communicate with the hub in order to
 //   submit a CertificateSigningRequest, begin the join flow with the hub, and
 //   to write the 'hub' kubeconfig.
@@ -94,13 +96,26 @@ func NewSpokeAgentOptions() *SpokeAgentOptions {
 // create a valid hub kubeconfig. Once the hub kubeconfig is valid, the
 // temporary controller is stopped and the main controllers are started.
 func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext *controllercmd.ControllerContext) error {
-	// create kube client
-	spokeKubeClient, err := kubernetes.NewForConfig(controllerContext.KubeConfig)
+	// create agent kube client
+	agentKubeClient, err := kubernetes.NewForConfig(controllerContext.KubeConfig)
 	if err != nil {
 		return err
 	}
 
-	if err := o.Complete(spokeKubeClient.CoreV1(), ctx, controllerContext.EventRecorder); err != nil {
+	// load spoke client config and create spoke clients,
+	// the registration agent may not running in the spoke/managed cluster.
+	spokeClientConfig, err := o.spokeKubeConfig(controllerContext)
+	if err != nil {
+		return err
+	}
+
+	spokeKubeClient, err := kubernetes.NewForConfig(spokeClientConfig)
+	if err != nil {
+		return err
+	}
+
+	// the hub kubeconfig secret stored in the cluster where the agent pod runs
+	if err := o.Complete(agentKubeClient.CoreV1(), ctx, controllerContext.EventRecorder); err != nil {
 		klog.Fatal(err)
 	}
 
@@ -112,14 +127,16 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext
 
 	// create shared informer factory for spoke cluster
 	spokeKubeInformerFactory := informers.NewSharedInformerFactory(spokeKubeClient, 10*time.Minute)
-	namespacedSpokeKubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(spokeKubeClient, 10*time.Minute, informers.WithNamespace(o.ComponentNamespace))
 
 	// get spoke cluster CA bundle
-	spokeClusterCABundle, err := o.getSpokeClusterCABundle(controllerContext.KubeConfig)
+	spokeClusterCABundle, err := o.getSpokeClusterCABundle(spokeClientConfig)
 	if err != nil {
 		return err
 	}
 
+	// create a shared informer factory with specific namespace for the agent cluster.
+	namespacedAgentKubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(agentKubeClient, 10*time.Minute, informers.WithNamespace(o.ComponentNamespace))
+
 	// load bootstrap client config and create bootstrap clients
 	bootstrapClientConfig, err := clientcmd.BuildConfigFromFlags("", o.BootstrapKubeconfig)
 	if err != nil {
@@ -145,8 +162,9 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext
 
 	hubKubeconfigSecretController := managedcluster.NewHubKubeconfigSecretController(
 		o.HubKubeconfigDir, o.ComponentNamespace, o.HubKubeconfigSecret,
-		spokeKubeClient.CoreV1(),
-		namespacedSpokeKubeInformerFactory.Core().V1().Secrets(),
+		// the hub kubeconfig secret stored in the cluster where the agent pod runs
+		agentKubeClient.CoreV1(),
+		namespacedAgentKubeInformerFactory.Core().V1().Secrets(),
 		controllerContext.EventRecorder,
 	)
 	go hubKubeconfigSecretController.Run(ctx, 1)
@@ -177,18 +195,20 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext
 		clientCertForHubController := managedcluster.NewClientCertForHubController(
 			o.ClusterName, o.AgentName, o.ComponentNamespace, o.HubKubeconfigSecret,
 			kubeconfigData,
-			spokeKubeClient.CoreV1(),
+			// store the secret in the cluster where the agent pod runs
+			agentKubeClient.CoreV1(),
+			namespacedAgentKubeInformerFactory.Core().V1().Secrets(),
 			bootstrapKubeClient.CertificatesV1().CertificateSigningRequests(),
 			bootstrapInformerFactory.Certificates().V1().CertificateSigningRequests(),
-			namespacedSpokeKubeInformerFactory.Core().V1().Secrets(),
+
 			controllerContext.EventRecorder,
 			controllerName,
 		)
 
 		bootstrapCtx, stopBootstrap := context.WithCancel(ctx)
 
 		go bootstrapInformerFactory.Start(bootstrapCtx.Done())
-		go namespacedSpokeKubeInformerFactory.Start(bootstrapCtx.Done())
+		go namespacedAgentKubeInformerFactory.Start(bootstrapCtx.Done())
 
 		go clientCertForHubController.Run(bootstrapCtx, 1)
 
@@ -251,10 +271,12 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext
 	clientCertForHubController := managedcluster.NewClientCertForHubController(
 		o.ClusterName, o.AgentName, o.ComponentNamespace, o.HubKubeconfigSecret,
 		kubeconfigData,
-		spokeKubeClient.CoreV1(),
+		// store the secret in the cluster where the agent pod runs
+		agentKubeClient.CoreV1(),
+		namespacedAgentKubeInformerFactory.Core().V1().Secrets(),
 		hubKubeClient.CertificatesV1().CertificateSigningRequests(),
 		hubKubeInformerFactory.Certificates().V1().CertificateSigningRequests(),
-		namespacedSpokeKubeInformerFactory.Core().V1().Secrets(),
+
 		controllerContext.EventRecorder,
 		controllerName,
 	)
@@ -285,7 +307,7 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext
 		o.ClusterHealthCheckPeriod,
 		controllerContext.EventRecorder,
 	)
-	spokeClusterClient, err := clusterv1client.NewForConfig(controllerContext.KubeConfig)
+	spokeClusterClient, err := clusterv1client.NewForConfig(spokeClientConfig)
 	if err != nil {
 		return err
 	}
@@ -321,6 +343,9 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext
 			o.ClusterName,
 			o.AgentName,
 			kubeconfigData,
+			// TODO(zhujian7): By now, we only support all addon agents running on the managed cluster.
+			// In the future we need to maintainer the hub cluster kubeconfig secret on the **agent**
+			// cluster when there is an appropriate way to deploy addon agents on the agent cluster.
 			spokeKubeClient,
 			hubKubeInformerFactory.Certificates().V1().CertificateSigningRequests(),
 			addOnInformerFactory.Addon().V1alpha1().ManagedClusterAddOns(),
@@ -332,7 +357,7 @@ func (o *SpokeAgentOptions) RunSpokeAgent(ctx context.Context, controllerContext
 	go hubKubeInformerFactory.Start(ctx.Done())
 	go hubClusterInformerFactory.Start(ctx.Done())
 	go spokeKubeInformerFactory.Start(ctx.Done())
-	go namespacedSpokeKubeInformerFactory.Start(ctx.Done())
+	go namespacedAgentKubeInformerFactory.Start(ctx.Done())
 	go spokeClusterInformerFactory.Start(ctx.Done())
 	go addOnInformerFactory.Start(ctx.Done())
 
@@ -363,6 +388,8 @@ func (o *SpokeAgentOptions) AddFlags(fs *pflag.FlagSet) {
 		"The name of secret in component namespace storing kubeconfig for hub.")
 	fs.StringVar(&o.HubKubeconfigDir, "hub-kubeconfig-dir", o.HubKubeconfigDir,
 		"The mount path of hub-kubeconfig-secret in the container.")
+	fs.StringVar(&o.SpokeKubeconfig, "spoke-kubeconfig", o.SpokeKubeconfig,
+		"The path of the kubeconfig file for managed/spoke cluster. If this is not set, will use '--kubeconfig' to build client to connect to the managed cluster.")
 	fs.StringArrayVar(&o.SpokeExternalServerURLs, "spoke-external-server-urls", o.SpokeExternalServerURLs,
 		"A list of reachable spoke cluster api server URLs for hub cluster.")
 	fs.DurationVar(&o.ClusterHealthCheckPeriod, "cluster-healthcheck-period", o.ClusterHealthCheckPeriod,
@@ -559,3 +586,16 @@ func (o *SpokeAgentOptions) getSpokeClusterCABundle(kubeConfig *rest.Config) ([]
 	}
 	return data, nil
 }
+
+// spokeKubeConfig builds kubeconfig for the spoke/managed cluster
+func (o *SpokeAgentOptions) spokeKubeConfig(controllerContext *controllercmd.ControllerContext) (*rest.Config, error) {
+	if o.SpokeKubeconfig == "" {
+		return controllerContext.KubeConfig, nil
+	}
+
+	config, err := clientcmd.BuildConfigFromFlags("" /* leave masterurl as empty */, o.SpokeKubeconfig)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load spoke kubeconfig from file %q: %w", o.SpokeKubeconfig, err)
+	}
+	return config, nil
+}