fix(auth,platform): add authorization of rbac in inspectors

cmd/tke-business-api/app/config/config.go

@@ -44,6 +44,7 @@ import (
 	"tkestack.io/tke/pkg/auth/filter"
 	"tkestack.io/tke/pkg/business/apiserver"
 	controllerconfig "tkestack.io/tke/pkg/controller/config"
+	"tkestack.io/tke/pkg/util/log"
 )
 
 const (
@@ -131,7 +132,11 @@ func CreateConfigFromOptions(serverName string, opts *options.Options) (*Config,
 	if err != nil {
 		return nil, err
 	}
-	clusterInspector := filter.NewClusterInspector(platformClient.PlatformV1(), opts.Authentication.PrivilegedUsername)
+	clusterInspector, err := filter.NewClusterInspector(platformClient.PlatformV1(), opts.Authentication.PrivilegedUsername)
+	if err != nil {
+		log.Errorf("create clusterInspector failed: %+v", err)
+		return nil, err
+	}
 	genericAPIServerConfig.BuildHandlerChainFunc = handler.BuildHandlerChain(nil, nil, []filter.Inspector{clusterInspector})
 
 	cfg := &Config{

cmd/tke-platform-api/app/config/config.go

@@ -41,6 +41,7 @@ import (
 	"tkestack.io/tke/pkg/apiserver/util"
 	"tkestack.io/tke/pkg/auth/filter"
 	"tkestack.io/tke/pkg/platform/apiserver"
+	"tkestack.io/tke/pkg/util/log"
 )
 
 const (
@@ -104,7 +105,11 @@ func CreateConfigFromOptions(serverName string, opts *options.Options) (*Config,
 	if err != nil {
 		return nil, fmt.Errorf("failed to create real external clientset: %v", err)
 	}
-	clusterInspector := filter.NewClusterInspector(clientgoExternalClient.PlatformV1(), opts.Authentication.PrivilegedUsername)
+	clusterInspector, err := filter.NewClusterInspector(clientgoExternalClient.PlatformV1(), opts.Authentication.PrivilegedUsername)
+	if err != nil {
+		log.Errorf("create clusterInspector failed: %+v", err)
+		return nil, err
+	}
 	genericAPIServerConfig.BuildHandlerChainFunc = handler.BuildHandlerChain(nil, nil, []filter.Inspector{clusterInspector})
 	versionedInformers := versionedinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
 

pkg/registry/util/client.go → pkg/apiserver/util/signals.go

@@ -1,7 +1,7 @@
 /*
  * Tencent is pleased to support the open source community by making TKEStack available.
  *
- * Copyright (C) 2012-2020 Tencent. All Rights Reserved.
+ * Copyright (C) 2012-2021 Tencent. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use
  * this file except in compliance with the License. You may obtain a copy of the
@@ -18,20 +18,24 @@
 package util
 
 import (
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
+	"os"
+	"os/signal"
+	"syscall"
 )
 
-func BuildKubeClient() (*kubernetes.Clientset, error) {
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		return nil, err
-	}
+// SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned
+// which is closed on one of these signals. If a second signal is caught, the program
+// is terminated with exit code 1.
+func SetupSignalHandler() (stopCh <-chan struct{}) {
+	stop := make(chan struct{})
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-c
+		close(stop)
+		<-c
+		os.Exit(1) // second signal. Exit directly.
+	}()
 
-	clientset, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	return clientset, nil
+	return stop
 }

pkg/auth/filter/inspector.go

@@ -20,41 +20,125 @@ package filter
 import (
 	"context"
 	"fmt"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"net/http"
+	"strings"
+	"time"
+	"tkestack.io/tke/pkg/apiserver/util"
 
 	platformv1 "tkestack.io/tke/api/client/clientset/versioned/typed/platform/v1"
 	"tkestack.io/tke/pkg/apiserver/authentication"
+	"tkestack.io/tke/pkg/util/apiclient"
 	"tkestack.io/tke/pkg/util/log"
 
+	rbacv1 "k8s.io/api/rbac/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	genericfilters "k8s.io/apiserver/pkg/endpoints/filters"
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	rbaclisters "k8s.io/client-go/listers/rbac/v1"
+	"k8s.io/client-go/tools/cache"
 )
 
 type Inspector interface {
 	Inspect(handler http.Handler, c *genericapiserver.Config) http.Handler
 }
 
 type clusterInspector struct {
+	k8sClient          kubernetes.Interface
+	crbLister          rbaclisters.ClusterRoleBindingLister
+	crLister           rbaclisters.ClusterRoleLister
 	platformClient     platformv1.PlatformV1Interface
 	privilegedUsername string
 }
 
-func NewClusterInspector(platformClient platformv1.PlatformV1Interface, privilegedUsername string) Inspector {
+func NewClusterInspector(platformClient platformv1.PlatformV1Interface, privilegedUsername string) (Inspector, error) {
+	k8sClient, err := apiclient.BuildKubeClient()
+	if err != nil {
+		return nil, err
+	}
+	informerFactory := informers.NewSharedInformerFactory(k8sClient, time.Minute)
+	clusterRoleBindingInformer := informerFactory.Rbac().V1().ClusterRoleBindings()
+	clusterRoleBindingLister := clusterRoleBindingInformer.Lister()
+	clusterRoleInformer := informerFactory.Rbac().V1().ClusterRoles()
+	clusterRoleLister := clusterRoleInformer.Lister()
+	stopCh := util.SetupSignalHandler()
+	informerFactory.Start(stopCh)
+	if ok := cache.WaitForCacheSync(stopCh, clusterRoleBindingInformer.Informer().HasSynced,
+		clusterRoleInformer.Informer().HasSynced); !ok {
+		return nil, fmt.Errorf("failed to wait for namespaces caches to sync")
+	}
 	return &clusterInspector{
+		k8sClient:          k8sClient,
+		crbLister:          clusterRoleBindingLister,
+		crLister:           clusterRoleLister,
 		platformClient:     platformClient,
 		privilegedUsername: privilegedUsername,
+	}, nil
+}
+
+func isClusterAdmin(rules []rbacv1.PolicyRule) bool {
+	if len(rules) != 2 {
+		return false
 	}
+	isAdmin := true
+	for _, rul := range rules {
+		if len(rul.APIGroups) == 1 && rul.APIGroups[0] == "*" &&
+			len(rul.Resources) == 1 && rul.Resources[0] == "*" &&
+			len(rul.Verbs) == 1 && rul.Verbs[0] == "*" {
+			continue
+		}
+		if len(rul.NonResourceURLs) == 1 && rul.NonResourceURLs[0] == "*" &&
+			len(rul.Verbs) == 1 && rul.Verbs[0] == "*" {
+			continue
+		}
+		isAdmin = false
+		break
+	}
+	return isAdmin
+}
+
+func (i *clusterInspector) needInspect(ctx context.Context, privilegedUsername string) bool {
+	username, tenantID := authentication.UsernameAndTenantID(ctx)
+	if (username == privilegedUsername || username == "system:apiserver") && tenantID == "" {
+		return false
+	}
+
+	clusterRoleBindings, err := i.crbLister.List(labels.Everything())
+	if err != nil {
+		log.Errorf("query clusterRoleBindings failed: %+v", err)
+		return true
+	}
+	username = strings.TrimPrefix(username, "system:serviceaccount:kube-system:")
+	for _, crb := range clusterRoleBindings {
+		for _, sub := range crb.Subjects {
+			if sub.Name == username && sub.Namespace == "kube-system" {
+				cr, err := i.crLister.Get(crb.RoleRef.Name)
+				if err != nil {
+					log.Errorf("query clusterRole: %+v failed: %+v", crb.RoleRef.Name, err)
+					continue
+				}
+				if len(cr.Rules) != 2 {
+					continue
+				}
+				log.Debugf("needInspect: username: %+v clusterRole: %+v->%v", username, cr.Name, cr.Rules)
+				if isClusterAdmin(cr.Rules) {
+					return false
+				}
+			}
+		}
+	}
+	return true
 }
 
 func (i *clusterInspector) Inspect(handler http.Handler, c *genericapiserver.Config) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		ctx := req.Context()
-		username, tenantID := authentication.UsernameAndTenantID(ctx)
-		if (username == i.privilegedUsername || username == "system:apiserver") && tenantID == "" {
+		if !i.needInspect(ctx, i.privilegedUsername) {
 			handler.ServeHTTP(w, req)
 			return
 		}
@@ -77,7 +161,8 @@ func (i *clusterInspector) Inspect(handler http.Handler, c *genericapiserver.Con
 				"invalid request: too many clusterName in request")
 			return
 		}
-		log.Infof("WithTKEAuthorization clusterNames: %+v, username: %+v, tenant: %+v, "+
+		username, tenantID := authentication.UsernameAndTenantID(ctx)
+		log.Infof(" clusterNames: %+v, username: %+v, tenant: %+v, "+
 			"action: %+v, resource: %+v, name: %+v",
 			clusterNames, username, tenantID, tkeAttributes.GetVerb(),
 			tkeAttributes.GetResource(), tkeAttributes.GetName())

pkg/registry/util/coredns.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"sync"
 
+	"tkestack.io/tke/pkg/util/apiclient"
 	"tkestack.io/tke/pkg/util/log"
 
 	"github.com/caddyserver/caddy/caddyfile"
@@ -55,7 +56,7 @@ type CoreDNS struct {
 }
 
 func NewCoreDNS() (*CoreDNS, error) {
-	kubeClient, err := BuildKubeClient()
+	kubeClient, err := apiclient.BuildKubeClient()
 	if err != nil {
 		return nil, err
 	}

pkg/util/apiclient/client.go

@@ -37,6 +37,20 @@ import (
 	toolswatch "k8s.io/client-go/tools/watch"
 )
 
+func BuildKubeClient() (*kubernetes.Clientset, error) {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	return clientset, nil
+}
+
 // GetClientset return clientset
 func GetClientset(masterEndpoint string, token string, caCert []byte) (*kubernetes.Clientset, error) {
 	restConfig := &rest.Config{