From be2a2ec3cdaa39f47f0122a35255144f191632b3 Mon Sep 17 00:00:00 2001 From: jayvyas Date: Thu, 1 Oct 2015 22:59:00 -0400 Subject: [PATCH] NodePort apiserver option for exposing KubernetesMasterService NodePort on startup. --- cmd/kube-apiserver/app/server.go | 70 +++++++++++++++++-------------- hack/verify-flags/known-flags.txt | 1 + pkg/master/controller.go | 35 ++++++++++++---- pkg/master/controller_test.go | 2 +- pkg/master/master.go | 15 ++++--- 5 files changed, 78 insertions(+), 45 deletions(-) diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index a9a580a299a0b..da83674be81c0 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -114,6 +114,7 @@ type APIServer struct { SSHUser string SSHKeyfile string MaxConnectionBytesPerSec int64 + KubernetesServiceNodePort int } // NewAPIServer creates a new APIServer object with default parameters @@ -242,6 +243,8 @@ func (s *APIServer) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.KubeletConfig.CertFile, "kubelet-client-certificate", s.KubeletConfig.CertFile, "Path to a client cert file for TLS.") fs.StringVar(&s.KubeletConfig.KeyFile, "kubelet-client-key", s.KubeletConfig.KeyFile, "Path to a client key file for TLS.") fs.StringVar(&s.KubeletConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.CAFile, "Path to a cert. file for the certificate authority.") + //See #14282 for details on how to test/try this option out. TODO remove this comment once this option is tested in CI. + fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", 0, "If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be of type NodePort, using this as the value of the port. If zero, the Kubernetes master service will be of type ClusterIP.") } // TODO: Longer term we should read this from some config store, rather than a flag. @@ -353,6 +356,10 @@ func (s *APIServer) Run(_ []string) error { glog.Fatalf("specify either --etcd-servers or --etcd-config") } + if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) { + glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort)) + } + capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: s.AllowPrivileged, // TODO(vmarmol): Implement support for HostNetworkSources. @@ -508,37 +515,38 @@ func (s *APIServer) Run(_ []string) error { } } config := &master.Config{ - StorageDestinations: storageDestinations, - StorageVersions: storageVersions, - EventTTL: s.EventTTL, - KubeletClient: kubeletClient, - ServiceClusterIPRange: &n, - EnableCoreControllers: true, - EnableLogsSupport: s.EnableLogsSupport, - EnableUISupport: true, - EnableSwaggerSupport: true, - EnableProfiling: s.EnableProfiling, - EnableWatchCache: s.EnableWatchCache, - EnableIndex: true, - APIPrefix: s.APIPrefix, - APIGroupPrefix: s.APIGroupPrefix, - CorsAllowedOriginList: s.CorsAllowedOriginList, - ReadWritePort: s.SecurePort, - PublicAddress: s.AdvertiseAddress, - Authenticator: authenticator, - SupportsBasicAuth: len(s.BasicAuthFile) > 0, - Authorizer: authorizer, - AdmissionControl: admissionController, - DisableV1: disableV1, - EnableExp: enableExp, - MasterServiceNamespace: s.MasterServiceNamespace, - ClusterName: s.ClusterName, - ExternalHost: s.ExternalHost, - MinRequestTimeout: s.MinRequestTimeout, - SSHUser: s.SSHUser, - SSHKeyfile: s.SSHKeyfile, - InstallSSHKey: installSSH, - ServiceNodePortRange: s.ServiceNodePortRange, + StorageDestinations: storageDestinations, + StorageVersions: storageVersions, + EventTTL: s.EventTTL, + KubeletClient: kubeletClient, + ServiceClusterIPRange: &n, + EnableCoreControllers: true, + EnableLogsSupport: s.EnableLogsSupport, + EnableUISupport: true, + EnableSwaggerSupport: true, + EnableProfiling: s.EnableProfiling, + EnableWatchCache: s.EnableWatchCache, + EnableIndex: true, + APIPrefix: s.APIPrefix, + APIGroupPrefix: s.APIGroupPrefix, + CorsAllowedOriginList: s.CorsAllowedOriginList, + ReadWritePort: s.SecurePort, + PublicAddress: s.AdvertiseAddress, + Authenticator: authenticator, + SupportsBasicAuth: len(s.BasicAuthFile) > 0, + Authorizer: authorizer, + AdmissionControl: admissionController, + DisableV1: disableV1, + EnableExp: enableExp, + MasterServiceNamespace: s.MasterServiceNamespace, + ClusterName: s.ClusterName, + ExternalHost: s.ExternalHost, + MinRequestTimeout: s.MinRequestTimeout, + SSHUser: s.SSHUser, + SSHKeyfile: s.SSHKeyfile, + InstallSSHKey: installSSH, + ServiceNodePortRange: s.ServiceNodePortRange, + KubernetesServiceNodePort: s.KubernetesServiceNodePort, } m := master.New(config) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 639915e166cfb..a98c16acd5d2b 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -17,6 +17,7 @@ api-servers api-server-port api-token api-version +kubernetes-service-node-port authorization-mode authorization-policy-file auth-path diff --git a/pkg/master/controller.go b/pkg/master/controller.go index b0c1704d1680a..41fd7cb25cd37 100644 --- a/pkg/master/controller.go +++ b/pkg/master/controller.go @@ -57,9 +57,10 @@ type Controller struct { PublicIP net.IP - ServiceIP net.IP - ServicePort int - PublicServicePort int + ServiceIP net.IP + ServicePort int + PublicServicePort int + KubernetesServiceNodePort int runner *util.Runner } @@ -110,7 +111,7 @@ func (c *Controller) UpdateKubernetesService() error { return err } if c.ServiceIP != nil { - if err := c.CreateMasterServiceIfNeeded("kubernetes", c.ServiceIP, c.ServicePort); err != nil { + if err := c.CreateMasterServiceIfNeeded("kubernetes", c.ServiceIP, c.ServicePort, c.KubernetesServiceNodePort); err != nil { return err } if err := c.SetEndpoints("kubernetes", c.PublicIP, c.PublicServicePort); err != nil { @@ -140,14 +141,33 @@ func (c *Controller) CreateNamespaceIfNeeded(ns string) error { return err } +// createPortAndServiceSpec creates an array of service ports. +// If the NodePort value is 0, just the servicePort is used, otherwise, a node port is exposed. +func createPortAndServiceSpec(servicePort int, nodePort int) ([]api.ServicePort, api.ServiceType) { + //Use the Cluster IP type for the service port if NodePort isn't provided. + //Otherwise, we will be binding the master service to a NodePort. + if nodePort <= 0 { + return []api.ServicePort{{Protocol: api.ProtocolTCP, + Port: servicePort, + TargetPort: util.NewIntOrStringFromInt(servicePort)}}, api.ServiceTypeClusterIP + } + return []api.ServicePort{{Protocol: api.ProtocolTCP, + Port: servicePort, + TargetPort: util.NewIntOrStringFromInt(servicePort), + NodePort: nodePort, + }}, api.ServiceTypeNodePort +} + // CreateMasterServiceIfNeeded will create the specified service if it // doesn't already exist. -func (c *Controller) CreateMasterServiceIfNeeded(serviceName string, serviceIP net.IP, servicePort int) error { +func (c *Controller) CreateMasterServiceIfNeeded(serviceName string, serviceIP net.IP, servicePort, nodePort int) error { ctx := api.NewDefaultContext() if _, err := c.ServiceRegistry.GetService(ctx, serviceName); err == nil { // The service already exists. return nil } + + ports, serviceType := createPortAndServiceSpec(servicePort, nodePort) svc := &api.Service{ ObjectMeta: api.ObjectMeta{ Name: serviceName, @@ -155,15 +175,14 @@ func (c *Controller) CreateMasterServiceIfNeeded(serviceName string, serviceIP n Labels: map[string]string{"provider": "kubernetes", "component": "apiserver"}, }, Spec: api.ServiceSpec{ - Ports: []api.ServicePort{{Port: servicePort, Protocol: api.ProtocolTCP, TargetPort: util.NewIntOrStringFromInt(servicePort)}}, + Ports: ports, // maintained by this code, not by the pod selector Selector: nil, ClusterIP: serviceIP.String(), SessionAffinity: api.ServiceAffinityNone, - Type: api.ServiceTypeClusterIP, + Type: serviceType, }, } - if err := rest.BeforeCreate(rest.Services, ctx, svc); err != nil { return err } diff --git a/pkg/master/controller_test.go b/pkg/master/controller_test.go index 4e5497d7a37da..e7a4acf8483c1 100644 --- a/pkg/master/controller_test.go +++ b/pkg/master/controller_test.go @@ -259,7 +259,7 @@ func TestSetEndpoints(t *testing.T) { } if test.expectUpdate != nil { if len(registry.Updates) != 1 { - t.Errorf("case %q: unexpected updates: %v", test.testName, registry.Updates) + t.Errorf("case %q: unexpected updates: (%v). Expected exactly 1 change. ", test.testName, registry.Updates) } else if e, a := test.expectUpdate, ®istry.Updates[0]; !reflect.DeepEqual(e, a) { t.Errorf("case %q: expected update:\n%#v\ngot:\n%#v\n", test.testName, e, a) } diff --git a/pkg/master/master.go b/pkg/master/master.go index 21d2e18a0882c..619436db2ee72 100644 --- a/pkg/master/master.go +++ b/pkg/master/master.go @@ -244,6 +244,8 @@ type Config struct { SSHUser string SSHKeyfile string InstallSSHKey InstallSSHKey + + KubernetesServiceNodePort int } type InstallSSHKey func(user string, data []byte) error @@ -317,7 +319,8 @@ type Master struct { // map from api path to storage for those objects thirdPartyResources map[string]*thirdpartyresourcedataetcd.REST // protects the map - thirdPartyResourcesLock sync.RWMutex + thirdPartyResourcesLock sync.RWMutex + KubernetesServiceNodePort int } // NewEtcdStorage returns a storage.Interface for the provided arguments or an error if the version @@ -450,7 +453,8 @@ func New(c *Config) *Master { // TODO: serviceReadWritePort should be passed in as an argument, it may not always be 443 serviceReadWritePort: 443, - installSSHKey: c.InstallSSHKey, + installSSHKey: c.InstallSSHKey, + KubernetesServiceNodePort: c.KubernetesServiceNodePort, } var handlerContainer *restful.Container @@ -781,9 +785,10 @@ func (m *Master) NewBootstrapController() *Controller { PublicIP: m.clusterIP, - ServiceIP: m.serviceReadWriteIP, - ServicePort: m.serviceReadWritePort, - PublicServicePort: m.publicReadWritePort, + ServiceIP: m.serviceReadWriteIP, + ServicePort: m.serviceReadWritePort, + PublicServicePort: m.publicReadWritePort, + KubernetesServiceNodePort: m.KubernetesServiceNodePort, } }