Move Bucekt DNS feature to sidecar (#1494)

Signed-off-by: Daniel Valdivia <18384552+dvaldivia@users.noreply.github.com>

examples/kustomization/sts-example/sample-clients/minio-sdk/go/main.go

@@ -89,7 +89,6 @@ func main() {
 		Secure:    tenantEndpointURL.Scheme == "https",
 		Transport: httpsTransport,
 	})
-
 	if err != nil {
 		log.Fatalf("Error initializing client: %v", err)
 		panic(1)

helm/operator/templates/minio.min.io_tenants.yaml

@@ -3845,11 +3845,6 @@ spec:
                 type: object
               requestAutoCert:
                 type: boolean
-              s3:
-                properties:
-                  bucketDNS:
-                    type: boolean
-                type: object
               serviceAccountName:
                 type: string
               serviceMetadata:

pkg/apis/minio.min.io/v2/helper.go

@@ -37,6 +37,8 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/minio/operator/pkg/common"
+
 	"github.com/miekg/dns"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -54,23 +56,10 @@ import (
 
 // Webhook API constants
 const (
-	WebhookAPIVersion       = "/webhook/v1"
-	WebhookDefaultPort      = "4222"
-	WebhookSecret           = "operator-webhook-secret"
-	WebhookOperatorUsername = "webhookUsername"
-	WebhookOperatorPassword = "webhookPassword"
-
-	// Webhook environment variable constants
-	WebhookMinIOArgs   = "MINIO_ARGS"
-	WebhookMinIOBucket = "MINIO_DNS_WEBHOOK_ENDPOINT"
-
 	MinIOServerURL          = "MINIO_SERVER_URL"
 	MinIODomain             = "MINIO_DOMAIN"
 	MinIOBrowserRedirectURL = "MINIO_BROWSER_REDIRECT_URL"
 
-	MinIORootUser     = "MINIO_ROOT_USER"
-	MinIORootPassword = "MINIO_ROOT_PASSWORD"
-
 	defaultPrometheusJWTExpiry = 100 * 365 * 24 * time.Hour
 )
 
@@ -87,16 +76,7 @@ func envGet(key, defaultValue string) string {
 
 // List of webhook APIs
 const (
-	WebhookAPIGetenv        = WebhookAPIVersion + "/getenv"
-	WebhookAPIBucketService = WebhookAPIVersion + "/bucketsrv"
-	WebhookAPIUpdate        = WebhookAPIVersion + "/update"
-	WebhookCRDConversaion   = WebhookAPIVersion + "/crd-conversion"
-)
-
-// STS API constants
-const (
-	STSDefaultPort = "4223"
-	STSEndpoint    = "/sts"
+	WebhookAPIUpdate = common.WebhookAPIVersion + "/update"
 )
 
 type hostsTemplateValues struct {
@@ -560,8 +540,7 @@ func (t *Tenant) KESServiceHost() string {
 
 // BucketDNS indicates if Bucket DNS feature is enabled.
 func (t *Tenant) BucketDNS() bool {
-	// we've deprecated .spec.s3 and will top working in future releases of operator
-	return (t.Spec.Features != nil && t.Spec.Features.BucketDNS) || (t.Spec.S3 != nil && t.Spec.S3.BucketDNS)
+	return (t.Spec.Features != nil && t.Spec.Features.BucketDNS)
 }
 
 // HasKESEnabled checks if kes configuration is provided by user

pkg/apis/minio.min.io/v2/types.go

@@ -72,16 +72,6 @@ type TenantDomains struct {
 	Console string `json:"console,omitempty"`
 }
 
-// S3Features (`s3`) - Object describing which MinIO features to enable/disable in the MinIO Tenant. +
-// *Deprecated in Operator v4.3.2* +
-type S3Features struct {
-	// *Optional* +
-	//
-	// Specify `true` to allow clients to access buckets using the DNS path `<bucket>.minio.default.svc.cluster.local`. Defaults to `false`.
-	//
-	BucketDNS bool `json:"bucketDNS,omitempty"`
-}
-
 // Features (`features`) - Object describing which MinIO features to enable/disable in the MinIO Tenant. +
 type Features struct {
 	// *Optional* +
@@ -251,12 +241,6 @@ type TenantSpec struct {
 	// +optional
 	Startup *corev1.Probe `json:"startup,omitempty"`
 
-	// *Optional* +
-	// *Deprecated in Operator v4.3.2* +
-	//
-	// S3 related features can be disabled or enabled such as `bucketDNS` etc.
-	S3 *S3Features `json:"s3,omitempty"`
-
 	// S3 related features can be disabled or enabled such as `bucketDNS` etc.
 	Features *Features `json:"features,omitempty"`
 	// *Optional* +

pkg/common/const.go

@@ -0,0 +1,24 @@
+// This file is part of MinIO Operator
+// Copyright (c) 2023 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package common
+
+// Constants for the webhook endpoints
+const (
+	WebhookAPIVersion       = "/webhook/v1"
+	WebhookDefaultPort      = "4222"
+	WebhookAPIBucketService = WebhookAPIVersion + "/bucketsrv"
+)

pkg/controller/cluster/main-controller.go

@@ -27,6 +27,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/minio/operator/pkg/common"
+
 	xcerts "github.com/minio/pkg/certs"
 
 	"github.com/minio/operator/pkg/controller/cluster/certificates"
@@ -48,10 +50,8 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -266,10 +266,10 @@ func NewController(podName string, namespacesToWatch set.StringSet, kubeClientSe
 	}
 
 	// Initialize operator webhook handlers
-	controller.ws = configureWebhookServer(controller)
+	controller.ws = configureWebhookServer()
 
 	// Initialize operator HTTP upgrade server handlers
-	controller.us = configureHTTPUpgradeServer(controller)
+	controller.us = configureHTTPUpgradeServer()
 
 	// Initialize STS API server handlers
 	controller.sts = configureSTSServer(controller)
@@ -344,29 +344,6 @@ func NewController(podName string, namespacesToWatch set.StringSet, kubeClientSe
 	return controller
 }
 
-func getSecretForTenant(tenant *miniov2.Tenant, accessKey, secretKey string) *v1.Secret {
-	secret := &corev1.Secret{
-		Type: "Opaque",
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      miniov2.WebhookSecret,
-			Namespace: tenant.Namespace,
-			OwnerReferences: []metav1.OwnerReference{
-				*metav1.NewControllerRef(tenant, schema.GroupVersionKind{
-					Group:   miniov2.SchemeGroupVersion.Group,
-					Version: miniov2.SchemeGroupVersion.Version,
-					Kind:    miniov2.MinIOCRDResourceKind,
-				}),
-			},
-		},
-		Data: map[string][]byte{
-			miniov2.WebhookOperatorUsername: []byte(accessKey),
-			miniov2.WebhookOperatorPassword: []byte(secretKey),
-			miniov2.WebhookMinIOArgs:        secretData(tenant, accessKey, secretKey),
-		},
-	}
-	return secret
-}
-
 // Start will set up the event handlers for types we are interested in, as well
 // as syncing informer caches and starting workers. It will block until stopCh
 // is closed, at which point it will shutdown the workqueue and wait for
@@ -814,11 +791,6 @@ func (c *Controller) syncHandler(key string) error {
 		klog.V(2).Infof(err.Error())
 	}
 
-	secret, err := c.applyOperatorWebhookSecret(ctx, tenant)
-	if err != nil {
-		return err
-	}
-
 	// In case the operator certificate is removed or expired, re-create them
 	if err := c.recreateOperatorCertsIfRequired(ctx); err != nil {
 		return err
@@ -994,7 +966,6 @@ func (c *Controller) syncHandler(key string) error {
 			}
 			ss = statefulsets.NewPool(&statefulsets.NewPoolArgs{
 				Tenant:          tenant,
-				WsSecret:        secret,
 				SkipEnvVars:     skipEnvVars,
 				Pool:            &pool,
 				PoolStatus:      &tenant.Status.Pools[i],
@@ -1165,7 +1136,7 @@ func (c *Controller) syncHandler(key string) error {
 		updateURL, err := tenant.UpdateURL(latest, fmt.Sprintf("%s://operator.%s.svc.%s:%s%s",
 			protocol,
 			miniov2.GetNSFromFile(), miniov2.GetClusterDomain(),
-			miniov2.WebhookDefaultPort, miniov2.WebhookAPIUpdate,
+			common.WebhookDefaultPort, miniov2.WebhookAPIUpdate,
 		))
 		if err != nil {
 			_ = c.removeArtifacts()
@@ -1231,7 +1202,6 @@ func (c *Controller) syncHandler(key string) error {
 			// Now proceed to make the yaml changes for the tenant statefulset.
 			ss := statefulsets.NewPool(&statefulsets.NewPoolArgs{
 				Tenant:          tenant,
-				WsSecret:        secret,
 				SkipEnvVars:     skipEnvVars,
 				Pool:            &pool,
 				PoolStatus:      &tenant.Status.Pools[i],
@@ -1283,7 +1253,6 @@ func (c *Controller) syncHandler(key string) error {
 		// generated the expected StatefulSet based on the new tenant configuration
 		expectedStatefulSet := statefulsets.NewPool(&statefulsets.NewPoolArgs{
 			Tenant:          tenant,
-			WsSecret:        secret,
 			SkipEnvVars:     skipEnvVars,
 			Pool:            &pool,
 			PoolStatus:      &tenant.Status.Pools[i],

pkg/controller/cluster/service-account.go

@@ -121,6 +121,19 @@ func getTenantRole(tenant *miniov2.Tenant) *rbacv1.Role {
 					"watch",
 				},
 			},
+			{
+				APIGroups: []string{
+					"",
+				},
+				Resources: []string{
+					"services",
+				},
+				Verbs: []string{
+					"create",
+					"delete",
+					"get",
+				},
+			},
 			{
 				APIGroups: []string{
 					"minio.min.io",

pkg/controller/cluster/status.go

@@ -63,39 +63,6 @@ func (c *Controller) updateTenantStatusWithRetry(ctx context.Context, tenant *mi
 	return t, nil
 }
 
-func (c *Controller) increaseTenantRevision(ctx context.Context, tenant *miniov2.Tenant) (*miniov2.Tenant, error) {
-	return c.increaseTenantRevisionWithRetry(ctx, tenant, true)
-}
-
-func (c *Controller) increaseTenantRevisionWithRetry(ctx context.Context, tenant *miniov2.Tenant, retry bool) (*miniov2.Tenant, error) {
-	// NEVER modify objects from the store. It's a read-only, local cache.
-	// You can use DeepCopy() to make a deep copy of original object and modify this copy
-	// Or create a copy manually for better performance
-	tenantCopy := tenant.DeepCopy()
-	tenantCopy.Status = *tenant.Status.DeepCopy()
-	tenantCopy.Status.Revision = tenantCopy.Status.Revision + 1
-	// If the CustomResourceSubresources feature gate is not enabled,
-	// we must use Update instead of UpdateStatus to update the Status block of the Tenant resource.
-	// UpdateStatus will not allow changes to the Spec of the resource,
-	// which is ideal for ensuring nothing other than resource status has been updated.
-	opts := metav1.UpdateOptions{}
-	t, err := c.minioClientSet.MinioV2().Tenants(tenant.Namespace).UpdateStatus(ctx, tenantCopy, opts)
-	t.EnsureDefaults()
-	if err != nil {
-		// if rejected due to conflict, get the latest tenant and retry once
-		if k8serrors.IsConflict(err) && retry {
-			klog.Info("Hit conflict issue, getting latest version of tenant")
-			tenant, err = c.minioClientSet.MinioV2().Tenants(tenant.Namespace).Get(ctx, tenant.Name, metav1.GetOptions{})
-			if err != nil {
-				return tenant, err
-			}
-			return c.increaseTenantRevisionWithRetry(ctx, tenant, false)
-		}
-		return t, err
-	}
-	return t, nil
-}
-
 func (c *Controller) updatePoolStatus(ctx context.Context, tenant *miniov2.Tenant) (*miniov2.Tenant, error) {
 	return c.updatePoolStatusWithRetry(ctx, tenant, true)
 }

pkg/controller/cluster/sts.go

@@ -32,6 +32,12 @@ const (
 	// stsRoleArn          = "RoleArn"
 )
 
+// STS API constants
+const (
+	STSDefaultPort = "4223"
+	STSEndpoint    = "/sts"
+)
+
 const (
 	// STSEnabled Env variable name to turn on and off the STS Service is enabled, disabled by default
 	STSEnabled = "OPERATOR_STS_ENABLED"
@@ -251,13 +257,13 @@ func configureSTSServer(c *Controller) *http.Server {
 	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
 
 	router.Methods(http.MethodPost).
-		Path(miniov2.STSEndpoint + "/{tenantNamespace}").
+		Path(STSEndpoint + "/{tenantNamespace}").
 		HandlerFunc(c.AssumeRoleWithWebIdentityHandler)
 
 	router.NotFoundHandler = http.NotFoundHandler()
 
 	s := &http.Server{
-		Addr:           ":" + miniov2.STSDefaultPort,
+		Addr:           ":" + STSDefaultPort,
 		Handler:        router,
 		ReadTimeout:    time.Minute,
 		WriteTimeout:   time.Minute,