Skip to content

Commit

Permalink
fix: support auto mount certs for minioJob (#2288)
Browse files Browse the repository at this point in the history 
* fix: support auto mount cert for minioJob

fix: support auto mount cert for minioJob
  • Loading branch information
@jiuker
jiuker authored Aug 26, 2024
1 parent d027725 commit 2c5289c
Show file tree
Hide file tree
Showing 3 changed files with 140 additions and 11 deletions.
7 changes: 5 additions & 2 deletions docs/examples.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ This example will deploy a MinIO tenant with TLS using certificates provided by
certificate keypairs:

```sh
mkcert "*.minio-tenant.svc.cluster.local"
mkcert "minio.minio-tenant.svc.cluster.local"
mkcert "*.myminio.minio-tenant.svc.cluster.local"
mkcert "*.myminio-hl.minio-tenant.svc.cluster.local"
```
Expand All @@ -90,9 +90,12 @@ inter-node communication.
Create `kubernetes secrets` based on the previous certificates

```$xslt
kubectl create secret tls minio-tls-cert --key="_wildcard.minio-tenant.svc.cluster.local-key.pem" --cert="_wildcard.minio-tenant.svc.cluster.local.pem" -n minio-tenant
kubectl create secret tls minio-tls-cert --key="minio.minio-tenant.svc.cluster.local-key.pem" --cert="minio.minio-tenant.svc.cluster.local.pem" -n minio-tenant
kubectl create secret tls minio-buckets-cert --key="_wildcard.myminio.minio-tenant.svc.cluster.local-key.pem" --cert="_wildcard.myminio.minio-tenant.svc.cluster.local.pem" -n minio-tenant
kubectl create secret tls minio-hl-cert --key="_wildcard.myminio-hl.minio-tenant.svc.cluster.local-key.pem" --cert="_wildcard.myminio-hl.minio-tenant.svc.cluster.local.pem" -n minio-tenant
# create a new secret for the operator certs
kubectl create secret tls operator-ca-tls-minio-tls-cert --key="minio.minio-tenant.svc.cluster.local-key.pem" --cert="minio.minio-tenant.svc.cluster.local.pem" -n minio-tenant
```

You need to provide those `kubernetes secrets` in your Tenant `YAML` overlay using the `externalCertSecret` fields, ie:
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/job-controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ func (c *JobController) SyncHandler(key string) (_ Result, err error) {
if err != nil {
return WrapResult(Result{}, err)
}
err = intervalJob.CreateCommandJob(ctx, c.k8sClient, STSDefaultPort, tenant.TLS())
err = intervalJob.CreateCommandJob(ctx, c.k8sClient, STSDefaultPort, tenant)
if err != nil {
return WrapResult(Result{}, fmt.Errorf("create job error: %w", err))
}
Expand Down
142 changes: 134 additions & 8 deletions pkg/utils/miniojob/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/minio/operator/pkg/apis/job.min.io/v1alpha1"
miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2"
"github.com/minio/operator/pkg/certs"
"github.com/minio/operator/pkg/runtime"
batchjobv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -118,7 +119,7 @@ func (jobCommand *MinIOIntervalJobCommand) Success() bool {
}

// createJob - create job
func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client.Client, jobCR *v1alpha1.MinIOJob, stsPort int, isTLS bool) (objs []client.Object) {
func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client.Client, jobCR *v1alpha1.MinIOJob, stsPort int, t *miniov2.Tenant) (objs []client.Object) {
if jobCommand == nil {
return nil
}
Expand All @@ -129,6 +130,7 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client
}
jobCommand.mutex.RUnlock()
jobCommands := []string{}
insecure := false
if len(jobCommand.CommandSpec.Command) == 0 {
commands := []string{"mc"}
commands = append(commands, strings.SplitN(jobCommand.MCOperation, "/", -1)...)
Expand All @@ -142,6 +144,13 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client
} else {
jobCommands = append(jobCommands, jobCommand.CommandSpec.Command...)
}

for _, command := range jobCommands {
if command == "--insecure" {
insecure = true
}
}

mcImage := jobCR.Spec.MCImage
if mcImage == "" {
mcImage = DefaultMCImage
Expand All @@ -152,7 +161,6 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client
MountPath: "/.mc",
},
}
baseVolumeMounts = append(baseVolumeMounts, jobCommand.CommandSpec.VolumeMounts...)
baseVolumes := []corev1.Volume{
{
Name: "config-dir",
Expand All @@ -161,7 +169,16 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client
},
},
}
// if auto cert is not enabled and insecure is not enabled and tenant tls is enabled, add cert volumes
if !t.AutoCert() && !insecure && t.TLS() {
certsVolumes, certsVolumeMounts := getCertVolumes(t)
baseVolumeMounts = append(baseVolumeMounts, certsVolumeMounts...)
baseVolumes = append(baseVolumes, certsVolumes...)
}

baseVolumeMounts = append(baseVolumeMounts, jobCommand.CommandSpec.VolumeMounts...)
baseVolumes = append(baseVolumes, jobCommand.CommandSpec.Volumes...)

baseEnvFrom := []corev1.EnvFromSource{
{
SecretRef: &corev1.SecretEnvSource{
Expand All @@ -173,7 +190,7 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client
}
baseEnvFrom = append(baseEnvFrom, jobCommand.CommandSpec.EnvFrom...)
scheme := "http"
if isTLS {
if t.TLS() {
scheme = "https"
}
secret := &corev1.Secret{
Expand Down Expand Up @@ -239,8 +256,8 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client
}

// CreateJob - create job
func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort int, isTLS bool) error {
for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR, stsPort, isTLS) {
func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort int, t *miniov2.Tenant) error {
for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR, stsPort, t) {
if obj == nil {
continue
}
Expand Down Expand Up @@ -314,10 +331,10 @@ func (intervalJob *MinIOIntervalJob) GetMinioJobStatus(_ context.Context) v1alph
}

// CreateCommandJob - create command job
func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client, stsPort int, isTLS bool) error {
func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client, stsPort int, t *miniov2.Tenant) error {
for _, command := range intervalJob.Command {
if len(command.CommandSpec.DependsOn) == 0 {
err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort, isTLS)
err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort, t)
if err != nil {
return err
}
Expand All @@ -334,7 +351,7 @@ func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sCl
}
}
if allDepsSuccess {
err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort, isTLS)
err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort, t)
if err != nil {
return err
}
Expand Down Expand Up @@ -379,3 +396,112 @@ func GenerateMinIOIntervalJobCommand(commandSpec v1alpha1.CommandSpec, commandIn
}
return jobCommand, nil
}

// getCertVolumes - get cert volumes
// from statefulsets.NewPool implementation
func getCertVolumes(t *miniov2.Tenant) (certsVolumes []corev1.Volume, certsVolumeMounts []corev1.VolumeMount) {
var certVolumeSources []corev1.VolumeProjection
for index, secret := range t.Spec.ExternalCertSecret {
crtMountPath := fmt.Sprintf("hostname-%d/%s", index, certs.PublicCertFile)
caMountPath := fmt.Sprintf("CAs/hostname-%d.crt", index)

if index == 0 {
crtMountPath = certs.PublicCertFile
caMountPath = fmt.Sprintf("%s/%s", certs.CertsCADir, certs.PublicCertFile)
}

var serverCertPaths []corev1.KeyToPath
if secret.Type == "kubernetes.io/tls" {
serverCertPaths = []corev1.KeyToPath{
{Key: certs.TLSCertFile, Path: crtMountPath},
{Key: certs.TLSCertFile, Path: caMountPath},
}
} else if secret.Type == "cert-manager.io/v1alpha2" || secret.Type == "cert-manager.io/v1" {
serverCertPaths = []corev1.KeyToPath{
{Key: certs.TLSCertFile, Path: crtMountPath},
{Key: certs.CAPublicCertFile, Path: caMountPath},
}
} else {
serverCertPaths = []corev1.KeyToPath{
{Key: certs.PublicCertFile, Path: crtMountPath},
{Key: certs.PublicCertFile, Path: caMountPath},
}
}
certVolumeSources = append(certVolumeSources, corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret.Name,
},
Items: serverCertPaths,
},
})
}
for index, secret := range t.Spec.ExternalClientCertSecrets {
crtMountPath := fmt.Sprintf("client-%d/client.crt", index)
var clientKeyPairPaths []corev1.KeyToPath
if secret.Type == "kubernetes.io/tls" {
clientKeyPairPaths = []corev1.KeyToPath{
{Key: certs.TLSCertFile, Path: crtMountPath},
}
} else if secret.Type == "cert-manager.io/v1alpha2" || secret.Type == "cert-manager.io/v1" {
clientKeyPairPaths = []corev1.KeyToPath{
{Key: certs.TLSCertFile, Path: crtMountPath},
{Key: certs.CAPublicCertFile, Path: fmt.Sprintf("%s/client-ca-%d.crt", certs.CertsCADir, index)},
}
} else {
clientKeyPairPaths = []corev1.KeyToPath{
{Key: certs.PublicCertFile, Path: crtMountPath},
}
}
certVolumeSources = append(certVolumeSources, corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret.Name,
},
Items: clientKeyPairPaths,
},
})
}
for index, secret := range t.Spec.ExternalCaCertSecret {
var caCertPaths []corev1.KeyToPath
// This covers both secrets of type "kubernetes.io/tls" and
// "cert-manager.io/v1alpha2" because of same keys in both.
if secret.Type == "kubernetes.io/tls" {
caCertPaths = []corev1.KeyToPath{
{Key: certs.TLSCertFile, Path: fmt.Sprintf("%s/ca-%d.crt", certs.CertsCADir, index)},
}
} else if secret.Type == "cert-manager.io/v1alpha2" || secret.Type == "cert-manager.io/v1" {
caCertPaths = []corev1.KeyToPath{
{Key: certs.CAPublicCertFile, Path: fmt.Sprintf("%s/ca-%d.crt", certs.CertsCADir, index)},
}
} else {
caCertPaths = []corev1.KeyToPath{
{Key: certs.PublicCertFile, Path: fmt.Sprintf("%s/ca-%d.crt", certs.CertsCADir, index)},
}
}
certVolumeSources = append(certVolumeSources, corev1.VolumeProjection{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{
Name: secret.Name,
},
Items: caCertPaths,
},
})
}

if len(certVolumeSources) > 0 {
certsVolumes = append(certsVolumes, corev1.Volume{
Name: "certs",
VolumeSource: corev1.VolumeSource{
Projected: &corev1.ProjectedVolumeSource{
Sources: certVolumeSources,
},
},
})
certsVolumeMounts = append(certsVolumeMounts, corev1.VolumeMount{
Name: "certs",
MountPath: "/root/.mc/certs",
})
}
return certsVolumes, certsVolumeMounts
}

0 comments on commit 2c5289c

Please sign in to comment.