Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions api/v1alpha1/valkeycluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,41 @@ type ValkeyClusterSpec struct {
// +kubebuilder:default:={enabled:true}
// +optional
Exporter ExporterSpec `json:"exporter,omitempty"`

// TLS configuration
// +optional
TLS *TLSConfig `json:"tls,omitempty"`
}

// TLSConfig defines the TLS configuration for a Valkey cluster.
type TLSConfig struct {
// Enable TLS
// +kubebuilder:default=false
Enabled bool `json:"enabled,omitempty"`

// Name of the Secret containing TLS keys
ExistingSecret string `json:"existingSecret,omitempty"`
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we use the name "existing" here, will the cert-manager create a separate secret which it updates?
Or it named so that a user should know that is needs to exist?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe we can change ExistingSecret to secretRef so that it can be used for both exisiting secret or secret generated by Certificate CRD.


// Secret key name containing server public certificate (default=server.crt)
// +kubebuilder:default=server.crt
Cert string `json:"cert,omitempty"`

// Secret key name containing server private key (default=server.key)
// +kubebuilder:default=server.key
Key string `json:"key,omitempty"`

// Secret key name containing Certificate Authority public certificate (default=ca.crt)
// +kubebuilder:default=ca.crt
CA string `json:"ca,omitempty"`

// TODO: cert-manager integration and rotation
// IssuerRef references a cert-manager Issuer or ClusterIssuer.
IssuerRef *IssuerRef `json:"issuerRef,omitempty"`
}

type IssuerRef struct {
Name string `json:"name"`
Kind string `json:"kind,omitempty"`
}

type ExporterSpec struct {
Expand Down Expand Up @@ -142,6 +177,8 @@ const (
ReasonClusterHealthy = "ClusterHealthy"
ReasonServiceError = "ServiceError"
ReasonConfigMapError = "ConfigMapError"
ReasonInvalidTLSSecret = "InvalidTLSSecret"
ReasonMissingTLSSecret = "MissingTLSSecret"
ReasonDeploymentError = "DeploymentError"
ReasonPodListError = "PodListError"
ReasonAddingNodes = "AddingNodes"
Expand Down
40 changes: 40 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

35 changes: 35 additions & 0 deletions config/crd/bases/valkey.io_valkeyclusters.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1124,6 +1124,41 @@ spec:
format: int32
minimum: 1
type: integer
tls:
description: TLS configuration
properties:
ca:
default: ca.crt
description: Secret key name containing Certificate Authority
public certificate (default=ca.crt)
type: string
cert:
default: server.crt
description: Secret key name containing server public certificate
(default=server.crt)
type: string
enabled:
default: false
description: Enable TLS
type: boolean
existingSecret:
description: Name of the Secret containing TLS keys
type: string
issuerRef:
description: IssuerRef references a cert-manager Issuer or ClusterIssuer.
properties:
kind:
type: string
name:
type: string
required:
- name
type: object
key:
default: server.key
description: Secret key name containing server private key (default=server.key)
type: string
type: object
tolerations:
description: Tolerations to apply to the pods
items:
Expand Down
1 change: 1 addition & 0 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ rules:
- ""
resources:
- pods
- secrets
verbs:
- get
- list
Expand Down
124 changes: 86 additions & 38 deletions internal/controller/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,60 @@ func generateContainersDef(cluster *valkeyiov1alpha1.ValkeyCluster) []corev1.Con
if cluster.Spec.Image != "" {
image = cluster.Spec.Image
}
var valkeyArgs []string
var envVars []corev1.EnvVar
volumeMounts := []corev1.VolumeMount{
{
Name: "scripts",
MountPath: "/scripts",
},
{
Name: "valkey-conf",
MountPath: "/config",
ReadOnly: true,
},
}

if cluster.Spec.TLS != nil && cluster.Spec.TLS.Enabled {
certName, keyName, caName := getTLSFileNames(cluster.Spec.TLS)
certPath := fmt.Sprintf("%s/%s", tlsCertMountPath, certName)
keyPath := fmt.Sprintf("%s/%s", tlsCertMountPath, keyName)
caPath := fmt.Sprintf("%s/%s", tlsCertMountPath, caName)

valkeyArgs = []string{
"--tls-port", fmt.Sprintf("%d", DefaultPort),
"--port", "0",
"--tls-cluster", "yes",
"--tls-replication", "yes",
"--tls-cert-file", certPath,
"--tls-key-file", keyPath,
"--tls-ca-cert-file", caPath,
}

envVars = append(envVars, corev1.EnvVar{
Name: "VALKEY_TLS_ARGS",
Value: fmt.Sprintf("--tls --cert %s --key %s --cacert %s", certPath, keyPath, caPath),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, we don't need --insecure here but we have InsecureSkipVerify=true?

})

volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: "tls-certs",
MountPath: tlsCertMountPath,
ReadOnly: true,
})
}

command := append([]string{
"valkey-server",
"/config/valkey.conf",
}, valkeyArgs...)

containers := []corev1.Container{
{
Name: "valkey-server",
Image: image,
Resources: cluster.Spec.Resources,
Command: []string{
"valkey-server",
"/config/valkey.conf",
},
Command: command,
Env: envVars,
Ports: []corev1.ContainerPort{
{
Name: "client",
Expand Down Expand Up @@ -98,17 +143,7 @@ func generateContainersDef(cluster *valkeyiov1alpha1.ValkeyCluster) []corev1.Con
},
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "scripts",
MountPath: "/scripts",
},
{
Name: "valkey-conf",
MountPath: "/config",
ReadOnly: true,
},
},
VolumeMounts: volumeMounts,
},
}

Expand Down Expand Up @@ -146,6 +181,41 @@ func createClusterDeployment(cluster *valkeyiov1alpha1.ValkeyCluster, shardIndex
nodeLabels[LabelShardIndex] = strconv.Itoa(shardIndex)
nodeLabels[LabelNodeIndex] = strconv.Itoa(nodeIndex)

volumes := []corev1.Volume{
{
Name: "scripts",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: cluster.Name,
},
DefaultMode: func(i int32) *int32 { return &i }(0755),
},
},
},
{
Name: "valkey-conf",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: cluster.Name,
},
},
},
},
}

if cluster.Spec.TLS != nil && cluster.Spec.TLS.Enabled {
volumes = append(volumes, corev1.Volume{
Name: "tls-certs",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: cluster.Spec.TLS.ExistingSecret,
},
},
})
}

deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: deploymentName(cluster.Name, shardIndex, nodeIndex),
Expand All @@ -166,29 +236,7 @@ func createClusterDeployment(cluster *valkeyiov1alpha1.ValkeyCluster, shardIndex
Affinity: cluster.Spec.Affinity,
NodeSelector: cluster.Spec.NodeSelector,
Tolerations: cluster.Spec.Tolerations,
Volumes: []corev1.Volume{
{
Name: "scripts",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: cluster.Name,
},
DefaultMode: func(i int32) *int32 { return &i }(0755),
},
},
},
{
Name: "valkey-conf",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: cluster.Name,
},
},
},
},
},
Volumes: volumes,
},
},
},
Expand Down
64 changes: 61 additions & 3 deletions internal/controller/metrics_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,74 @@ import (
valkeyiov1alpha1 "valkey.io/valkey-operator/api/v1alpha1"
)

// getExporterEnvironmentVariables returns the environment variables for the Redis Exporter container.
// The Redis address is set to the Redis host and port, and TLS configuration is set if enabled.
// The function returns list of environment variables.
func getExporterEnvironmentVariables(tlsEnabled bool, certPath, keyPath, caPath string) []corev1.EnvVar {
var envVars []corev1.EnvVar
redisHost := "redis://localhost"
if tlsEnabled {
redisHost = "rediss://localhost"
envVars = append(envVars, corev1.EnvVar{
Name: "REDIS_EXPORTER_TLS_CA_CERT_FILE",
Value: caPath,
})
envVars = append(envVars, corev1.EnvVar{
Name: "REDIS_EXPORTER_SKIP_TLS_VERIFICATION",
Value: "true",
})
envVars = append(envVars, corev1.EnvVar{
Name: "REDIS_EXPORTER_TLS_CLIENT_CERT_FILE",
Value: certPath,
})
envVars = append(envVars, corev1.EnvVar{
Name: "REDIS_EXPORTER_TLS_CLIENT_KEY_FILE",
Value: keyPath,
})
}

envVars = append(envVars, corev1.EnvVar{
Name: "REDIS_ADDR",
Value: fmt.Sprintf("%s:%d", redisHost, DefaultPort),
})

envVars = append(envVars, corev1.EnvVar{
Name: "REDIS_EXPORTER_WEB_LISTEN_ADDRESS",
Value: fmt.Sprintf(":%d", DefaultExporterPort),
})

return envVars
}

// generateMetricsExporterContainerDef generates the container definition for the metrics exporter sidecar.
func generateMetricsExporterContainerDef(cluster *valkeyiov1alpha1.ValkeyCluster) corev1.Container {
exporterImage := DefaultExporterImage
if cluster.Spec.Exporter.Image != "" {
exporterImage = cluster.Spec.Exporter.Image
}
var volumeMounts []corev1.VolumeMount

tlsEnabled := cluster.Spec.TLS != nil && cluster.Spec.TLS.Enabled
var certPath, keyPath, caPath string
if tlsEnabled {
certName, keyName, caName := getTLSFileNames(cluster.Spec.TLS)
certPath = fmt.Sprintf("%s/%s", tlsCertMountPath, certName)
keyPath = fmt.Sprintf("%s/%s", tlsCertMountPath, keyName)
caPath = fmt.Sprintf("%s/%s", tlsCertMountPath, caName)
volumeMounts = []corev1.VolumeMount{
{
Name: "tls-certs",
MountPath: tlsCertMountPath,
ReadOnly: true,
},
}
}
envVars := getExporterEnvironmentVariables(tlsEnabled, certPath, keyPath, caPath)
return corev1.Container{
Name: "metrics-exporter",
Image: exporterImage,
Args: []string{fmt.Sprintf("--redis.addr=localhost:%d", DefaultPort)},
Name: "metrics-exporter",
Image: exporterImage,
Env: envVars,
VolumeMounts: volumeMounts,
Ports: []corev1.ContainerPort{
{
Name: "metrics",
Expand Down
2 changes: 1 addition & 1 deletion internal/controller/scripts/liveness-check.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ function timeout {
# Perform check
response=$(
timeout $timeout \
valkey-cli -h localhost -p $port PING)
valkey-cli -h localhost -p $port $VALKEY_TLS_ARGS PING)

responseFirstWord=$(echo "$response" | head -n1 | awk '{print $1;}')
if [ "$response" != "PONG" ] && [ "$responseFirstWord" != "LOADING" ] && [ "$responseFirstWord" != "MASTERDOWN" ]; then
Expand Down
Loading