Skip to content

Commit

Permalink
Add Patch to give optional option to enable ServiceMonitor to use cer…
Browse files Browse the repository at this point in the history
…t-manager-managed serving-cert with TLS verification

Adds a patch to configure ServiceMonitor with  `insecureSkipVerify: false`  to ensure TLS verification using cert-manager certificates. Updates documentation and corrects misaligned comments.
  • Loading branch information
camilamacedo86 committed Nov 1, 2024
1 parent c331b70 commit f8ba05c
Show file tree
Hide file tree
Showing 31 changed files with 396 additions and 66 deletions.
6 changes: 6 additions & 0 deletions docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,12 @@ func main() {
}

if secureMetrics {
// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
// you can uncomment the following lines to use the certificate managed by cert-manager.
// metricsServerOptions.CertDir = "/var/run/secrets/kubernetes.io/certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"

// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,11 @@
resources:
- monitor.yaml

# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [WEBHOOK] patch under config/default/kustomization.yaml
# to mount the "serving-cert" secret in the Manager Deployment.
#patches:
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'serving-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
# using certificates managed by cert-manager
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: controller-manager-metrics-monitor
namespace: system
spec:
endpoints:
- tlsConfig:
insecureSkipVerify: false
ca:
secret:
name: serving-cert
key: ca.crt
cert:
secret:
name: serving-cert
key: tls.crt
keySecret:
name: serving-cert
key: tls.key
6 changes: 6 additions & 0 deletions docs/book/src/getting-started/testdata/project/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,12 @@ func main() {
}

if secureMetrics {
// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
// you can uncomment the following lines to use the certificate managed by cert-manager.
// metricsServerOptions.CertDir = "/var/run/secrets/kubernetes.io/certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"

// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,11 @@
resources:
- monitor.yaml

# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [WEBHOOK] patch under config/default/kustomization.yaml
# to mount the "serving-cert" secret in the Manager Deployment.
#patches:
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'serving-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
# using certificates managed by cert-manager
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: controller-manager-metrics-monitor
namespace: system
spec:
endpoints:
- tlsConfig:
insecureSkipVerify: false
ca:
secret:
name: serving-cert
key: ca.crt
cert:
secret:
name: serving-cert
key: tls.crt
keySecret:
name: serving-cert
key: tls.key
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ func main() {
}

if secureMetrics {
// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
// you can uncomment the following lines to use the certificate managed by cert-manager.
// metricsServerOptions.CertDir = "/var/run/secrets/kubernetes.io/certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"

// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,11 @@
resources:
- monitor.yaml

# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [WEBHOOK] patch under config/default/kustomization.yaml
# to mount the "serving-cert" secret in the Manager Deployment.
#patches:
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'serving-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
# using certificates managed by cert-manager
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: controller-manager-metrics-monitor
namespace: system
spec:
endpoints:
- tlsConfig:
insecureSkipVerify: false
ca:
secret:
name: serving-cert
key: ca.crt
cert:
secret:
name: serving-cert
key: tls.crt
keySecret:
name: serving-cert
key: tls.key
50 changes: 40 additions & 10 deletions docs/book/src/reference/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,46 @@ An [issue](https://github.com/kubernetes-sigs/controller-runtime/issues/2781) ha
enhance the controller-runtime and address these considerations.
</aside>

### By exposing the metrics endpoint using HTTPS and Cert-Manager

Integrating `cert-manager` with your metrics service enables secure
HTTPS access via TLS encryption. Follow the steps below to configure
your project to expose the metrics endpoint using HTTPS with cert-manager.

1. **Enable Cert-Manager in `config/default/kustomization.yaml`:**
- Uncomment the cert-manager resource to include it in your project:

```yaml
- ../certmanager
```

2. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:**
- Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification:

```yaml
- path: monitor_tls_patch.yaml
target:
kind: ServiceMonitor
```

3. **Enable the Patch to Mount the Cert-Manager-Managed Secret in the Controller Deployment in `config/default/kustomization.yaml`:**
- Use the `manager_webhook_patch.yaml` (or create a custom metrics patch) to mount the `serving-cert` secret in the Manager Deployment.

```yaml
- path: manager_webhook_patch.yaml
```

4. **Update `cmd/main.go` to Use the Certificate Managed by Cert-Manager:**
- Modify `cmd/main.go` to configure the metrics server to use the cert-manager-managed certificates.
Uncomment the lines for `CertDir`, `CertName`, and `KeyName`:

```go
if secureMetrics {
metricsServerOptions.CertDir = "/var/run/secrets/kubernetes.io/certs"
metricsServerOptions.CertName = "tls.crt"
metricsServerOptions.KeyName = "tls.key"
}
```

### By using Network Policy (You can optionally enable)

Expand All @@ -194,16 +234,6 @@ Uncomment the following line in the `config/default/kustomization.yaml`:
#- ../network-policy
```

### By exposing the metrics endpoint using HTTPS and CertManager

Integrating `cert-manager` with your metrics service can secure the endpoint via TLS encryption.

To modify your project setup to expose metrics using HTTPS with
the help of cert-manager, you'll need to change the configuration of both
the `Service` under `config/default/metrics_service.yaml` and
the `ServiceMonitor` under `config/prometheus/monitor.yaml` to use a secure HTTPS port
and ensure the necessary certificate is applied.

## Exporting Metrics for Prometheus

Follow the steps below to export the metrics using the Prometheus Operator:
Expand Down
1 change: 1 addition & 0 deletions pkg/plugins/common/kustomize/v2/scaffolds/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func (s *initScaffolder) Scaffold() error {
&network_policy.NetworkPolicyAllowMetrics{},
&prometheus.Kustomization{},
&prometheus.Monitor{},
&prometheus.ServiceMonitorPatch{},
}

return scaffold.Execute(templates...)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,13 @@ func (f *Kustomization) SetTemplateDefaults() error {

const kustomizationTemplate = `resources:
- monitor.yaml
# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [WEBHOOK] patch under config/default/kustomization.yaml
# to mount the "serving-cert" secret in the Manager Deployment.
#patches:
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor
`
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func (f *Monitor) SetTemplateDefaults() error {
return nil
}

// nolint:lll
const serviceMonitorTemplate = `# Prometheus Monitor Service (Metrics)
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
Expand All @@ -59,14 +60,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'serving-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package prometheus

import (
"path/filepath"

"sigs.k8s.io/kubebuilder/v4/pkg/machinery"
)

var _ machinery.Template = &ServiceMonitorPatch{}

// ServiceMonitorPatch scaffolds a file that defines the patch for the ServiceMonitor
// to use cert-manager managed certificates for secure TLS configuration.
type ServiceMonitorPatch struct {
machinery.TemplateMixin
machinery.ProjectNameMixin
}

// SetTemplateDefaults implements file.Template
func (f *ServiceMonitorPatch) SetTemplateDefaults() error {
if f.Path == "" {
f.Path = filepath.Join("config", "prometheus", "monitor_tls_patch.yaml")
}

f.TemplateBody = serviceMonitorPatchTemplate

return nil
}

const serviceMonitorPatchTemplate = `# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
# using certificates managed by cert-manager
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: controller-manager-metrics-monitor
namespace: system
spec:
endpoints:
- tlsConfig:
insecureSkipVerify: false
ca:
secret:
name: serving-cert
key: ca.crt
cert:
secret:
name: serving-cert
key: tls.crt
keySecret:
name: serving-cert
key: tls.key
`
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,12 @@ func main() {
}
if secureMetrics {
// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
// you can uncomment the following lines to use the certificate managed by cert-manager.
// metricsServerOptions.CertDir = "/var/run/secrets/kubernetes.io/certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"
// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
Expand Down
Loading

0 comments on commit f8ba05c

Please sign in to comment.