Skip to content

Commit

Permalink
Add Patch to give optional option to enable ServiceMonitor to use cer…
Browse files Browse the repository at this point in the history
…t-manager-managed serving-cert with TLS verification

Adds a patch to configure ServiceMonitor with  `insecureSkipVerify: false`  to ensure TLS verification using cert-manager certificates. Updates documentation and corrects misaligned comments.
  • Loading branch information
camilamacedo86 committed Oct 30, 2024
1 parent f7a02ad commit 170d528
Show file tree
Hide file tree
Showing 31 changed files with 412 additions and 146 deletions.
14 changes: 7 additions & 7 deletions docs/book/src/cronjob-tutorial/testdata/project/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,16 +123,16 @@ func main() {
metricsServerOptions := metricsserver.Options{
BindAddress: metricsAddr,
SecureServing: secureMetrics,
// TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are
// not provided, self-signed certificates will be generated by default. This option is not recommended for
// production environments as self-signed certificates do not offer the same level of trust and security
// as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing
// unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName
// to provide certificates, ensuring the server communicates using trusted and secure certificates.
TLSOpts: tlsOpts,
TLSOpts: tlsOpts,
}

if secureMetrics {
// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
// you can uncomment the following lines to use the certificate managed by cert-manager.
// metricsServerOptions.CertDir = "/var/run/secrets/kubernetes.io/certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"

// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,11 @@
resources:
- monitor.yaml

# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [WEBHOOK] patch under config/default/kustomization.yaml
# to mount the "serving-cert" secret in the Manager Deployment.
#patches:
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'serving-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
# using certificates managed by cert-manager
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: controller-manager-metrics-monitor
namespace: system
spec:
endpoints:
- tlsConfig:
insecureSkipVerify: false
ca:
secret:
name: serving-cert
key: ca.crt
cert:
secret:
name: serving-cert
key: tls.crt
keySecret:
name: serving-cert
key: tls.key
14 changes: 7 additions & 7 deletions docs/book/src/getting-started/testdata/project/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,16 +103,16 @@ func main() {
metricsServerOptions := metricsserver.Options{
BindAddress: metricsAddr,
SecureServing: secureMetrics,
// TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are
// not provided, self-signed certificates will be generated by default. This option is not recommended for
// production environments as self-signed certificates do not offer the same level of trust and security
// as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing
// unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName
// to provide certificates, ensuring the server communicates using trusted and secure certificates.
TLSOpts: tlsOpts,
TLSOpts: tlsOpts,
}

if secureMetrics {
// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
// you can uncomment the following lines to use the certificate managed by cert-manager.
// metricsServerOptions.CertDir = "/var/run/secrets/kubernetes.io/certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"

// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,11 @@
resources:
- monitor.yaml

# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [WEBHOOK] patch under config/default/kustomization.yaml
# to mount the "serving-cert" secret in the Manager Deployment.
#patches:
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'serving-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
# using certificates managed by cert-manager
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: controller-manager-metrics-monitor
namespace: system
spec:
endpoints:
- tlsConfig:
insecureSkipVerify: false
ca:
secret:
name: serving-cert
key: ca.crt
cert:
secret:
name: serving-cert
key: tls.crt
keySecret:
name: serving-cert
key: tls.key
14 changes: 7 additions & 7 deletions docs/book/src/multiversion-tutorial/testdata/project/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,16 +122,16 @@ func main() {
metricsServerOptions := metricsserver.Options{
BindAddress: metricsAddr,
SecureServing: secureMetrics,
// TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are
// not provided, self-signed certificates will be generated by default. This option is not recommended for
// production environments as self-signed certificates do not offer the same level of trust and security
// as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing
// unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName
// to provide certificates, ensuring the server communicates using trusted and secure certificates.
TLSOpts: tlsOpts,
TLSOpts: tlsOpts,
}

if secureMetrics {
// TODO(user): If cert-manager is enabled in config/default/kustomization.yaml,
// you can uncomment the following lines to use the certificate managed by cert-manager.
// metricsServerOptions.CertDir = "/var/run/secrets/kubernetes.io/certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"

// FilterProvider is used to protect the metrics endpoint with authn/authz.
// These configurations ensure that only authorized users and service accounts
// can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,11 @@
resources:
- monitor.yaml

# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [WEBHOOK] patch under config/default/kustomization.yaml
# to mount the "serving-cert" secret in the Manager Deployment.
#patches:
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'serving-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
# using certificates managed by cert-manager
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: controller-manager-metrics-monitor
namespace: system
spec:
endpoints:
- tlsConfig:
insecureSkipVerify: false
ca:
secret:
name: serving-cert
key: ca.crt
cert:
secret:
name: serving-cert
key: tls.crt
keySecret:
name: serving-cert
key: tls.key
92 changes: 50 additions & 42 deletions docs/book/src/reference/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,40 +136,16 @@ spec:
<aside class="warning">
<h1>Changes Recommended for Production</h1>

The default scaffold to configure the metrics server in `cmd/main.go` uses `TLSOpts` that rely on self-signed certificates
(SelfCerts), which are generated automatically. However, self-signed certificates are **not** recommended for production
environments as they do not offer the same level of trust and security as certificates issued by a trusted
Certificate Authority (CA).
The default scaffold in `cmd/main.go` uses a **controller-runtime feature**
to generate a certificate for securing the metrics server. While convenient
for development and testing, this setup is **not recommended for production**.

While self-signed certificates are convenient for development and testing, they are unsuitable for production
because they do not establish a chain of trust, making them vulnerable to security threats.
Additionally, review the configuration file at `config/prometheus/monitor.yaml`
to ensure secure integration with Prometheus. If `insecureSkipVerify: true` is
enabled, certificate verification is turned off. **This is not recommended for production**
as it exposes the system to man-in-the-middle attacks, potentially allowing
unauthorized access to metrics data.

Furthermore, check the configuration file located at `config/prometheus/monitor.yaml` to
ensure secure integration with Prometheus. If the `insecureSkipVerify: true` option is enabled,
it means that certificate verification is turned off. This is **not** recommended for production as
it poses a significant security risk by making the system vulnerable to man-in-the-middle attacks,
where an attacker could intercept and manipulate the communication between Prometheus and the monitored services.
This could lead to unauthorized access to metrics data, compromising the integrity and confidentiality of the information.

**In both cases, the primary risk is potentially allowing unauthorized access to sensitive metrics data.**

### Recommended Actions for a Secure Production Setup

1. **Replace Self-Signed Certificates:**
- Instead of using `TLSOpts`, configure the `CertDir`, `CertName`, and `KeyName` options to use your own certificates.
This ensures that your server communicates using trusted and secure certificates.

2. **Configure Prometheus Monitoring Securely:**
- Check and update your Prometheus configuration file (`config/prometheus/monitor.yaml`) to ensure secure settings.
- Replace `insecureSkipVerify: true` with the following secure options:

```yaml
caFile: The path to the CA certificate file, e.g., /etc/metrics-certs/ca.crt.
certFile: The path to the client certificate file, e.g., /etc/metrics-certs/tls.crt.
keyFile: The path to the client key file, e.g., /etc/metrics-certs/tls.key.
```

These settings ensure encrypted and authenticated communication between Prometheus and the monitored services, providing a secure monitoring setup.
</aside>

<aside class="note">
Expand All @@ -187,6 +163,48 @@ An [issue](https://github.com/kubernetes-sigs/controller-runtime/issues/2781) ha
enhance the controller-runtime and address these considerations.
</aside>

### By exposing the metrics endpoint using HTTPS and Cert-Manager

Integrating `cert-manager` with your metrics service enables secure
HTTPS access via TLS encryption. Follow the steps below to configure
your project to expose the metrics endpoint using HTTPS with cert-manager.

1. **Enable Cert-Manager in `config/default/kustomization.yaml`:**
- Uncomment the cert-manager resource to include it in your project:

```yaml
- ../certmanager
```

2. **Enable the Patch for the `ServiceMonitor` to Use the Cert-Manager-Managed Secret `config/prometheus/kustomization.yaml`:**
- Add or uncomment the `ServiceMonitor` patch to securely reference the cert-manager-managed secret, replacing insecure configurations with secure certificate verification:

```yaml
- path: monitor_tls_patch.yaml
target:
kind: ServiceMonitor
```

3. **Enable the Patch to Mount the Cert-Manager-Managed Secret in the Controller Deployment in `config/default/kustomization.yaml`:**
- Use the `manager_webhook_patch.yaml` (or create a custom metrics patch) to mount the `serving-cert` secret in the Manager Deployment.

```yaml
- path: manager_webhook_patch.yaml
```

4. **Update `cmd/main.go` to Use the Certificate Managed by Cert-Manager:**
- Modify `cmd/main.go` to configure the metrics server to use the cert-manager-managed certificates.
Uncomment the lines for `CertDir`, `CertName`, and `KeyName`:

```go
if secureMetrics {
// TODO(user): If cert-manager is enabled under config/default/kustomization.yaml, you can uncomment the following
// lines to use the certificate managed by cert-manager, mounted as a Kubernetes secret named 'serving-cert'.
// metricsServerOptions.CertDir = "/var/run/secrets/kubernetes.io/certs"
// metricsServerOptions.CertName = "tls.crt"
// metricsServerOptions.KeyName = "tls.key"
}
```

### By using Network Policy (You can optionally enable)

Expand All @@ -202,16 +220,6 @@ Uncomment the following line in the `config/default/kustomization.yaml`:
#- ../network-policy
```

### By exposing the metrics endpoint using HTTPS and CertManager

Integrating `cert-manager` with your metrics service can secure the endpoint via TLS encryption.

To modify your project setup to expose metrics using HTTPS with
the help of cert-manager, you'll need to change the configuration of both
the `Service` under `config/default/metrics_service.yaml` and
the `ServiceMonitor` under `config/prometheus/monitor.yaml` to use a secure HTTPS port
and ensure the necessary certificate is applied.

## Exporting Metrics for Prometheus

Follow the steps below to export the metrics using the Prometheus Operator:
Expand Down
1 change: 1 addition & 0 deletions pkg/plugins/common/kustomize/v2/scaffolds/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func (s *initScaffolder) Scaffold() error {
&network_policy.NetworkPolicyAllowMetrics{},
&prometheus.Kustomization{},
&prometheus.Monitor{},
&prometheus.ServiceMonitorPatch{},
}

return scaffold.Execute(templates...)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,13 @@ func (f *Kustomization) SetTemplateDefaults() error {

const kustomizationTemplate = `resources:
- monitor.yaml
# [PROMETHEUS WITH CERTMANAGER] The following patch configures the ServiceMonitor in ../prometheus
# to securely reference certificates created and managed by cert-manager.
# Additionally, ensure that you uncomment the [WEBHOOK] patch under config/default/kustomization.yaml
# to mount the "serving-cert" secret in the Manager Deployment.
#patches:
# - path: monitor_tls_patch.yaml
# target:
# kind: ServiceMonitor
`
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func (f *Monitor) SetTemplateDefaults() error {
return nil
}

// nolint:lll
const serviceMonitorTemplate = `# Prometheus Monitor Service (Metrics)
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
Expand All @@ -59,14 +60,10 @@ spec:
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
tlsConfig:
# TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
# certificate verification. This poses a significant security risk by making the system vulnerable to
# man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between
# Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data,
# compromising the integrity and confidentiality of the information.
# Please use the following options for secure configurations:
# caFile: /etc/metrics-certs/ca.crt
# certFile: /etc/metrics-certs/tls.crt
# keyFile: /etc/metrics-certs/tls.key
# certificate verification, exposing the system to potential man-in-the-middle attacks.
# For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
# To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
# which securely references the certificate from the 'serving-cert' secret.
insecureSkipVerify: true
selector:
matchLabels:
Expand Down
Loading

0 comments on commit 170d528

Please sign in to comment.