Skip to content

Commit 171ae5a

Browse files
rubenvp8510rubenvp8510
committed
Service monitor scrap tempo metrics for monolithic
Signed-off-by: Ruben Vargas <ruben.vp8510@gmail.com>
1 parent c3d8027 commit 171ae5a

File tree

9 files changed

+121
-49
lines changed

9 files changed

+121
-49
lines changed

‎.chloggen/fix_monolithic_metrics.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
2+
change_type: bug_fix
3+
4+
# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
5+
component: tempomonolithic
6+
7+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
8+
note: Scrap tempo metrics for monolithic.
9+
10+
# One or more tracking issues related to the change
11+
issues: [1275]
12+
13+
# (Optional) One or more lines of additional information to render under the primary note.
14+
# These lines will be padded with 2 spaces and then inserted directly into the document.
15+
# Use pipe (|) for multiline entries.
16+
subtext:

‎internal/manifests/monolithic/configmap.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,8 +177,9 @@ func buildTempoConfig(opts Options) ([]byte, error) {
177177
config.Server.HttpServerReadTimeout = opts.Tempo.Spec.Timeout.Duration
178178
config.Server.HttpServerWriteTimeout = opts.Tempo.Spec.Timeout.Duration
179179
if tempo.Spec.Multitenancy.IsGatewayEnabled() {
180+
// We need this to scrap metrics.
181+
config.Server.HTTPListenAddress = "0.0.0.0"
180182
// all connections to tempo must go via gateway
181-
config.Server.HTTPListenAddress = "localhost"
182183
config.Server.GRPCListenAddress = "localhost"
183184
}
184185

‎internal/manifests/monolithic/servicemonitor.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,14 @@ func BuildServiceMonitor(opts Options) *monitoringv1.ServiceMonitor {
1212
tempo := opts.Tempo
1313
if tempo.Spec.Multitenancy.IsGatewayEnabled() {
1414
labels := ComponentLabels(manifestutils.GatewayComponentName, tempo.Name)
15-
return servicemonitor.NewServiceMonitor(tempo.Namespace, tempo.Name, labels, false, manifestutils.GatewayComponentName, manifestutils.GatewayInternalHttpPortName)
15+
return servicemonitor.NewServiceMonitor(tempo.Namespace, tempo.Name, labels, false, manifestutils.GatewayComponentName,
16+
[]string{
17+
manifestutils.GatewayInternalHttpPortName,
18+
manifestutils.HttpPortName,
19+
})
1620
} else {
1721
labels := ComponentLabels(manifestutils.TempoMonolithComponentName, tempo.Name)
18-
return servicemonitor.NewServiceMonitor(tempo.Namespace, tempo.Name, labels, false, manifestutils.TempoMonolithComponentName, manifestutils.HttpPortName)
22+
return servicemonitor.NewServiceMonitor(
23+
tempo.Namespace, tempo.Name, labels, false, manifestutils.TempoMonolithComponentName, []string{manifestutils.HttpPortName})
1924
}
2025
}

‎internal/manifests/monolithic/servicemonitor_test.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,23 @@ func TestBuildServiceMonitorGateway(t *testing.T) {
120120
TargetLabel: "job",
121121
},
122122
},
123-
}},
123+
}, {
124+
Scheme: "http",
125+
Port: "http",
126+
Path: "/metrics",
127+
RelabelConfigs: []*monitoringv1.RelabelConfig{
128+
{
129+
SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_service_label_app_kubernetes_io_instance"},
130+
TargetLabel: "cluster",
131+
},
132+
{
133+
SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_namespace", "__meta_kubernetes_service_label_app_kubernetes_io_component"},
134+
Separator: ptr.To("/"),
135+
TargetLabel: "job",
136+
},
137+
},
138+
},
139+
},
124140
NamespaceSelector: monitoringv1.NamespaceSelector{
125141
MatchNames: []string{"default"},
126142
},

‎internal/manifests/monolithic/services.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,12 @@ func buildGatewayService(opts Options) *corev1.Service {
146146
Port: manifestutils.GatewayPortInternalHTTPServer,
147147
TargetPort: intstr.FromString(manifestutils.GatewayInternalHttpPortName),
148148
},
149+
{
150+
Name: manifestutils.HttpPortName,
151+
Protocol: corev1.ProtocolTCP,
152+
Port: manifestutils.PortHTTPServer,
153+
TargetPort: intstr.FromString(manifestutils.HttpPortName),
154+
},
149155
}
150156

151157
if tempo.Spec.Ingestion != nil && tempo.Spec.Ingestion.OTLP != nil &&

‎internal/manifests/monolithic/services_test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,12 @@ func TestBuildServices(t *testing.T) {
269269
Port: 8081,
270270
TargetPort: intstr.FromString("internal"),
271271
},
272+
{
273+
Name: "http",
274+
Protocol: corev1.ProtocolTCP,
275+
Port: 3200,
276+
TargetPort: intstr.FromString("http"),
277+
},
272278
{
273279
Name: "otlp-grpc",
274280
Protocol: corev1.ProtocolTCP,

‎internal/manifests/servicemonitor/servicemonitor.go‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,14 @@ func BuildServiceMonitors(params manifestutils.Params) []client.Object {
3535

3636
func buildServiceMonitor(params manifestutils.Params, component string, port string) *monitoringv1.ServiceMonitor {
3737
labels := manifestutils.ComponentLabels(component, params.Tempo.Name)
38-
return NewServiceMonitor(params.Tempo.Namespace, params.Tempo.Name, labels, params.CtrlConfig.Gates.HTTPEncryption, component, port)
38+
return NewServiceMonitor(params.Tempo.Namespace, params.Tempo.Name, labels, params.CtrlConfig.Gates.HTTPEncryption, component, []string{port})
3939
}
4040

4141
func buildFrontEndServiceMonitor(params manifestutils.Params, port string) *monitoringv1.ServiceMonitor {
4242
labels := manifestutils.ComponentLabels(manifestutils.QueryFrontendComponentName, params.Tempo.Name)
4343
tls := params.CtrlConfig.Gates.HTTPEncryption && params.Tempo.Spec.Template.Gateway.Enabled
4444
return NewServiceMonitor(params.Tempo.Namespace, params.Tempo.Name, labels, tls,
45-
manifestutils.QueryFrontendComponentName, port)
45+
manifestutils.QueryFrontendComponentName, []string{port})
4646
}
4747

4848
// NewServiceMonitor creates a ServiceMonitor.
@@ -52,7 +52,7 @@ func NewServiceMonitor(
5252
labels labels.Set,
5353
tls bool,
5454
component string,
55-
port string,
55+
ports []string,
5656
) *monitoringv1.ServiceMonitor {
5757
scheme := "http"
5858
var tlsConfig *monitoringv1.TLSConfig
@@ -91,6 +91,30 @@ func NewServiceMonitor(
9191
}
9292
}
9393

94+
var endpoints []monitoringv1.Endpoint
95+
96+
for _, port := range ports {
97+
endpoints = append(endpoints, monitoringv1.Endpoint{
98+
Scheme: scheme,
99+
Port: port,
100+
Path: "/metrics",
101+
TLSConfig: tlsConfig,
102+
// Custom relabel configs to be compatible with predefined Tempo dashboards:
103+
// https://grafana.com/docs/tempo/latest/operations/monitoring/#dashboards
104+
RelabelConfigs: []*monitoringv1.RelabelConfig{
105+
{
106+
SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_service_label_app_kubernetes_io_instance"},
107+
TargetLabel: "cluster",
108+
},
109+
{
110+
SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_namespace", "__meta_kubernetes_service_label_app_kubernetes_io_component"},
111+
Separator: ptr.To("/"),
112+
TargetLabel: "job",
113+
},
114+
},
115+
})
116+
}
117+
94118
return &monitoringv1.ServiceMonitor{
95119
TypeMeta: metav1.TypeMeta{
96120
APIVersion: monitoringv1.SchemeGroupVersion.String(),
@@ -102,25 +126,7 @@ func NewServiceMonitor(
102126
Labels: labels,
103127
},
104128
Spec: monitoringv1.ServiceMonitorSpec{
105-
Endpoints: []monitoringv1.Endpoint{{
106-
Scheme: scheme,
107-
Port: port,
108-
Path: "/metrics",
109-
TLSConfig: tlsConfig,
110-
// Custom relabel configs to be compatible with predefined Tempo dashboards:
111-
// https://grafana.com/docs/tempo/latest/operations/monitoring/#dashboards
112-
RelabelConfigs: []*monitoringv1.RelabelConfig{
113-
{
114-
SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_service_label_app_kubernetes_io_instance"},
115-
TargetLabel: "cluster",
116-
},
117-
{
118-
SourceLabels: []monitoringv1.LabelName{"__meta_kubernetes_namespace", "__meta_kubernetes_service_label_app_kubernetes_io_component"},
119-
Separator: ptr.To("/"),
120-
TargetLabel: "job",
121-
},
122-
},
123-
}},
129+
Endpoints: endpoints,
124130
NamespaceSelector: monitoringv1.NamespaceSelector{
125131
MatchNames: []string{namespace},
126132
},

‎tests/e2e-openshift/monolithic-multitenancy-openshift/01-assert.yaml‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -29,26 +29,6 @@ status:
2929
started: true
3030
phase: Running
3131

32-
---
33-
apiVersion: v1
34-
kind: Service
35-
metadata:
36-
name: tempo-mmo-gateway
37-
spec:
38-
ports:
39-
- name: public
40-
port: 8080
41-
protocol: TCP
42-
targetPort: public
43-
- name: internal
44-
port: 8081
45-
protocol: TCP
46-
targetPort: internal
47-
- name: otlp-grpc
48-
port: 4317
49-
protocol: TCP
50-
targetPort: grpc-public
51-
5232
---
5333
apiVersion: rbac.authorization.k8s.io/v1
5434
kind: ClusterRole
@@ -88,6 +68,30 @@ subjects:
8868
namespace: chainsaw-monolithic-multitenancy
8969

9070
---
71+
apiVersion: v1
72+
kind: Service
73+
metadata:
74+
name: tempo-mmo-gateway
75+
spec:
76+
ports:
77+
- name: public
78+
protocol: TCP
79+
port: 8080
80+
targetPort: public
81+
- name: internal
82+
protocol: TCP
83+
port: 8081
84+
targetPort: internal
85+
- name: http
86+
protocol: TCP
87+
port: 3200
88+
targetPort: http
89+
- name: otlp-grpc
90+
protocol: TCP
91+
port: 4317
92+
targetPort: grpc-public
93+
---
94+
9195
apiVersion: monitoring.coreos.com/v1
9296
kind: ServiceMonitor
9397
metadata:
@@ -114,6 +118,20 @@ spec:
114118
- __meta_kubernetes_service_label_app_kubernetes_io_component
115119
targetLabel: job
116120
scheme: http
121+
- path: /metrics
122+
port: http
123+
relabelings:
124+
- action: replace
125+
sourceLabels:
126+
- __meta_kubernetes_service_label_app_kubernetes_io_instance
127+
targetLabel: cluster
128+
- action: replace
129+
separator: /
130+
sourceLabels:
131+
- __meta_kubernetes_namespace
132+
- __meta_kubernetes_service_label_app_kubernetes_io_component
133+
targetLabel: job
134+
scheme: http
117135
namespaceSelector:
118136
matchNames:
119137
- chainsaw-monolithic-multitenancy

‎tests/e2e-openshift/monolithic-multitenancy-openshift/check_metrics.sh‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,7 @@ TOKEN=$(oc create token e2e-test-metrics-reader -n $NAMESPACE)
77
THANOS_QUERIER_HOST=$(oc get route thanos-querier -n openshift-monitoring -o json | jq -r '.spec.host')
88

99
#Check TempoMonolithc metircs
10-
# Tempo component metrics not exposed due to bug. https://issues.redhat.com/browse/TRACING-5472
11-
#metrics="tempo_query_frontend_queries_total tempo_distributor_bytes_received_total tempo_distributor_spans_received_total tempo_ingester_bytes_received_total tempo_distributor_traces_per_batch_count tempo_build_info"
12-
metrics="http_request_duration_seconds_bucket"
10+
metrics="tempo_query_frontend_queries_total tempo_distributor_bytes_received_total tempo_distributor_spans_received_total tempo_ingester_bytes_received_total tempo_distributor_traces_per_batch_count tempo_build_info"
1311

1412
for metric in $metrics; do
1513
query="$metric"

0 commit comments

Comments
 (0)