Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

record backend-errors with latency in epp #17802

Merged
merged 5 commits into from
Jul 10, 2023
Merged

record backend-errors with latency in epp #17802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a054291
record metrics for failed backend requests
k15r Jul 6, 2023
6d4dbf1
fix some previously missed tests
k15r Jul 6, 2023
45af24c
bump image
k15r Jul 6, 2023
9f63f7a
change image folder
k15r Jul 6, 2023
1c56021
Merge branch 'main' into record-backend-errors-for-nats
k15r Jul 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions components/event-publisher-proxy/pkg/handler/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,12 @@ func (h *Handler) sendEventAndRecordMetrics(ctx context.Context, event *cev2even
result, err := h.Sender.Send(ctx, event)
duration := time.Since(start)
if err != nil {
status := 500
if result != nil {
status = result.HTTPStatus()
}
h.collector.RecordBackendLatency(duration, status, host)
h.collector.RecordBackendRequests(status, host)
h.collector.RecordBackendError()
return nil, err
}
Expand Down
115 changes: 105 additions & 10 deletions components/event-publisher-proxy/pkg/handler/handler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,10 +268,29 @@ func TestHandler_publishCloudEvents(t *testing.T) {
},
wantStatus: 500,
wantTEF: `
# HELP eventing_epp_backend_duration_milliseconds The duration of sending events to the messaging server in milliseconds
# TYPE eventing_epp_backend_duration_milliseconds histogram
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.005"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.01"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.025"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.05"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.25"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="2.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="10"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="+Inf"} 1
eventing_epp_backend_duration_milliseconds_sum{code="500",destination_service=""} 0
eventing_epp_backend_duration_milliseconds_count{code="500",destination_service=""} 1
# HELP eventing_epp_backend_errors_total The total number of backend errors while sending events to the messaging server
# TYPE eventing_epp_backend_errors_total counter
eventing_epp_backend_errors_total 1
`,
# HELP eventing_epp_backend_requests_total The total number of backend requests
# TYPE eventing_epp_backend_requests_total counter
eventing_epp_backend_requests_total{code="500",destination_service=""} 1
`,
},
{
name: "Publish binary CloudEvent but backend is full",
Expand All @@ -287,9 +306,28 @@ func TestHandler_publishCloudEvents(t *testing.T) {
},
wantStatus: 507,
wantTEF: `
# HELP eventing_epp_backend_duration_milliseconds The duration of sending events to the messaging server in milliseconds
# TYPE eventing_epp_backend_duration_milliseconds histogram
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.005"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.01"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.025"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.05"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.25"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="2.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="10"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="+Inf"} 1
eventing_epp_backend_duration_milliseconds_sum{code="500",destination_service=""} 0
eventing_epp_backend_duration_milliseconds_count{code="500",destination_service=""} 1
# HELP eventing_epp_backend_errors_total The total number of backend errors while sending events to the messaging server
# TYPE eventing_epp_backend_errors_total counter
eventing_epp_backend_errors_total 1
# HELP eventing_epp_backend_requests_total The total number of backend requests
# TYPE eventing_epp_backend_requests_total counter
eventing_epp_backend_requests_total{code="500",destination_service=""} 1
`,
},
}
Expand Down Expand Up @@ -398,9 +436,28 @@ func TestHandler_publishLegacyEventsAsCE(t *testing.T) {
givenRequest: legacytest.ValidLegacyRequestOrDie(t, "v1", "testapp", "object.created"),
wantHTTPStatus: http.StatusBadGateway,
wantTEF: `
# HELP eventing_epp_backend_errors_total The total number of backend errors while sending events to the messaging server
# TYPE eventing_epp_backend_errors_total counter
eventing_epp_backend_errors_total 1
# HELP eventing_epp_backend_duration_milliseconds The duration of sending events to the messaging server in milliseconds
# TYPE eventing_epp_backend_duration_milliseconds histogram
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.005"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.01"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.025"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.05"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.25"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="2.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="10"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="+Inf"} 1
eventing_epp_backend_duration_milliseconds_sum{code="500",destination_service="FOO"} 0
eventing_epp_backend_duration_milliseconds_count{code="500",destination_service="FOO"} 1
# HELP eventing_epp_backend_errors_total The total number of backend errors while sending events to the messaging server
# TYPE eventing_epp_backend_errors_total counter
eventing_epp_backend_errors_total 1
# HELP eventing_epp_backend_requests_total The total number of backend requests
# TYPE eventing_epp_backend_requests_total counter
eventing_epp_backend_requests_total{code="500",destination_service="FOO"} 1
`,
},
{
Expand All @@ -414,9 +471,28 @@ func TestHandler_publishLegacyEventsAsCE(t *testing.T) {
givenRequest: legacytest.ValidLegacyRequestOrDie(t, "v1", "testapp", "object.created"),
wantHTTPStatus: 507,
wantTEF: `
# HELP eventing_epp_backend_errors_total The total number of backend errors while sending events to the messaging server
# TYPE eventing_epp_backend_errors_total counter
eventing_epp_backend_errors_total 1
# HELP eventing_epp_backend_duration_milliseconds The duration of sending events to the messaging server in milliseconds
# TYPE eventing_epp_backend_duration_milliseconds histogram
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.005"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.01"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.025"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.05"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.25"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="2.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="10"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="+Inf"} 1
eventing_epp_backend_duration_milliseconds_sum{code="500",destination_service="FOO"} 0
eventing_epp_backend_duration_milliseconds_count{code="500",destination_service="FOO"} 1
# HELP eventing_epp_backend_errors_total The total number of backend errors while sending events to the messaging server
# TYPE eventing_epp_backend_errors_total counter
eventing_epp_backend_errors_total 1
# HELP eventing_epp_backend_requests_total The total number of backend requests
# TYPE eventing_epp_backend_requests_total counter
eventing_epp_backend_requests_total{code="500",destination_service="FOO"} 1
`,
},
{
Expand All @@ -430,9 +506,28 @@ func TestHandler_publishLegacyEventsAsCE(t *testing.T) {
givenRequest: legacytest.ValidLegacyRequestOrDie(t, "v1", "testapp", "object.created"),
wantHTTPStatus: 500,
wantTEF: `
# HELP eventing_epp_backend_errors_total The total number of backend errors while sending events to the messaging server
# TYPE eventing_epp_backend_errors_total counter
eventing_epp_backend_errors_total 1
# HELP eventing_epp_backend_duration_milliseconds The duration of sending events to the messaging server in milliseconds
# TYPE eventing_epp_backend_duration_milliseconds histogram
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.005"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.01"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.025"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.05"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.25"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="0.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="2.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="10"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="FOO",le="+Inf"} 1
eventing_epp_backend_duration_milliseconds_sum{code="500",destination_service="FOO"} 0
eventing_epp_backend_duration_milliseconds_count{code="500",destination_service="FOO"} 1
# HELP eventing_epp_backend_errors_total The total number of backend errors while sending events to the messaging server
# TYPE eventing_epp_backend_errors_total counter
eventing_epp_backend_errors_total 1
# HELP eventing_epp_backend_requests_total The total number of backend requests
# TYPE eventing_epp_backend_requests_total counter
eventing_epp_backend_requests_total{code="500",destination_service="FOO"} 1
`,
},
{
Expand Down
60 changes: 58 additions & 2 deletions components/event-publisher-proxy/pkg/handler/handler_v1alpha1_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,9 +332,28 @@ func TestHandler_publishCloudEvents_v1alpha1(t *testing.T) {
wantStatus: 500,

wantTEF: `
# HELP eventing_epp_backend_duration_milliseconds The duration of sending events to the messaging server in milliseconds
# TYPE eventing_epp_backend_duration_milliseconds histogram
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.005"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.01"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.025"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.05"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.25"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="2.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="10"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="+Inf"} 1
eventing_epp_backend_duration_milliseconds_sum{code="500",destination_service=""} 0
eventing_epp_backend_duration_milliseconds_count{code="500",destination_service=""} 1
# HELP eventing_epp_backend_errors_total The total number of backend errors while sending events to the messaging server
# TYPE eventing_epp_backend_errors_total counter
eventing_epp_backend_errors_total 1
# HELP eventing_epp_backend_requests_total The total number of backend requests
# TYPE eventing_epp_backend_requests_total counter
eventing_epp_backend_requests_total{code="500",destination_service=""} 1
`,
},
{
Expand All @@ -351,9 +370,28 @@ func TestHandler_publishCloudEvents_v1alpha1(t *testing.T) {
},
wantStatus: 507,
wantTEF: `
# HELP eventing_epp_backend_duration_milliseconds The duration of sending events to the messaging server in milliseconds
# TYPE eventing_epp_backend_duration_milliseconds histogram
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.005"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.01"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.025"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.05"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.25"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="0.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="2.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="10"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="",le="+Inf"} 1
eventing_epp_backend_duration_milliseconds_sum{code="500",destination_service=""} 0
eventing_epp_backend_duration_milliseconds_count{code="500",destination_service=""} 1
# HELP eventing_epp_backend_errors_total The total number of backend errors while sending events to the messaging server
# TYPE eventing_epp_backend_errors_total counter
eventing_epp_backend_errors_total 1
# HELP eventing_epp_backend_requests_total The total number of backend requests
# TYPE eventing_epp_backend_requests_total counter
eventing_epp_backend_requests_total{code="500",destination_service=""} 1
`,
},
}
Expand Down Expand Up @@ -591,9 +629,27 @@ func TestHandler_sendEventAndRecordMetrics(t *testing.T) {
result: nil,
assertionFunc: assert.Error,
metricErrors: 1,
metricTotal: 0,
metricLatency: 0,
metricTotal: 1,
metricLatency: 1,
metricPublished: 0,
metricLatencyTEF: `
# HELP eventing_epp_backend_duration_milliseconds The duration of sending events to the messaging server in milliseconds
# TYPE eventing_epp_backend_duration_milliseconds histogram
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="0.005"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="0.01"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="0.025"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="0.05"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="0.1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="0.25"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="0.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="1"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="2.5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="5"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="10"} 1
eventing_epp_backend_duration_milliseconds_bucket{code="500",destination_service="foo",le="+Inf"} 1
eventing_epp_backend_duration_milliseconds_sum{code="500",destination_service="foo"} 0
eventing_epp_backend_duration_milliseconds_count{code="500",destination_service="foo"} 1
`,
},
},
}
Expand Down
2 changes: 1 addition & 1 deletion resources/eventing/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ global:
pullPolicy: "IfNotPresent"
publisher_proxy:
name: event-publisher-proxy
version: PR-17794
version: PR-17802
directory: dev
certHandler:
name: eventing-webhook-certificates
Expand Down