From 3ecb6c03fbb8ecceb38110b637a5931d11bcffca Mon Sep 17 00:00:00 2001 From: Steven Blumenthal Date: Fri, 9 Aug 2024 14:35:24 -0400 Subject: [PATCH] Fix rare bug where some k8s events would be emitted without a timestamp (#28342) Co-authored-by: Jen Gilbert --- .../kubernetes_eventbundle.go | 13 ++- .../kubernetesapiserver/unbundled_events.go | 9 +- .../unbundled_events_test.go | 93 +++++++++++++++++++ ...ts-missing-timestamp-3617d812a9d75d5d.yaml | 12 +++ 4 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 releasenotes-dca/notes/apiserver-events-missing-timestamp-3617d812a9d75d5d.yaml diff --git a/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_eventbundle.go b/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_eventbundle.go index 282669b823949..4fc133477a85d 100644 --- a/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_eventbundle.go +++ b/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_eventbundle.go @@ -50,8 +50,17 @@ func (b *kubernetesEventBundle) addEvent(event *v1.Event) error { // We do not process the events in chronological order necessarily. // We only care about the first time they occurred, the last time and the count. - b.timeStamp = float64(event.FirstTimestamp.Unix()) - b.lastTimestamp = math.Max(b.lastTimestamp, float64(event.LastTimestamp.Unix())) + if event.FirstTimestamp.IsZero() { + b.timeStamp = float64(event.EventTime.Unix()) + } else { + b.timeStamp = float64(event.FirstTimestamp.Unix()) + } + + if event.LastTimestamp.IsZero() { + b.lastTimestamp = math.Max(b.lastTimestamp, float64(event.EventTime.Unix())) + } else { + b.lastTimestamp = math.Max(b.lastTimestamp, float64(event.LastTimestamp.Unix())) + } b.countByAction[fmt.Sprintf("**%s**: %s\n", event.Reason, event.Message)] += int(event.Count) diff --git a/pkg/collector/corechecks/cluster/kubernetesapiserver/unbundled_events.go b/pkg/collector/corechecks/cluster/kubernetesapiserver/unbundled_events.go index 6efd9e4558984..d6ae0ca1a4a0c 100644 --- a/pkg/collector/corechecks/cluster/kubernetesapiserver/unbundled_events.go +++ b/pkg/collector/corechecks/cluster/kubernetesapiserver/unbundled_events.go @@ -103,13 +103,20 @@ func (c *unbundledTransformer) Transform(events []*v1.Event) ([]event.Event, []e "false", ) + var timestamp int64 + if ev.FirstTimestamp.IsZero() { + timestamp = int64(ev.EventTime.Unix()) + } else { + timestamp = int64(ev.FirstTimestamp.Unix()) + } + event := event.Event{ Title: fmt.Sprintf("%s: %s", readableKey, ev.Reason), Priority: event.PriorityNormal, Host: hostInfo.hostname, SourceTypeName: source, EventType: CheckName, - Ts: int64(ev.LastTimestamp.Unix()), + Ts: timestamp, Tags: tags, AggregationKey: fmt.Sprintf("kubernetes_apiserver:%s", involvedObject.UID), AlertType: getDDAlertType(ev.Type), diff --git a/pkg/collector/corechecks/cluster/kubernetesapiserver/unbundled_events_test.go b/pkg/collector/corechecks/cluster/kubernetesapiserver/unbundled_events_test.go index 0f7bd26efad7a..8cafd44921e67 100644 --- a/pkg/collector/corechecks/cluster/kubernetesapiserver/unbundled_events_test.go +++ b/pkg/collector/corechecks/cluster/kubernetesapiserver/unbundled_events_test.go @@ -62,6 +62,23 @@ func TestUnbundledEventsTransform(t *testing.T) { LastTimestamp: ts, Count: 1, }, + { + InvolvedObject: v1.ObjectReference{ + Kind: "Pod", + Namespace: "default", + Name: "squirtle-8fff95dbb-tsc7v", + UID: "43b7e0d3-9212-4355-a957-4ac15ce3a263", + }, + Type: "Normal", + Reason: "Scheduled", + Message: "Successfully assigned default/squirtle-8fff95dbb-tsc7v to test-host", + Source: v1.EventSource{ + Host: "test-host", + }, + ReportingController: "default-scheduler", + EventTime: metav1.NewMicroTime(ts.Time), + Count: 1, + }, { InvolvedObject: v1.ObjectReference{ Kind: "ReplicaSet", @@ -213,6 +230,30 @@ func TestUnbundledEventsTransform(t *testing.T) { SourceTypeName: "kubernetes", EventType: "kubernetes_apiserver", }, + { + Title: "Pod default/squirtle-8fff95dbb-tsc7v: Scheduled", + Text: "Successfully assigned default/squirtle-8fff95dbb-tsc7v to test-host", + Ts: ts.Time.Unix(), + Priority: event.PriorityNormal, + Host: "test-host-test-cluster", + Tags: []string{ + "event_reason:Scheduled", + "kube_kind:Pod", + "kube_name:squirtle-8fff95dbb-tsc7v", + "kube_namespace:default", + "kubernetes_kind:Pod", + "name:squirtle-8fff95dbb-tsc7v", + "namespace:default", + "pod_name:squirtle-8fff95dbb-tsc7v", + "reporting_controller:default-scheduler", + "orchestrator:kubernetes", + "source_component:", + }, + AlertType: event.AlertTypeInfo, + AggregationKey: "kubernetes_apiserver:43b7e0d3-9212-4355-a957-4ac15ce3a263", + SourceTypeName: "kubernetes", + EventType: "kubernetes_apiserver", + }, { Title: "Pod default/wartortle-8fff95dbb-tsc7v: Failed", Text: "All containers terminated", @@ -271,6 +312,30 @@ func TestUnbundledEventsTransform(t *testing.T) { SourceTypeName: "kubernetes", EventType: "kubernetes_apiserver", }, + { + Title: "Pod default/squirtle-8fff95dbb-tsc7v: Scheduled", + Text: "Successfully assigned default/squirtle-8fff95dbb-tsc7v to test-host", + Ts: ts.Time.Unix(), + Priority: event.PriorityNormal, + Host: "test-host-test-cluster", + Tags: []string{ + "event_reason:Scheduled", + "kube_kind:Pod", + "kube_name:squirtle-8fff95dbb-tsc7v", + "kube_namespace:default", + "kubernetes_kind:Pod", + "name:squirtle-8fff95dbb-tsc7v", + "namespace:default", + "pod_name:squirtle-8fff95dbb-tsc7v", + "reporting_controller:default-scheduler", + "orchestrator:kubernetes", + "source_component:", + }, + AlertType: event.AlertTypeInfo, + AggregationKey: "kubernetes_apiserver:43b7e0d3-9212-4355-a957-4ac15ce3a263", + SourceTypeName: "kubernetes", + EventType: "kubernetes_apiserver", + }, { Title: "Pod default/wartortle-8fff95dbb-tsc7v: Failed", Text: "All containers terminated", @@ -333,6 +398,34 @@ func TestUnbundledEventsTransform(t *testing.T) { SourceTypeName: "kubernetes", EventType: "kubernetes_apiserver", }, + { + Title: "Events from the Pod default/squirtle-8fff95dbb-tsc7v", + Text: fmt.Sprintf(`%%%%%%%[1]s +1 **Scheduled**: Successfully assigned default/squirtle-8fff95dbb-tsc7v to test-host +%[1]s + _Events emitted by the seen at %[2]s since %[2]s_%[1]s + + %%%%%%`, " ", ts.String()), + Ts: ts.Time.Unix(), + Priority: event.PriorityNormal, + Tags: []string{ + "kube_kind:Pod", + "kube_name:squirtle-8fff95dbb-tsc7v", + "kubernetes_kind:Pod", + "name:squirtle-8fff95dbb-tsc7v", + "kube_namespace:default", + "namespace:default", + "pod_name:squirtle-8fff95dbb-tsc7v", + "source_component:", + "orchestrator:kubernetes", + "reporting_controller:default-scheduler", + }, + Host: "test-host-test-cluster", + AlertType: event.AlertTypeInfo, + AggregationKey: "kubernetes_apiserver:43b7e0d3-9212-4355-a957-4ac15ce3a263", + SourceTypeName: "kubernetes", + EventType: "kubernetes_apiserver", + }, { Title: "Events from the Pod default/wartortle-8fff95dbb-tsc7v", Text: fmt.Sprintf(`%%%%%%%[1]s diff --git a/releasenotes-dca/notes/apiserver-events-missing-timestamp-3617d812a9d75d5d.yaml b/releasenotes-dca/notes/apiserver-events-missing-timestamp-3617d812a9d75d5d.yaml new file mode 100644 index 0000000000000..67554ffc60edb --- /dev/null +++ b/releasenotes-dca/notes/apiserver-events-missing-timestamp-3617d812a9d75d5d.yaml @@ -0,0 +1,12 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +fixes: + - | + Fixes a rare bug where some Kubernetes events would be emitted + without a timestamp and would be dropped upstream as a result.