Fix rare bug where some k8s events would be emitted without a timesta…

…mp (#28342)

Co-authored-by: Jen Gilbert <j.h.gilbert@gmail.com>

pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_eventbundle.go

@@ -50,8 +50,17 @@ func (b *kubernetesEventBundle) addEvent(event *v1.Event) error {
 
 	// We do not process the events in chronological order necessarily.
 	// We only care about the first time they occurred, the last time and the count.
-	b.timeStamp = float64(event.FirstTimestamp.Unix())
-	b.lastTimestamp = math.Max(b.lastTimestamp, float64(event.LastTimestamp.Unix()))
+	if event.FirstTimestamp.IsZero() {
+		b.timeStamp = float64(event.EventTime.Unix())
+	} else {
+		b.timeStamp = float64(event.FirstTimestamp.Unix())
+	}
+
+	if event.LastTimestamp.IsZero() {
+		b.lastTimestamp = math.Max(b.lastTimestamp, float64(event.EventTime.Unix()))
+	} else {
+		b.lastTimestamp = math.Max(b.lastTimestamp, float64(event.LastTimestamp.Unix()))
+	}
 
 	b.countByAction[fmt.Sprintf("**%s**: %s\n", event.Reason, event.Message)] += int(event.Count)
 

pkg/collector/corechecks/cluster/kubernetesapiserver/unbundled_events.go

@@ -103,13 +103,20 @@ func (c *unbundledTransformer) Transform(events []*v1.Event) ([]event.Event, []e
 			"false",
 		)
 
+		var timestamp int64
+		if ev.FirstTimestamp.IsZero() {
+			timestamp = int64(ev.EventTime.Unix())
+		} else {
+			timestamp = int64(ev.FirstTimestamp.Unix())
+		}
+
 		event := event.Event{
 			Title:          fmt.Sprintf("%s: %s", readableKey, ev.Reason),
 			Priority:       event.PriorityNormal,
 			Host:           hostInfo.hostname,
 			SourceTypeName: source,
 			EventType:      CheckName,
-			Ts:             int64(ev.LastTimestamp.Unix()),
+			Ts:             timestamp,
 			Tags:           tags,
 			AggregationKey: fmt.Sprintf("kubernetes_apiserver:%s", involvedObject.UID),
 			AlertType:      getDDAlertType(ev.Type),

pkg/collector/corechecks/cluster/kubernetesapiserver/unbundled_events_test.go

@@ -62,6 +62,23 @@ func TestUnbundledEventsTransform(t *testing.T) {
 			LastTimestamp:  ts,
 			Count:          1,
 		},
+		{
+			InvolvedObject: v1.ObjectReference{
+				Kind:      "Pod",
+				Namespace: "default",
+				Name:      "squirtle-8fff95dbb-tsc7v",
+				UID:       "43b7e0d3-9212-4355-a957-4ac15ce3a263",
+			},
+			Type:    "Normal",
+			Reason:  "Scheduled",
+			Message: "Successfully assigned default/squirtle-8fff95dbb-tsc7v to test-host",
+			Source: v1.EventSource{
+				Host: "test-host",
+			},
+			ReportingController: "default-scheduler",
+			EventTime:           metav1.NewMicroTime(ts.Time),
+			Count:               1,
+		},
 		{
 			InvolvedObject: v1.ObjectReference{
 				Kind:      "ReplicaSet",
@@ -213,6 +230,30 @@ func TestUnbundledEventsTransform(t *testing.T) {
 					SourceTypeName: "kubernetes",
 					EventType:      "kubernetes_apiserver",
 				},
+				{
+					Title:    "Pod default/squirtle-8fff95dbb-tsc7v: Scheduled",
+					Text:     "Successfully assigned default/squirtle-8fff95dbb-tsc7v to test-host",
+					Ts:       ts.Time.Unix(),
+					Priority: event.PriorityNormal,
+					Host:     "test-host-test-cluster",
+					Tags: []string{
+						"event_reason:Scheduled",
+						"kube_kind:Pod",
+						"kube_name:squirtle-8fff95dbb-tsc7v",
+						"kube_namespace:default",
+						"kubernetes_kind:Pod",
+						"name:squirtle-8fff95dbb-tsc7v",
+						"namespace:default",
+						"pod_name:squirtle-8fff95dbb-tsc7v",
+						"reporting_controller:default-scheduler",
+						"orchestrator:kubernetes",
+						"source_component:",
+					},
+					AlertType:      event.AlertTypeInfo,
+					AggregationKey: "kubernetes_apiserver:43b7e0d3-9212-4355-a957-4ac15ce3a263",
+					SourceTypeName: "kubernetes",
+					EventType:      "kubernetes_apiserver",
+				},
 				{
 					Title:    "Pod default/wartortle-8fff95dbb-tsc7v: Failed",
 					Text:     "All containers terminated",
@@ -271,6 +312,30 @@ func TestUnbundledEventsTransform(t *testing.T) {
 					SourceTypeName: "kubernetes",
 					EventType:      "kubernetes_apiserver",
 				},
+				{
+					Title:    "Pod default/squirtle-8fff95dbb-tsc7v: Scheduled",
+					Text:     "Successfully assigned default/squirtle-8fff95dbb-tsc7v to test-host",
+					Ts:       ts.Time.Unix(),
+					Priority: event.PriorityNormal,
+					Host:     "test-host-test-cluster",
+					Tags: []string{
+						"event_reason:Scheduled",
+						"kube_kind:Pod",
+						"kube_name:squirtle-8fff95dbb-tsc7v",
+						"kube_namespace:default",
+						"kubernetes_kind:Pod",
+						"name:squirtle-8fff95dbb-tsc7v",
+						"namespace:default",
+						"pod_name:squirtle-8fff95dbb-tsc7v",
+						"reporting_controller:default-scheduler",
+						"orchestrator:kubernetes",
+						"source_component:",
+					},
+					AlertType:      event.AlertTypeInfo,
+					AggregationKey: "kubernetes_apiserver:43b7e0d3-9212-4355-a957-4ac15ce3a263",
+					SourceTypeName: "kubernetes",
+					EventType:      "kubernetes_apiserver",
+				},
 				{
 					Title:    "Pod default/wartortle-8fff95dbb-tsc7v: Failed",
 					Text:     "All containers terminated",
@@ -333,6 +398,34 @@ func TestUnbundledEventsTransform(t *testing.T) {
 					SourceTypeName: "kubernetes",
 					EventType:      "kubernetes_apiserver",
 				},
+				{
+					Title: "Events from the Pod default/squirtle-8fff95dbb-tsc7v",
+					Text: fmt.Sprintf(`%%%%%%%[1]s
+1 **Scheduled**: Successfully assigned default/squirtle-8fff95dbb-tsc7v to test-host
+%[1]s
+ _Events emitted by the  seen at %[2]s since %[2]s_%[1]s
+
+ %%%%%%`, " ", ts.String()),
+					Ts:       ts.Time.Unix(),
+					Priority: event.PriorityNormal,
+					Tags: []string{
+						"kube_kind:Pod",
+						"kube_name:squirtle-8fff95dbb-tsc7v",
+						"kubernetes_kind:Pod",
+						"name:squirtle-8fff95dbb-tsc7v",
+						"kube_namespace:default",
+						"namespace:default",
+						"pod_name:squirtle-8fff95dbb-tsc7v",
+						"source_component:",
+						"orchestrator:kubernetes",
+						"reporting_controller:default-scheduler",
+					},
+					Host:           "test-host-test-cluster",
+					AlertType:      event.AlertTypeInfo,
+					AggregationKey: "kubernetes_apiserver:43b7e0d3-9212-4355-a957-4ac15ce3a263",
+					SourceTypeName: "kubernetes",
+					EventType:      "kubernetes_apiserver",
+				},
 				{
 					Title: "Events from the Pod default/wartortle-8fff95dbb-tsc7v",
 					Text: fmt.Sprintf(`%%%%%%%[1]s

releasenotes-dca/notes/apiserver-events-missing-timestamp-3617d812a9d75d5d.yaml

@@ -0,0 +1,12 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+fixes:
+  - |
+    Fixes a rare bug where some Kubernetes events would be emitted
+    without a timestamp and would be dropped upstream as a result.