feat(metrics): add metrics for last event receipt timestamps

migrations/1740074359683_last-node-events-index.js

@@ -0,0 +1,10 @@
+/* eslint-disable camelcase */
+
+exports.shorthands = undefined;
+
+exports.up = pgm => {
+  pgm.createIndex('event_observer_requests', [
+    'event_path',
+    { name: 'receive_timestamp', sort: 'DESC' },
+  ]);
+};

src/datastore/pg-store.ts

@@ -4675,4 +4675,14 @@ export class PgStore extends BasePgStore {
       tx_total_size: parseInt(result[0].tx_total_size),
     };
   }
+
+  /// Returns timestamps for the last Stacks node events received by the API Event Server, grouped
+  /// by event type.
+  async getLastStacksNodeEventTimestamps() {
+    return await this.sql<{ event_path: string; receive_timestamp: Date }[]>`
+      SELECT DISTINCT ON (event_path) event_path, receive_timestamp
+      FROM event_observer_requests
+      ORDER BY event_path, receive_timestamp DESC
+    `;
+  }
 }

src/event-stream/event-server.ts

@@ -639,6 +639,7 @@ function createMessageProcessorQueue(db: PgWriteStore): EventMessageHandler {
   let metrics:
     | {
         eventTimer: prom.Histogram;
+        lastEventTimestamps: prom.Gauge;
         blocksInPreviousBurnBlock: prom.Gauge;
       }
     | undefined;
@@ -650,6 +651,18 @@ function createMessageProcessorQueue(db: PgWriteStore): EventMessageHandler {
         labelNames: ['event'],
         buckets: prom.exponentialBuckets(50, 3, 10), // 10 buckets, from 50 ms to 15 minutes
       }),
+      lastEventTimestamps: new prom.Gauge({
+        name: 'stacks_last_event_timestamps',
+        help: 'Last Stacks node events received timestamp',
+        labelNames: ['event'] as const,
+        async collect() {
+          const events = await db.getLastStacksNodeEventTimestamps();
+          this.reset();
+          for (const event of events) {
+            this.set({ event: event.event_path }, event.receive_timestamp.getTime());
+          }
+        },
+      }),
       blocksInPreviousBurnBlock: new prom.Gauge({
         name: 'stacks_blocks_in_previous_burn_block',
         help: 'Number of Stacks blocks produced in the previous burn block',