Bring in VideoFrame FrameMetadata from FFI

user_timestamped_video to cpp-example-collection

align with latest protos

update rust SDK commit

copilot cmake suggestions

update collection commit

Author: stephen-derosa

client-sdk-rust

@@ -270,6 +270,15 @@ class Room {
                                VideoFrameCallback callback,
                                VideoStream::Options opts = {});
 
+  /**
+   * @brief Sets the video frame event callback via
+   * SubscriptionThreadDispatcher.
+   */
+  void setOnVideoFrameEventCallback(const std::string &participant_identity,
+                                    TrackSource source,
+                                    VideoFrameEventCallback callback,
+                                    VideoStream::Options opts = {});
+
   /**
    * @brief Clears the audio frame callback via SubscriptionThreadDispatcher.
    */

cpp-example-collection

@@ -307,6 +307,17 @@ struct AudioEncodingOptions {
   std::uint64_t max_bitrate = 0;
 };
 
+/**
+ * Optional RTP packet-trailer features for published video tracks.
+ */
+struct PacketTrailerFeatures {
+  /** Embed a user-supplied wall-clock timestamp. */
+  bool user_timestamp = false;
+
+  /** Embed a monotonically increasing frame identifier. */
+  bool frame_id = false;
+};
+
 /**
  * Options for publishing a track to the room.
  */
@@ -337,6 +348,9 @@ struct TrackPublishOptions {
 
   /** Enable pre-connect buffering for lower startup latency. */
   std::optional<bool> preconnect_buffer;
+
+  /** Optional packet-trailer features to enable for published video. */
+  PacketTrailerFeatures packet_trailer_features{};
 };
 
 // ---------------------------------------------------------

include/livekit/room.h

@@ -159,6 +159,26 @@ class SubscriptionThreadDispatcher {
                                VideoFrameCallback callback,
                                VideoStream::Options opts = {});
 
+  /**
+   * Register or replace a rich video frame event callback for a remote
+   * subscription.
+   *
+   * The callback is keyed by remote participant identity plus \p source.
+   * If the matching remote video track is already subscribed, \ref Room may
+   * immediately call \ref handleTrackSubscribed to start a reader.
+   *
+   * @param participant_identity Identity of the remote participant.
+   * @param source               Track source to match.
+   * @param callback             Function invoked for each decoded video frame
+   *                             event, including optional metadata.
+   * @param opts                 Options used when creating the backing
+   *                             \ref VideoStream.
+   */
+  void setOnVideoFrameEventCallback(const std::string &participant_identity,
+                                    TrackSource source,
+                                    VideoFrameEventCallback callback,
+                                    VideoStream::Options opts = {});
+
   /**
    * Remove an audio callback registration and stop any active reader.
    *
@@ -382,7 +402,8 @@ class SubscriptionThreadDispatcher {
 
   /// Stored video callback registration plus stream-construction options.
   struct RegisteredVideoCallback {
-    VideoFrameCallback callback;
+    VideoFrameCallback legacy_callback;
+    VideoFrameEventCallback event_callback;
     VideoStream::Options options;
   };
 
@@ -413,8 +434,7 @@ class SubscriptionThreadDispatcher {
   /// is extracted and returned to the caller for joining outside the lock.
   std::thread startVideoReaderLocked(const CallbackKey &key,
                                      const std::shared_ptr<Track> &track,
-                                     const VideoFrameCallback &cb,
-                                     const VideoStream::Options &opts);
+                                     const RegisteredVideoCallback &callback);
 
   /// Extract and close the data reader for a given callback ID, returning its
   /// thread.  Must be called with \ref lock_ held.

include/livekit/room_event_types.h

@@ -17,6 +17,7 @@
 #pragma once
 
 #include <cstdint>
+#include <optional>
 
 #include "livekit/ffi_handle.h"
 
@@ -36,6 +37,26 @@ enum class VideoRotation {
   VIDEO_ROTATION_270 = 270,
 };
 
+/**
+ * Optional packet-trailer metadata carried alongside a video frame.
+ *
+ * Each field is independently optional because the corresponding transport
+ * feature can be negotiated separately.
+ */
+struct VideoFrameMetadata {
+  std::optional<std::uint64_t> user_timestamp;
+  std::optional<std::uint32_t> frame_id;
+};
+
+/**
+ * Capture options for a single outbound video frame.
+ */
+struct VideoCaptureOptions {
+  std::int64_t timestamp_us = 0;
+  VideoRotation rotation = VideoRotation::VIDEO_ROTATION_0;
+  std::optional<VideoFrameMetadata> metadata;
+};
+
 /**
  * Represents a real-time video source that can accept frames from the
  * application and feed them into the LiveKit core.
@@ -69,14 +90,13 @@ class VideoSource {
   /**
    * Push a VideoFrame into the FFI video source.
    *
-   * @param frame         Video frame to send.
-   * @param timestamp_us  Optional timestamp in microseconds.
-   * @param rotation      Video rotation enum.
-   * @param timeout_ms    Controls waiting behavior:
-   *
-   * Notes:
-   *   - Fire-and-forget to send a frame to FFI
-   *     lifetime correctly (e.g., persistent frame pools, GPU buffers, etc.).
+   * @param frame    Video frame to send.
+   * @param options  Timestamp, rotation, and optional metadata for this frame.
+   */
+  void captureFrame(const VideoFrame &frame, const VideoCaptureOptions &options);
+
+  /**
+   * Backward-compatible convenience overload for timestamp + rotation only.
    */
   void captureFrame(const VideoFrame &frame, std::int64_t timestamp_us = 0,
                     VideoRotation rotation = VideoRotation::VIDEO_ROTATION_0);

include/livekit/subscription_thread_dispatcher.h

@@ -19,6 +19,7 @@
 #include <condition_variable>
 #include <cstdint>
 #include <deque>
+#include <functional>
 #include <memory>
 #include <mutex>
 #include <optional>
@@ -34,10 +35,17 @@ namespace livekit {
 // A single video frame event delivered by VideoStream::read().
 struct VideoFrameEvent {
   VideoFrame frame;
+  // WebRTC frame timestamp in microseconds.
+  // This may be translated onto WebRTC's internal capture-time timeline and
+  // should not be expected to match application-provided metadata such as
+  // VideoFrameMetadata::user_timestamp exactly.
   std::int64_t timestamp_us;
   VideoRotation rotation;
+  std::optional<VideoFrameMetadata> metadata;
 };
 
+using VideoFrameEventCallback = std::function<void(const VideoFrameEvent &)>;
+
 namespace proto {
 class FfiEvent;
 }

include/livekit/video_source.h

@@ -32,8 +32,8 @@
 #include "data_track.pb.h"
 #include "ffi.pb.h"
 #include "ffi_client.h"
-#include "lk_log.h"
 #include "livekit_ffi.h"
+#include "lk_log.h"
 #include "room.pb.h"
 #include "room_proto_converter.h"
 #include "trace/trace_event.h"
@@ -310,7 +310,17 @@ void Room::setOnVideoFrameCallback(const std::string &participant_identity,
                                    VideoStream::Options opts) {
   if (subscription_thread_dispatcher_) {
     subscription_thread_dispatcher_->setOnVideoFrameCallback(
-        participant_identity, track_name, std::move(callback), opts);
+        participant_identity, track_name, std::move(callback), std::move(opts));
+  }
+}
+
+void Room::setOnVideoFrameEventCallback(const std::string &participant_identity,
+                                        TrackSource source,
+                                        VideoFrameEventCallback callback,
+                                        VideoStream::Options opts) {
+  if (subscription_thread_dispatcher_) {
+    subscription_thread_dispatcher_->setOnVideoFrameEventCallback(
+        participant_identity, source, std::move(callback), std::move(opts));
   }
 }
 

include/livekit/video_stream.h

@@ -22,6 +22,40 @@
 
 namespace livekit {
 
+namespace {
+
+std::vector<proto::PacketTrailerFeature>
+toProto(const PacketTrailerFeatures &features) {
+  std::vector<proto::PacketTrailerFeature> out;
+  if (features.user_timestamp) {
+    out.push_back(proto::PacketTrailerFeature::PTF_USER_TIMESTAMP);
+  }
+  if (features.frame_id) {
+    out.push_back(proto::PacketTrailerFeature::PTF_FRAME_ID);
+  }
+  return out;
+}
+
+PacketTrailerFeatures
+fromProto(const google::protobuf::RepeatedField<int> &features) {
+  PacketTrailerFeatures out;
+  for (int feature : features) {
+    switch (static_cast<proto::PacketTrailerFeature>(feature)) {
+    case proto::PacketTrailerFeature::PTF_USER_TIMESTAMP:
+      out.user_timestamp = true;
+      break;
+    case proto::PacketTrailerFeature::PTF_FRAME_ID:
+      out.frame_id = true;
+      break;
+    default:
+      break;
+    }
+  }
+  return out;
+}
+
+} // namespace
+
 // --------- enum conversions ---------
 
 ConnectionQuality toConnectionQuality(proto::ConnectionQuality in) {
@@ -340,6 +374,9 @@ proto::TrackPublishOptions toProto(const TrackPublishOptions &in) {
   if (in.preconnect_buffer) {
     msg.set_preconnect_buffer(*in.preconnect_buffer);
   }
+  for (proto::PacketTrailerFeature feature : toProto(in.packet_trailer_features)) {
+    msg.add_packet_trailer_features(feature);
+  }
   return msg;
 }
 
@@ -372,6 +409,7 @@ TrackPublishOptions fromProto(const proto::TrackPublishOptions &in) {
   if (in.has_preconnect_buffer()) {
     out.preconnect_buffer = in.preconnect_buffer();
   }
+  out.packet_trailer_features = fromProto(in.packet_trailer_features());
   return out;
 }