user_timestamp FFI

livekit-ffi-node-bindings/proto/room_pb.d.ts

@@ -20,7 +20,7 @@
 import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf";
 import { Message, proto2 } from "@bufbuild/protobuf";
 import type { DisconnectReason, OwnedParticipant, ParticipantInfo, ParticipantPermission } from "./participant_pb.js";
-import type { OwnedTrack, OwnedTrackPublication, TrackSource } from "./track_pb.js";
+import type { OwnedTrack, OwnedTrackPublication, PacketTrailerFeature, TrackSource } from "./track_pb.js";
 import type { RtcStats } from "./stats_pb.js";
 import type { VideoCodec } from "./video_frame_pb.js";
 import type { E2eeOptions, EncryptionState } from "./e2ee_pb.js";
@@ -1622,6 +1622,11 @@ export declare class TrackPublishOptions extends Message<TrackPublishOptions> {
    */
   preconnectBuffer?: boolean;
 
+  /**
+   * @generated from field: repeated livekit.proto.PacketTrailerFeature packet_trailer_features = 10;
+   */
+  packetTrailerFeatures: PacketTrailerFeature[];
+
   constructor(data?: PartialMessage<TrackPublishOptions>);
 
   static readonly runtime: typeof proto2;
@@ -4102,4 +4107,3 @@ export declare class TextStreamOpened extends Message<TextStreamOpened> {
 
   static equals(a: TextStreamOpened | PlainMessage<TextStreamOpened> | undefined, b: TextStreamOpened | PlainMessage<TextStreamOpened> | undefined): boolean;
 }
-

livekit-ffi-node-bindings/proto/room_pb.js

@@ -22,7 +22,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 
 const { proto2 } = require("@bufbuild/protobuf");
 const { DisconnectReason, OwnedParticipant, ParticipantInfo, ParticipantPermission } = require("./participant_pb.js");
-const { OwnedTrack, OwnedTrackPublication, TrackSource } = require("./track_pb.js");
+const { OwnedTrack, OwnedTrackPublication, PacketTrailerFeature, TrackSource } = require("./track_pb.js");
 const { RtcStats } = require("./stats_pb.js");
 const { VideoCodec } = require("./video_frame_pb.js");
 const { E2eeOptions, EncryptionState } = require("./e2ee_pb.js");
@@ -640,6 +640,7 @@ const TrackPublishOptions = /*@__PURE__*/ proto2.makeMessageType(
     { no: 7, name: "source", kind: "enum", T: proto2.getEnumType(TrackSource), opt: true },
     { no: 8, name: "stream", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true },
     { no: 9, name: "preconnect_buffer", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true },
+    { no: 10, name: "packet_trailer_features", kind: "enum", T: proto2.getEnumType(PacketTrailerFeature), repeated: true },
   ],
 );
 

livekit-ffi-node-bindings/proto/video_frame_pb.d.ts

@@ -402,6 +402,11 @@ export declare class CaptureVideoFrameRequest extends Message<CaptureVideoFrameR
    */
   rotation?: VideoRotation;
 
+  /**
+   * @generated from field: optional livekit.proto.FrameMetadata metadata = 5;
+   */
+  metadata?: FrameMetadata;
+
   constructor(data?: PartialMessage<CaptureVideoFrameRequest>);
 
   static readonly runtime: typeof proto2;
@@ -654,6 +659,35 @@ export declare class OwnedVideoBuffer extends Message<OwnedVideoBuffer> {
   static equals(a: OwnedVideoBuffer | PlainMessage<OwnedVideoBuffer> | undefined, b: OwnedVideoBuffer | PlainMessage<OwnedVideoBuffer> | undefined): boolean;
 }
 
+/**
+ * @generated from message livekit.proto.FrameMetadata
+ */
+export declare class FrameMetadata extends Message<FrameMetadata> {
+  /**
+   * @generated from field: optional uint64 user_timestamp_us = 1;
+   */
+  userTimestampUs?: bigint;
+
+  /**
+   * @generated from field: optional uint32 frame_id = 2;
+   */
+  frameId?: number;
+
+  constructor(data?: PartialMessage<FrameMetadata>);
+
+  static readonly runtime: typeof proto2;
+  static readonly typeName = "livekit.proto.FrameMetadata";
+  static readonly fields: FieldList;
+
+  static fromBinary(bytes: Uint8Array, options?: Partial<BinaryReadOptions>): FrameMetadata;
+
+  static fromJson(jsonValue: JsonValue, options?: Partial<JsonReadOptions>): FrameMetadata;
+
+  static fromJsonString(jsonString: string, options?: Partial<JsonReadOptions>): FrameMetadata;
+
+  static equals(a: FrameMetadata | PlainMessage<FrameMetadata> | undefined, b: FrameMetadata | PlainMessage<FrameMetadata> | undefined): boolean;
+}
+
 /**
  * @generated from message livekit.proto.VideoStreamInfo
  */
@@ -769,6 +803,11 @@ export declare class VideoFrameReceived extends Message<VideoFrameReceived> {
    */
   rotation?: VideoRotation;
 
+  /**
+   * @generated from field: optional livekit.proto.FrameMetadata metadata = 4;
+   */
+  metadata?: FrameMetadata;
+
   constructor(data?: PartialMessage<VideoFrameReceived>);
 
   static readonly runtime: typeof proto2;
@@ -884,4 +923,3 @@ export declare class OwnedVideoSource extends Message<OwnedVideoSource> {
 
   static equals(a: OwnedVideoSource | PlainMessage<OwnedVideoSource> | undefined, b: OwnedVideoSource | PlainMessage<OwnedVideoSource> | undefined): boolean;
 }
-

livekit-ffi-node-bindings/proto/video_frame_pb.js

@@ -185,6 +185,7 @@ const CaptureVideoFrameRequest = /*@__PURE__*/ proto2.makeMessageType(
     { no: 2, name: "buffer", kind: "message", T: VideoBufferInfo, req: true },
     { no: 3, name: "timestamp_us", kind: "scalar", T: 3 /* ScalarType.INT64 */, req: true },
     { no: 4, name: "rotation", kind: "enum", T: proto2.getEnumType(VideoRotation), req: true },
+    { no: 5, name: "metadata", kind: "message", T: FrameMetadata, opt: true },
   ],
 );
 
@@ -270,6 +271,17 @@ const OwnedVideoBuffer = /*@__PURE__*/ proto2.makeMessageType(
   ],
 );
 
+/**
+ * @generated from message livekit.proto.FrameMetadata
+ */
+const FrameMetadata = /*@__PURE__*/ proto2.makeMessageType(
+  "livekit.proto.FrameMetadata",
+  () => [
+    { no: 1, name: "user_timestamp_us", kind: "scalar", T: 4 /* ScalarType.UINT64 */, opt: true },
+    { no: 2, name: "frame_id", kind: "scalar", T: 13 /* ScalarType.UINT32 */, opt: true },
+  ],
+);
+
 /**
  * @generated from message livekit.proto.VideoStreamInfo
  */
@@ -312,6 +324,7 @@ const VideoFrameReceived = /*@__PURE__*/ proto2.makeMessageType(
     { no: 1, name: "buffer", kind: "message", T: OwnedVideoBuffer, req: true },
     { no: 2, name: "timestamp_us", kind: "scalar", T: 3 /* ScalarType.INT64 */, req: true },
     { no: 3, name: "rotation", kind: "enum", T: proto2.getEnumType(VideoRotation), req: true },
+    { no: 4, name: "metadata", kind: "message", T: FrameMetadata, opt: true },
   ],
 );
 
@@ -375,6 +388,7 @@ exports.VideoResolution = VideoResolution;
 exports.VideoBufferInfo = VideoBufferInfo;
 exports.VideoBufferInfo_ComponentInfo = VideoBufferInfo_ComponentInfo;
 exports.OwnedVideoBuffer = OwnedVideoBuffer;
+exports.FrameMetadata = FrameMetadata;
 exports.VideoStreamInfo = VideoStreamInfo;
 exports.OwnedVideoStream = OwnedVideoStream;
 exports.VideoStreamEvent = VideoStreamEvent;

livekit-ffi/protocol/room.proto

@@ -278,6 +278,7 @@ message TrackPublishOptions {
   optional TrackSource source = 7;
   optional string stream = 8;
   optional bool preconnect_buffer = 9;
+  repeated PacketTrailerFeature packet_trailer_features = 10;
 }
 
 enum IceTransportType {

livekit-ffi/protocol/video_frame.proto

@@ -59,6 +59,7 @@ message CaptureVideoFrameRequest {
   required VideoBufferInfo buffer = 2;
   required int64 timestamp_us = 3; // In microseconds
   required VideoRotation rotation = 4;
+  optional FrameMetadata metadata = 5;
 }
 
 message CaptureVideoFrameResponse {}
@@ -135,6 +136,11 @@ message OwnedVideoBuffer {
   required VideoBufferInfo info = 2;
 }
 
+message FrameMetadata {
+  optional uint64 user_timestamp_us = 1;
+  optional uint32 frame_id = 2;
+}
+
 //
 // VideoStream
 //
@@ -166,6 +172,7 @@ message VideoFrameReceived {
   required OwnedVideoBuffer buffer = 1;
   required int64 timestamp_us = 2; // In microseconds
   required VideoRotation rotation = 3;
+  optional FrameMetadata metadata = 4;
 }
 
 message VideoStreamEOS {}

livekit-ffi/src/conversion/room.rs

@@ -18,7 +18,7 @@ use livekit::{
         key_provider::{KeyProvider, KeyProviderOptions},
         E2eeOptions, EncryptionType,
     },
-    options::{AudioEncoding, TrackPublishOptions, VideoEncoding},
+    options::{AudioEncoding, PacketTrailerFeatures, TrackPublishOptions, VideoEncoding},
     prelude::*,
     webrtc::{
         native::frame_cryptor::{EncryptionState, KeyDerivationAlgorithm},
@@ -28,6 +28,25 @@ use livekit::{
 };
 use std::time::Duration;
 
+fn packet_trailer_features_from_proto(features: Vec<i32>) -> PacketTrailerFeatures {
+    let mut packet_trailer_features = PacketTrailerFeatures::default();
+
+    for feature in
+        features.into_iter().filter_map(|value| proto::PacketTrailerFeature::try_from(value).ok())
+    {
+        match feature {
+            proto::PacketTrailerFeature::PtfUserTimestamp => {
+                packet_trailer_features.user_timestamp = true;
+            }
+            proto::PacketTrailerFeature::PtfFrameId => {
+                packet_trailer_features.frame_id = true;
+            }
+        }
+    }
+
+    packet_trailer_features
+}
+
 impl From<EncryptionState> for proto::EncryptionState {
     fn from(value: EncryptionState) -> Self {
         match value {
@@ -294,7 +313,9 @@ impl From<proto::TrackPublishOptions> for TrackPublishOptions {
             preconnect_buffer: opts
                 .preconnect_buffer
                 .unwrap_or(default_publish_options.preconnect_buffer),
-            packet_trailer_features: default_publish_options.packet_trailer_features,
+            packet_trailer_features: packet_trailer_features_from_proto(
+                opts.packet_trailer_features,
+            ),
         }
     }
 }
@@ -311,6 +332,31 @@ impl From<proto::AudioEncoding> for AudioEncoding {
     }
 }
 
+#[cfg(test)]
+mod tests {
+    use super::packet_trailer_features_from_proto;
+    use crate::proto;
+
+    #[test]
+    fn packet_trailer_features_default_to_empty() {
+        let features = packet_trailer_features_from_proto(Vec::new());
+
+        assert!(!features.user_timestamp);
+        assert!(!features.frame_id);
+    }
+
+    #[test]
+    fn packet_trailer_features_enable_known_flags() {
+        let features = packet_trailer_features_from_proto(vec![
+            proto::PacketTrailerFeature::PtfUserTimestamp.into(),
+            proto::PacketTrailerFeature::PtfFrameId.into(),
+        ]);
+
+        assert!(features.user_timestamp);
+        assert!(features.frame_id);
+    }
+}
+
 impl From<&FfiRoom> for proto::RoomInfo {
     fn from(value: &FfiRoom) -> Self {
         let room = &value.inner.room;

livekit-ffi/src/server/video_source.rs

@@ -14,7 +14,10 @@
 
 use super::{colorcvt, FfiHandle};
 use crate::{proto, server, FfiError, FfiHandleId, FfiResult};
-use livekit::webrtc::{prelude::*, video_frame::VideoFrame};
+use livekit::webrtc::{
+    prelude::*,
+    video_frame::{FrameMetadata, VideoFrame},
+};
 
 pub struct FfiVideoSource {
     pub handle_id: FfiHandleId,
@@ -24,6 +27,17 @@ pub struct FfiVideoSource {
 
 impl FfiHandle for FfiVideoSource {}
 
+fn frame_metadata_from_proto(metadata: Option<proto::FrameMetadata>) -> Option<FrameMetadata> {
+    let metadata = metadata?;
+    let frame_metadata = FrameMetadata {
+        user_timestamp_us: metadata.user_timestamp_us,
+        frame_id: metadata.frame_id,
+    };
+
+    (frame_metadata.user_timestamp_us.is_some() || frame_metadata.frame_id.is_some())
+        .then_some(frame_metadata)
+}
+
 impl FfiVideoSource {
     pub fn setup(
         server: &'static server::FfiServer,
@@ -67,7 +81,7 @@ impl FfiVideoSource {
                 let frame = VideoFrame {
                     rotation: capture.rotation().into(),
                     timestamp_us: capture.timestamp_us,
-                    frame_metadata: None,
+                    frame_metadata: frame_metadata_from_proto(capture.metadata),
                     buffer,
                 };
 
@@ -78,3 +92,26 @@ impl FfiVideoSource {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::frame_metadata_from_proto;
+    use crate::proto;
+
+    #[test]
+    fn empty_proto_frame_metadata_is_ignored() {
+        assert!(frame_metadata_from_proto(Some(proto::FrameMetadata::default())).is_none());
+    }
+
+    #[test]
+    fn proto_frame_metadata_preserves_present_fields() {
+        let metadata = frame_metadata_from_proto(Some(proto::FrameMetadata {
+            user_timestamp_us: Some(123),
+            frame_id: Some(456),
+        }))
+        .unwrap();
+
+        assert_eq!(metadata.user_timestamp_us, Some(123));
+        assert_eq!(metadata.frame_id, Some(456));
+    }
+}