wip stream compat for transcription segments

packages/core/etc/components-core.api.md

@@ -159,6 +159,7 @@ export const cssPrefix = "lk";
 // @public (undocumented)
 export const DataTopic: {
     readonly CHAT: "lk.chat";
+    readonly TRANSCRIPTION: "lk.transcription";
 };
 
 // @public (undocumented)

packages/core/src/components/chat.ts

@@ -8,6 +8,7 @@ import {
   sendMessage,
   setupDataMessageHandler,
 } from '../observables/dataChannel';
+import { log } from '../logger';
 
 /** @public */
 export type { ChatMessage };
@@ -165,10 +166,14 @@ export function setupChat(room: Room, options?: ChatOptions) {
         ...chatMsg,
         ignoreLegacy: serverSupportsDataStreams(),
       });
-      await sendMessage(room.localParticipant, encodedLegacyMsg, {
-        reliable: true,
-        topic: legacyTopic,
-      });
+      try {
+        await sendMessage(room.localParticipant, encodedLegacyMsg, {
+          reliable: true,
+          topic: legacyTopic,
+        });
+      } catch (error) {
+        log.info('could not send message in legacy chat format', error);
+      }
       return chatMsg;
     } finally {
       isSending$.next(false);

packages/core/src/components/textStream.ts

@@ -97,6 +97,7 @@ export function setupTextStream(room: Room, topic: string): Observable<TextStrea
 
   // Add cleanup when room is disconnected
   room.once(RoomEvent.Disconnected, () => {
+    room.unregisterTextStreamHandler(topic);
     textStreamsSubject.complete();
     getObservableCache().delete(cacheKey);
   });

packages/core/src/observables/dataChannel.ts

@@ -13,6 +13,7 @@ import { ReceivedChatMessage } from '../components/chat';
 
 export const DataTopic = {
   CHAT: 'lk.chat',
+  TRANSCRIPTION: 'lk.transcription',
 } as const;
 
 /** @deprecated */

packages/react/etc/components-react.api.md

@@ -1213,6 +1213,17 @@ export function useTrackTranscription(trackRef: TrackReferenceOrPlaceholder_4 |
 // @alpha
 export function useTrackVolume(trackOrTrackReference?: LocalAudioTrack | RemoteAudioTrack | TrackReference_3, options?: AudioAnalyserOptions): number;
 
+// @beta
+export function useTranscriptions(opts?: UseTranscriptionsOptions): TextStreamData_2[];
+
+// @beta (undocumented)
+export interface UseTranscriptionsOptions {
+    // (undocumented)
+    participantIdentities?: string[];
+    // (undocumented)
+    trackSids?: string[];
+}
+
 // @public
 export function useVisualStableUpdate(
 trackReferences: TrackReferenceOrPlaceholder_4[], maxItemsOnPage: number, options?: UseVisualStableUpdateOptions): TrackReferenceOrPlaceholder_4[];

packages/react/src/hooks/index.ts

@@ -55,3 +55,4 @@ export * from './useVoiceAssistant';
 export * from './useParticipantAttributes';
 export * from './useIsRecording';
 export * from './useTextStream';
+export * from './useTranscriptions';

packages/react/src/hooks/useTrackTranscription.ts

@@ -11,6 +11,7 @@ import {
 import type { TranscriptionSegment } from 'livekit-client';
 import * as React from 'react';
 import { useTrackSyncTime } from './useTrackSyncTime';
+import { useTranscriptions } from './useTranscriptions';
 
 /**
  * @alpha
@@ -35,19 +36,58 @@ const TRACK_TRANSCRIPTION_DEFAULTS = {
 } as const satisfies TrackTranscriptionOptions;
 
 /**
- * @returns An object consisting of `segments` with maximum length of opts.windowLength and `activeSegments` that are valid for the current track timestamp
+ * @returns An object consisting of `segments` with maximum length of opts.bufferSize
  * @alpha
  */
 export function useTrackTranscription(
   trackRef: TrackReferenceOrPlaceholder | undefined,
   options?: TrackTranscriptionOptions,
+) {
+  const legacyTranscription = useLegacyTranscription(trackRef, options);
+
+  const participantIdentities = React.useMemo(() => {
+    if (!trackRef) {
+      return [];
+    }
+    return [trackRef.participant.identity];
+  }, [trackRef]);
+  const trackSids = React.useMemo(() => {
+    if (!trackRef) {
+      return [];
+    }
+    return [getTrackReferenceId(trackRef)];
+  }, [trackRef]);
+  const useTranscriptionsOptions = React.useMemo(() => {
+    return { participantIdentities, trackSids };
+  }, [participantIdentities, trackSids]);
+  const transcriptions = useTranscriptions(useTranscriptionsOptions);
+
+  const streamAsSegments = transcriptions.map((t) => {
+    const legacySegment: ReceivedTranscriptionSegment = {
+      id: t.streamInfo.id,
+      text: t.text,
+      receivedAt: t.streamInfo.timestamp,
+      receivedAtMediaTimestamp: 0,
+      language: '',
+      startTime: 0,
+      endTime: 0,
+      final: false,
+      firstReceivedTime: 0,
+      lastReceivedTime: 0,
+    };
+    return legacySegment;
+  });
+
+  return streamAsSegments.length > 0 ? { segments: streamAsSegments } : legacyTranscription;
+}
+
+function useLegacyTranscription(
+  trackRef: TrackReferenceOrPlaceholder | undefined,
+  options?: TrackTranscriptionOptions,
 ) {
   const opts = { ...TRACK_TRANSCRIPTION_DEFAULTS, ...options };
   const [segments, setSegments] = React.useState<Array<ReceivedTranscriptionSegment>>([]);
-  // const [activeSegments, setActiveSegments] = React.useState<Array<ReceivedTranscriptionSegment>>(
-  //   [],
-  // );
-  // const prevActiveSegments = React.useRef<ReceivedTranscriptionSegment[]>([]);
+
   const syncTimestamps = useTrackSyncTime(trackRef);
   const handleSegmentMessage = (newSegments: TranscriptionSegment[]) => {
     opts.onTranscription?.(newSegments);
@@ -72,20 +112,5 @@ export function useTrackTranscription(
     };
   }, [trackRef && getTrackReferenceId(trackRef), handleSegmentMessage]);
 
-  // React.useEffect(() => {
-  //   if (syncTimestamps) {
-  //     const newActiveSegments = getActiveTranscriptionSegments(
-  //       segments,
-  //       syncTimestamps,
-  //       opts.maxAge,
-  //     );
-  //     // only update active segment array if content actually changed
-  //     if (didActiveSegmentsChange(prevActiveSegments.current, newActiveSegments)) {
-  //       setActiveSegments(newActiveSegments);
-  //       prevActiveSegments.current = newActiveSegments;
-  //     }
-  //   }
-  // }, [syncTimestamps, segments, opts.maxAge]);
-
   return { segments };
 }

packages/react/src/hooks/useTranscriptions.ts

@@ -0,0 +1,44 @@
+import * as React from 'react';
+import { useTextStream } from './useTextStream';
+import { DataTopic } from '@livekit/components-core';
+
+/**
+ * @beta
+ */
+export interface UseTranscriptionsOptions {
+  participantIdentities?: string[];
+  trackSids?: string[];
+}
+
+/**
+ * @beta
+ * useTranscriptions is a hook that returns the transcriptions for the given participant identities and track sids,
+ * if no options are provided, it will return all transcriptions
+ * @example
+ * ```tsx
+ * const transcriptions = useTranscriptions();
+ * return <div>{transcriptions.map((transcription) => transcription.text)}</div>;
+ * ```
+ */
+export function useTranscriptions(opts?: UseTranscriptionsOptions) {
+  const { participantIdentities, trackSids } = opts ?? {};
+  const { textStreams } = useTextStream(DataTopic.TRANSCRIPTION);
+
+  const filteredMessages = React.useMemo(
+    () =>
+      textStreams
+        .filter((stream) =>
+          participantIdentities
+            ? participantIdentities.includes(stream.participantInfo.identity)
+            : true,
+        )
+        .filter((stream) =>
+          trackSids
+            ? trackSids.includes(stream.streamInfo.attributes?.['lk.transcribed_track_id'] ?? '')
+            : true,
+        ),
+    [textStreams, participantIdentities, trackSids],
+  );
+
+  return filteredMessages;
+}