Add fallback for addTrack if addTransceiver is not supported by a…

… device (#403)

* wip

* move sender logic into RTCEngine

* renaming, make methods private

* remove debug config

* address comments

* remove unused code

* changeset

.changeset/wicked-toys-reflect.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Add fallback for addTrack if addTransceiver is not supported

src/index.ts

@@ -15,7 +15,13 @@ import RemoteTrack from './room/track/RemoteTrack';
 import RemoteTrackPublication from './room/track/RemoteTrackPublication';
 import RemoteVideoTrack, { ElementInfo } from './room/track/RemoteVideoTrack';
 import { TrackPublication } from './room/track/TrackPublication';
-import { getEmptyAudioStreamTrack, getEmptyVideoStreamTrack } from './room/utils';
+import {
+  getEmptyAudioStreamTrack,
+  getEmptyVideoStreamTrack,
+  isDeviceSupported,
+  supportsAdaptiveStream,
+  supportsDynacast,
+} from './room/utils';
 
 export * from './options';
 export * from './room/errors';
@@ -30,6 +36,9 @@ export {
   setLogExtension,
   getEmptyAudioStreamTrack,
   getEmptyVideoStreamTrack,
+  isDeviceSupported,
+  supportsAdaptiveStream,
+  supportsDynacast,
   LogLevel,
   Room,
   ConnectionState,

src/proto/google/protobuf/timestamp.ts

@@ -199,7 +199,7 @@ export type DeepPartial<T> = T extends Builtin
 type KeysOfUnion<T> = T extends T ? keyof T : never;
 export type Exact<P, I extends P> = P extends Builtin
   ? P
-  : P & { [K in keyof P]: Exact<P[K], I[K]> } & Record<Exclude<keyof I, KeysOfUnion<P>>, never>;
+  : P & { [K in keyof P]: Exact<P[K], I[K]> } & { [K in Exclude<keyof I, KeysOfUnion<P>>]: never };
 
 function longToNumber(long: Long): number {
   if (long.gt(Number.MAX_SAFE_INTEGER)) {

src/proto/livekit_models.ts

@@ -2921,25 +2921,29 @@ var globalThis: any = (() => {
   throw 'Unable to locate global object';
 })();
 
-const atob: (b64: string) => string =
-  globalThis.atob || ((b64) => globalThis.Buffer.from(b64, 'base64').toString('binary'));
 function bytesFromBase64(b64: string): Uint8Array {
-  const bin = atob(b64);
-  const arr = new Uint8Array(bin.length);
-  for (let i = 0; i < bin.length; ++i) {
-    arr[i] = bin.charCodeAt(i);
+  if (globalThis.Buffer) {
+    return Uint8Array.from(globalThis.Buffer.from(b64, 'base64'));
+  } else {
+    const bin = globalThis.atob(b64);
+    const arr = new Uint8Array(bin.length);
+    for (let i = 0; i < bin.length; ++i) {
+      arr[i] = bin.charCodeAt(i);
+    }
+    return arr;
   }
-  return arr;
 }
 
-const btoa: (bin: string) => string =
-  globalThis.btoa || ((bin) => globalThis.Buffer.from(bin, 'binary').toString('base64'));
 function base64FromBytes(arr: Uint8Array): string {
-  const bin: string[] = [];
-  arr.forEach((byte) => {
-    bin.push(String.fromCharCode(byte));
-  });
-  return btoa(bin.join(''));
+  if (globalThis.Buffer) {
+    return globalThis.Buffer.from(arr).toString('base64');
+  } else {
+    const bin: string[] = [];
+    arr.forEach((byte) => {
+      bin.push(String.fromCharCode(byte));
+    });
+    return globalThis.btoa(bin.join(''));
+  }
 }
 
 type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
@@ -2959,7 +2963,7 @@ export type DeepPartial<T> = T extends Builtin
 type KeysOfUnion<T> = T extends T ? keyof T : never;
 export type Exact<P, I extends P> = P extends Builtin
   ? P
-  : P & { [K in keyof P]: Exact<P[K], I[K]> } & Record<Exclude<keyof I, KeysOfUnion<P>>, never>;
+  : P & { [K in keyof P]: Exact<P[K], I[K]> } & { [K in Exclude<keyof I, KeysOfUnion<P>>]: never };
 
 function toTimestamp(date: Date): Timestamp {
   const seconds = date.getTime() / 1_000;

src/proto/livekit_rtc.ts

@@ -3480,7 +3480,7 @@ export type DeepPartial<T> = T extends Builtin
 type KeysOfUnion<T> = T extends T ? keyof T : never;
 export type Exact<P, I extends P> = P extends Builtin
   ? P
-  : P & { [K in keyof P]: Exact<P[K], I[K]> } & Record<Exclude<keyof I, KeysOfUnion<P>>, never>;
+  : P & { [K in keyof P]: Exact<P[K], I[K]> } & { [K in Exclude<keyof I, KeysOfUnion<P>>]: never };
 
 function longToNumber(long: Long): number {
   if (long.gt(Number.MAX_SAFE_INTEGER)) {

src/room/RTCEngine.ts

@@ -25,7 +25,11 @@ import { ConnectionError, TrackInvalidError, UnexpectedConnectionState } from '.
 import { EngineEvent } from './events';
 import PCTransport from './PCTransport';
 import { ReconnectContext, ReconnectPolicy } from './ReconnectPolicy';
-import { isWeb, sleep } from './utils';
+import LocalTrack from './track/LocalTrack';
+import LocalVideoTrack, { SimulcastTrackInfo } from './track/LocalVideoTrack';
+import { TrackPublishOptions, VideoCodec } from './track/options';
+import { Track } from './track/Track';
+import { isWeb, sleep, supportsAddTrack, supportsTransceiver } from './utils';
 
 const lossyDataChannel = '_lossy';
 const reliableDataChannel = '_reliable';
@@ -475,6 +479,142 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     }
   };
 
+  private setPreferredCodec(
+    transceiver: RTCRtpTransceiver,
+    kind: Track.Kind,
+    videoCodec: VideoCodec,
+  ) {
+    if (!('getCapabilities' in RTCRtpSender)) {
+      return;
+    }
+    const cap = RTCRtpSender.getCapabilities(kind);
+    if (!cap) return;
+    log.debug('get capabilities', cap);
+    const matched: RTCRtpCodecCapability[] = [];
+    const partialMatched: RTCRtpCodecCapability[] = [];
+    const unmatched: RTCRtpCodecCapability[] = [];
+    cap.codecs.forEach((c) => {
+      const codec = c.mimeType.toLowerCase();
+      if (codec === 'audio/opus') {
+        matched.push(c);
+        return;
+      }
+      const matchesVideoCodec = codec === `video/${videoCodec}`;
+      if (!matchesVideoCodec) {
+        unmatched.push(c);
+        return;
+      }
+      // for h264 codecs that have sdpFmtpLine available, use only if the
+      // profile-level-id is 42e01f for cross-browser compatibility
+      if (videoCodec === 'h264') {
+        if (c.sdpFmtpLine && c.sdpFmtpLine.includes('profile-level-id=42e01f')) {
+          matched.push(c);
+        } else {
+          partialMatched.push(c);
+        }
+        return;
+      }
+
+      matched.push(c);
+    });
+
+    if ('setCodecPreferences' in transceiver) {
+      transceiver.setCodecPreferences(matched.concat(partialMatched, unmatched));
+    }
+  }
+
+  async createSender(
+    track: LocalTrack,
+    opts: TrackPublishOptions,
+    encodings?: RTCRtpEncodingParameters[],
+  ) {
+    if (supportsTransceiver()) {
+      return this.createTransceiverRTCRtpSender(track, opts, encodings);
+    }
+    if (supportsAddTrack()) {
+      log.debug('using add-track fallback');
+      return this.createRTCRtpSender(track.mediaStreamTrack);
+    }
+    throw new UnexpectedConnectionState('Required webRTC APIs not supported on this device');
+  }
+
+  async createSimulcastSender(
+    track: LocalVideoTrack,
+    simulcastTrack: SimulcastTrackInfo,
+    opts: TrackPublishOptions,
+    encodings?: RTCRtpEncodingParameters[],
+  ) {
+    // store RTCRtpSender
+    // @ts-ignore
+    if (supportsTransceiver()) {
+      return this.createSimulcastTransceiverSender(track, simulcastTrack, opts, encodings);
+    }
+    if (supportsAddTrack()) {
+      log.debug('using add-track fallback');
+      return this.createRTCRtpSender(track.mediaStreamTrack);
+    }
+
+    throw new UnexpectedConnectionState('Cannot stream on this device');
+  }
+
+  private async createTransceiverRTCRtpSender(
+    track: LocalTrack,
+    opts: TrackPublishOptions,
+    encodings?: RTCRtpEncodingParameters[],
+  ) {
+    if (!this.publisher) {
+      throw new UnexpectedConnectionState('publisher is closed');
+    }
+
+    const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+    if (encodings) {
+      transceiverInit.sendEncodings = encodings;
+    }
+    // addTransceiver for react-native is async. web is synchronous, but await won't effect it.
+    const transceiver = await this.publisher.pc.addTransceiver(
+      track.mediaStreamTrack,
+      transceiverInit,
+    );
+    if (track.kind === Track.Kind.Video && opts.videoCodec) {
+      this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
+      track.codec = opts.videoCodec;
+    }
+    return transceiver.sender;
+  }
+
+  private async createSimulcastTransceiverSender(
+    track: LocalVideoTrack,
+    simulcastTrack: SimulcastTrackInfo,
+    opts: TrackPublishOptions,
+    encodings?: RTCRtpEncodingParameters[],
+  ) {
+    if (!this.publisher) {
+      throw new UnexpectedConnectionState('publisher is closed');
+    }
+    const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+    if (encodings) {
+      transceiverInit.sendEncodings = encodings;
+    }
+    // addTransceiver for react-native is async. web is synchronous, but await won't effect it.
+    const transceiver = await this.publisher.pc.addTransceiver(
+      simulcastTrack.mediaStreamTrack,
+      transceiverInit,
+    );
+    if (!opts.videoCodec) {
+      return;
+    }
+    this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
+    track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
+    return transceiver.sender;
+  }
+
+  private async createRTCRtpSender(track: MediaStreamTrack) {
+    if (!this.publisher) {
+      throw new UnexpectedConnectionState('publisher is closed');
+    }
+    return this.publisher.pc.addTrack(track);
+  }
+
   // websocket reconnect behavior. if websocket is interrupted, and the PeerConnection
   // continues to work, we can reconnect to websocket to continue the session
   // after a number of retries, we'll close and give up permanently

src/room/participant/LocalParticipant.ts

@@ -30,7 +30,6 @@ import {
   ScreenSharePresets,
   TrackPublishOptions,
   VideoCaptureOptions,
-  VideoCodec,
 } from '../track/options';
 import { Track } from '../track/Track';
 import { constraintsForOptions, mergeDefaultOptions } from '../track/utils';
@@ -552,19 +551,9 @@ export default class LocalParticipant extends Participant {
       throw new UnexpectedConnectionState('publisher is closed');
     }
     log.debug(`publishing ${track.kind} with encodings`, { encodings, trackInfo: ti });
-    const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
-    if (encodings) {
-      transceiverInit.sendEncodings = encodings;
-    }
-    // addTransceiver for react-native is async. web is synchronous, but await won't effect it.
-    const transceiver = await this.engine.publisher.pc.addTransceiver(
-      track.mediaStreamTrack,
-      transceiverInit,
-    );
-    if (track.kind === Track.Kind.Video && opts.videoCodec) {
-      this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
-      track.codec = opts.videoCodec;
-    }
+
+    // store RTPSender
+    track.sender = await this.engine.createSender(track, opts, encodings);
 
     if (track.codec === 'av1' && encodings && encodings[0]?.maxBitrate) {
       this.engine.publisher.setTrackCodecBitrate(
@@ -576,8 +565,6 @@ export default class LocalParticipant extends Participant {
 
     this.engine.negotiate();
 
-    // store RTPSender
-    track.sender = transceiver.sender;
     if (track instanceof LocalVideoTrack) {
       track.startMonitor(this.engine.client);
     } else if (track instanceof LocalAudioTrack) {
@@ -652,20 +639,11 @@ export default class LocalParticipant extends Participant {
 
     const ti = await this.engine.addTrack(req);
 
-    if (!this.engine.publisher) {
-      throw new UnexpectedConnectionState('publisher is closed');
-    }
     const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
     if (encodings) {
       transceiverInit.sendEncodings = encodings;
     }
-    // addTransceiver for react-native is async. web is synchronous, but await won't effect it.
-    const transceiver = await this.engine.publisher.pc.addTransceiver(
-      simulcastTrack.mediaStreamTrack,
-      transceiverInit,
-    );
-    this.setPreferredCodec(transceiver, track.kind, videoCodec);
-    track.setSimulcastTrackSender(videoCodec, transceiver.sender);
+    await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings);
 
     this.engine.negotiate();
     log.debug(`published ${videoCodec} for track ${track.sid}`, { encodings, trackInfo: ti });
@@ -1000,50 +978,6 @@ export default class LocalParticipant extends Participant {
     return publication;
   }
 
-  private setPreferredCodec(
-    transceiver: RTCRtpTransceiver,
-    kind: Track.Kind,
-    videoCodec: VideoCodec,
-  ) {
-    if (!('getCapabilities' in RTCRtpSender)) {
-      return;
-    }
-    const cap = RTCRtpSender.getCapabilities(kind);
-    if (!cap) return;
-    log.debug('get capabilities', cap);
-    const matched: RTCRtpCodecCapability[] = [];
-    const partialMatched: RTCRtpCodecCapability[] = [];
-    const unmatched: RTCRtpCodecCapability[] = [];
-    cap.codecs.forEach((c) => {
-      const codec = c.mimeType.toLowerCase();
-      if (codec === 'audio/opus') {
-        matched.push(c);
-        return;
-      }
-      const matchesVideoCodec = codec === `video/${videoCodec}`;
-      if (!matchesVideoCodec) {
-        unmatched.push(c);
-        return;
-      }
-      // for h264 codecs that have sdpFmtpLine available, use only if the
-      // profile-level-id is 42e01f for cross-browser compatibility
-      if (videoCodec === 'h264') {
-        if (c.sdpFmtpLine && c.sdpFmtpLine.includes('profile-level-id=42e01f')) {
-          matched.push(c);
-        } else {
-          partialMatched.push(c);
-        }
-        return;
-      }
-
-      matched.push(c);
-    });
-
-    if ('setCodecPreferences' in transceiver) {
-      transceiver.setCodecPreferences(matched.concat(partialMatched, unmatched));
-    }
-  }
-
   /** @internal */
   publishedTracksInfo(): TrackPublishedResponse[] {
     const infos: TrackPublishedResponse[] = [];

src/room/utils.ts

@@ -15,6 +15,28 @@ export async function sleep(duration: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, duration));
 }
 
+/** @internal */
+export function supportsTransceiver() {
+  return 'addTransceiver' in RTCPeerConnection;
+}
+
+/** @internal */
+export function supportsAddTrack() {
+  return 'addTrack' in RTCPeerConnection;
+}
+
+export function supportsAdaptiveStream() {
+  return typeof ResizeObserver !== undefined && typeof IntersectionObserver !== undefined;
+}
+
+export function supportsDynacast() {
+  return supportsTransceiver();
+}
+
+export function isDeviceSupported() {
+  return supportsTransceiver() || supportsAddTrack();
+}
+
 export function isFireFox(): boolean {
   if (!isWeb()) return false;
   return navigator.userAgent.indexOf('Firefox') !== -1;