Improve handling of incompatible published codecs (#911)

When the client publishes a codec that the server doesn't support (or if
server determines the client isn't compatible with it), we would reject
those track publications.

However, the better thing to do from a user experience standpoint is to
publish with another compatible codec. This PR requires v1.5.1

.changeset/purple-coats-attend.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Improve handling of incompatible published codecs

src/e2ee/E2eeManager.ts

@@ -11,6 +11,7 @@ import LocalTrack from '../room/track/LocalTrack';
 import type RemoteTrack from '../room/track/RemoteTrack';
 import type { Track } from '../room/track/Track';
 import type { VideoCodec } from '../room/track/options';
+import { mimeTypeToVideoCodecString } from '../room/track/utils';
 import type { BaseKeyProvider } from './KeyProvider';
 import { E2EE_FLAG } from './constants';
 import { type E2EEManagerCallbacks, EncryptionEvent, KeyProviderEvent } from './events';
@@ -28,7 +29,7 @@ import type {
   SifTrailerMessage,
   UpdateCodecMessage,
 } from './types';
-import { isE2EESupported, isScriptTransformSupported, mimeTypeToVideoCodecString } from './utils';
+import { isE2EESupported, isScriptTransformSupported } from './utils';
 
 /**
  * @experimental

src/e2ee/utils.ts

@@ -1,5 +1,3 @@
-import { videoCodecs } from '../room/track/options';
-import type { VideoCodec } from '../room/track/options';
 import { ENCRYPTION_ALGORITHM } from './constants';
 
 export function isE2EESupported() {
@@ -116,14 +114,6 @@ export function createE2EEKey(): Uint8Array {
   return window.crypto.getRandomValues(new Uint8Array(32));
 }
 
-export function mimeTypeToVideoCodecString(mimeType: string) {
-  const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec;
-  if (!videoCodecs.includes(codec)) {
-    throw Error(`Video codec not supported: ${codec}`);
-  }
-  return codec;
-}
-
 /**
  * Ratchets a key. See
  * https://tools.ietf.org/html/draft-omara-sframe-00#section-4.3.5.1

src/room/defaults.ts

@@ -1,11 +1,13 @@
 import type { InternalRoomConnectOptions, InternalRoomOptions } from '../options';
 import DefaultReconnectPolicy from './DefaultReconnectPolicy';
-import { AudioPresets, ScreenSharePresets, VideoPresets } from './track/options';
 import type {
   AudioCaptureOptions,
   TrackPublishDefaults,
   VideoCaptureOptions,
 } from './track/options';
+import { AudioPresets, ScreenSharePresets, VideoPresets } from './track/options';
+
+export const defaultVideoCodec = 'vp8';
 
 export const publishDefaults: TrackPublishDefaults = {
   /**
@@ -19,7 +21,7 @@ export const publishDefaults: TrackPublishDefaults = {
   simulcast: true,
   screenShareEncoding: ScreenSharePresets.h1080fps15.encoding,
   stopMicTrackOnMute: false,
-  videoCodec: 'vp8',
+  videoCodec: defaultVideoCodec,
   backupCodec: false,
 } as const;
 

src/room/participant/LocalParticipant.ts

@@ -18,6 +18,7 @@ import {
   TrackUnpublishedResponse,
 } from '../../proto/livekit_rtc_pb';
 import type RTCEngine from '../RTCEngine';
+import { defaultVideoCodec } from '../defaults';
 import { DeviceUnsupportedError, TrackInvalidError, UnexpectedConnectionState } from '../errors';
 import { EngineEvent, ParticipantEvent, TrackEvent } from '../events';
 import LocalAudioTrack from '../track/LocalAudioTrack';
@@ -33,10 +34,11 @@ import type {
   TrackPublishOptions,
   VideoCaptureOptions,
 } from '../track/options';
-import { VideoPresets, isBackupCodec, isCodecEqual } from '../track/options';
+import { VideoPresets, isBackupCodec } from '../track/options';
 import {
   constraintsForOptions,
   mergeDefaultOptions,
+  mimeTypeToVideoCodecString,
   screenCaptureToDisplayMediaStreamOptions,
 } from '../track/utils';
 import type { DataPublishOptions } from '../types';
@@ -629,6 +631,10 @@ export default class LocalParticipant extends Participant {
     if (opts.videoCodec === 'vp9' && !supportsVP9()) {
       opts.videoCodec = undefined;
     }
+    if (opts.videoCodec === undefined) {
+      opts.videoCodec = defaultVideoCodec;
+    }
+    const videoCodec = opts.videoCodec;
 
     // handle track actions
     track.on(TrackEvent.Muted, this.onTrackMuted);
@@ -654,7 +660,6 @@ export default class LocalParticipant extends Participant {
 
     // compute encodings and layers for video
     let encodings: RTCRtpEncodingParameters[] | undefined;
-    let simEncodings: RTCRtpEncodingParameters[] | undefined;
     if (track.kind === Track.Kind.Video) {
       let dims: Track.Dimensions = {
         width: 0,
@@ -679,50 +684,40 @@ export default class LocalParticipant extends Participant {
       req.height = dims.height;
       // for svc codecs, disable simulcast and use vp8 for backup codec
       if (track instanceof LocalVideoTrack) {
-        if (isSVCCodec(opts.videoCodec)) {
+        if (isSVCCodec(videoCodec)) {
           // vp9 svc with screenshare has problem to encode, always use L1T3 here
-          if (track.source === Track.Source.ScreenShare && opts.videoCodec === 'vp9') {
+          if (track.source === Track.Source.ScreenShare && videoCodec === 'vp9') {
             opts.scalabilityMode = 'L1T3';
           }
           // set scalabilityMode to 'L3T3_KEY' by default
           opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY';
         }
 
+        req.simulcastCodecs = [
+          new SimulcastCodec({
+            codec: videoCodec,
+            cid: track.mediaStreamTrack.id,
+          }),
+        ];
+
         // set up backup
-        if (opts.videoCodec && opts.backupCodec && opts.videoCodec !== opts.backupCodec.codec) {
+        if (opts.backupCodec && videoCodec !== opts.backupCodec.codec) {
           if (!this.roomOptions.dynacast) {
             this.roomOptions.dynacast = true;
           }
-          const simOpts = { ...opts };
-          simOpts.simulcast = true;
-          simEncodings = computeTrackBackupEncodings(track, opts.backupCodec.codec, simOpts);
-
-          req.simulcastCodecs = [
-            new SimulcastCodec({
-              codec: opts.videoCodec,
-              cid: track.mediaStreamTrack.id,
-            }),
+          req.simulcastCodecs.push(
             new SimulcastCodec({
               codec: opts.backupCodec.codec,
               cid: '',
             }),
-          ];
-        } else if (opts.videoCodec) {
-          // pass codec info to sfu so it can prefer codec for the client which don't support
-          // setCodecPreferences
-          req.simulcastCodecs = [
-            new SimulcastCodec({
-              codec: opts.videoCodec,
-              cid: track.mediaStreamTrack.id,
-            }),
-          ];
+          );
         }
       }
 
       encodings = computeVideoEncodings(
         track.source === Track.Source.ScreenShare,
-        dims.width,
-        dims.height,
+        req.width,
+        req.height,
         opts,
       );
       req.layers = videoLayersFromEncodings(
@@ -746,30 +741,28 @@ export default class LocalParticipant extends Participant {
     }
 
     const ti = await this.engine.addTrack(req);
-    let primaryCodecSupported = false;
-    let backupCodecSupported = false;
-    ti.codecs.forEach((c) => {
-      if (isCodecEqual(c.mimeType, opts.videoCodec)) {
-        primaryCodecSupported = true;
-      } else if (opts.backupCodec && isCodecEqual(c.mimeType, opts.backupCodec.codec)) {
-        backupCodecSupported = true;
+    // server might not support the codec the client has requested, in that case, fallback
+    // to a supported codec
+    let primaryCodecMime: string | undefined;
+    ti.codecs.forEach((codec) => {
+      if (primaryCodecMime === undefined) {
+        primaryCodecMime = codec.mimeType;
       }
     });
-
-    if (req.simulcastCodecs.length > 0) {
-      if (!primaryCodecSupported && !backupCodecSupported) {
-        throw Error('cannot publish track, codec not supported by server');
-      }
-
-      if (!primaryCodecSupported && opts.backupCodec) {
-        const backupCodec = opts.backupCodec;
-        opts = { ...opts };
-        log.debug(
-          `primary codec ${opts.videoCodec} not supported, fallback to ${backupCodec.codec}`,
+    if (primaryCodecMime && track.kind === Track.Kind.Video) {
+      const updatedCodec = mimeTypeToVideoCodecString(primaryCodecMime);
+      if (updatedCodec !== videoCodec) {
+        log.debug('falling back to server selected codec', { codec: updatedCodec });
+        /* @ts-ignore */
+        opts.videoCodec = updatedCodec;
+
+        // recompute encodings since bitrates/etc could have changed
+        encodings = computeVideoEncodings(
+          track.source === Track.Source.ScreenShare,
+          req.width,
+          req.height,
+          opts,
         );
-        opts.videoCodec = backupCodec.codec;
-        opts.videoEncoding = backupCodec.encoding;
-        encodings = simEncodings;
       }
     }
 
@@ -783,13 +776,12 @@ export default class LocalParticipant extends Participant {
     }
     log.debug(`publishing ${track.kind} with encodings`, { encodings, trackInfo: ti });
 
-    // store RTPSender
     track.sender = await this.engine.createSender(track, opts, encodings);
 
     if (encodings) {
       if (isFireFox() && track.kind === Track.Kind.Audio) {
         /* Refer to RFC https://datatracker.ietf.org/doc/html/rfc7587#section-6.1,
-           livekit-server uses maxaveragebitrate=510000in the answer sdp to permit client to
+           livekit-server uses maxaveragebitrate=510000 in the answer sdp to permit client to
            publish high quality audio track. But firefox always uses this value as the actual
            bitrates, causing the audio bitrates to rise to 510Kbps in any stereo case unexpectedly.
            So the client need to modify maxaverragebitrates in answer sdp to user provided value to

src/room/track/LocalVideoTrack.ts

@@ -284,7 +284,8 @@ export default class LocalVideoTrack extends LocalTrack {
 
   /**
    * @internal
-   * Sets codecs that should be publishing
+   * Sets codecs that should be publishing, returns new codecs that have not yet
+   * been published
    */
   async setPublishingCodecs(codecs: SubscribedCodec[]): Promise<VideoCodec[]> {
     log.debug('setting publishing codecs', {

src/room/track/options.ts

@@ -301,13 +301,6 @@ export function isBackupCodec(codec: string): codec is BackupVideoCodec {
   return !!backupCodecs.find((backup) => backup === codec);
 }
 
-export function isCodecEqual(c1: string | undefined, c2: string | undefined): boolean {
-  return (
-    c1?.toLowerCase().replace(/audio\/|video\//y, '') ===
-    c2?.toLowerCase().replace(/audio\/|video\//y, '')
-  );
-}
-
 /**
  * scalability modes for svc.
  */

src/room/track/utils.ts

@@ -1,11 +1,13 @@
 import { cloneDeep } from '../../utils/cloneDeep';
 import { isSafari, sleep } from '../utils';
 import { Track } from './Track';
-import type {
-  AudioCaptureOptions,
-  CreateLocalTracksOptions,
-  ScreenShareCaptureOptions,
-  VideoCaptureOptions,
+import {
+  type AudioCaptureOptions,
+  type CreateLocalTracksOptions,
+  type ScreenShareCaptureOptions,
+  type VideoCaptureOptions,
+  VideoCodec,
+  videoCodecs,
 } from './options';
 import type { AudioTrack } from './types';
 
@@ -180,3 +182,11 @@ export function screenCaptureToDisplayMediaStreamOptions(
     systemAudio: options.systemAudio,
   };
 }
+
+export function mimeTypeToVideoCodecString(mimeType: string) {
+  const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec;
+  if (!videoCodecs.includes(codec)) {
+    throw Error(`Video codec not supported: ${codec}`);
+  }
+  return codec;
+}