From fb2b221555eccc9ec0f1bc663d3b383dd513f384 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Fri, 19 Aug 2022 12:23:36 +0200 Subject: [PATCH] Add fallback for `addTrack` if `addTransceiver` is not supported by a device (#403) * wip * move sender logic into RTCEngine * renaming, make methods private * remove debug config * address comments * remove unused code * changeset --- .changeset/wicked-toys-reflect.md | 5 + src/index.ts | 11 +- src/proto/google/protobuf/timestamp.ts | 2 +- src/proto/livekit_models.ts | 34 +++--- src/proto/livekit_rtc.ts | 2 +- src/room/RTCEngine.ts | 142 ++++++++++++++++++++++- src/room/participant/LocalParticipant.ts | 74 +----------- src/room/utils.ts | 22 ++++ 8 files changed, 203 insertions(+), 89 deletions(-) create mode 100644 .changeset/wicked-toys-reflect.md diff --git a/.changeset/wicked-toys-reflect.md b/.changeset/wicked-toys-reflect.md new file mode 100644 index 0000000000..6ba1441490 --- /dev/null +++ b/.changeset/wicked-toys-reflect.md @@ -0,0 +1,5 @@ +--- +'livekit-client': patch +--- + +Add fallback for addTrack if addTransceiver is not supported diff --git a/src/index.ts b/src/index.ts index c2d76bf154..fad439dc80 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,7 +15,13 @@ import RemoteTrack from './room/track/RemoteTrack'; import RemoteTrackPublication from './room/track/RemoteTrackPublication'; import RemoteVideoTrack, { ElementInfo } from './room/track/RemoteVideoTrack'; import { TrackPublication } from './room/track/TrackPublication'; -import { getEmptyAudioStreamTrack, getEmptyVideoStreamTrack } from './room/utils'; +import { + getEmptyAudioStreamTrack, + getEmptyVideoStreamTrack, + isDeviceSupported, + supportsAdaptiveStream, + supportsDynacast, +} from './room/utils'; export * from './options'; export * from './room/errors'; @@ -30,6 +36,9 @@ export { setLogExtension, getEmptyAudioStreamTrack, getEmptyVideoStreamTrack, + isDeviceSupported, + supportsAdaptiveStream, + supportsDynacast, LogLevel, Room, ConnectionState, diff --git a/src/proto/google/protobuf/timestamp.ts b/src/proto/google/protobuf/timestamp.ts index ae39ae3f77..6cde2c460c 100644 --- a/src/proto/google/protobuf/timestamp.ts +++ b/src/proto/google/protobuf/timestamp.ts @@ -199,7 +199,7 @@ export type DeepPartial = T extends Builtin type KeysOfUnion = T extends T ? keyof T : never; export type Exact = P extends Builtin ? P - : P & { [K in keyof P]: Exact } & Record>, never>; + : P & { [K in keyof P]: Exact } & { [K in Exclude>]: never }; function longToNumber(long: Long): number { if (long.gt(Number.MAX_SAFE_INTEGER)) { diff --git a/src/proto/livekit_models.ts b/src/proto/livekit_models.ts index ffccd09e55..e6785cbfd2 100644 --- a/src/proto/livekit_models.ts +++ b/src/proto/livekit_models.ts @@ -2921,25 +2921,29 @@ var globalThis: any = (() => { throw 'Unable to locate global object'; })(); -const atob: (b64: string) => string = - globalThis.atob || ((b64) => globalThis.Buffer.from(b64, 'base64').toString('binary')); function bytesFromBase64(b64: string): Uint8Array { - const bin = atob(b64); - const arr = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; ++i) { - arr[i] = bin.charCodeAt(i); + if (globalThis.Buffer) { + return Uint8Array.from(globalThis.Buffer.from(b64, 'base64')); + } else { + const bin = globalThis.atob(b64); + const arr = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; ++i) { + arr[i] = bin.charCodeAt(i); + } + return arr; } - return arr; } -const btoa: (bin: string) => string = - globalThis.btoa || ((bin) => globalThis.Buffer.from(bin, 'binary').toString('base64')); function base64FromBytes(arr: Uint8Array): string { - const bin: string[] = []; - arr.forEach((byte) => { - bin.push(String.fromCharCode(byte)); - }); - return btoa(bin.join('')); + if (globalThis.Buffer) { + return globalThis.Buffer.from(arr).toString('base64'); + } else { + const bin: string[] = []; + arr.forEach((byte) => { + bin.push(String.fromCharCode(byte)); + }); + return globalThis.btoa(bin.join('')); + } } type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined; @@ -2959,7 +2963,7 @@ export type DeepPartial = T extends Builtin type KeysOfUnion = T extends T ? keyof T : never; export type Exact = P extends Builtin ? P - : P & { [K in keyof P]: Exact } & Record>, never>; + : P & { [K in keyof P]: Exact } & { [K in Exclude>]: never }; function toTimestamp(date: Date): Timestamp { const seconds = date.getTime() / 1_000; diff --git a/src/proto/livekit_rtc.ts b/src/proto/livekit_rtc.ts index a4af47b6f9..89fd6c78c8 100644 --- a/src/proto/livekit_rtc.ts +++ b/src/proto/livekit_rtc.ts @@ -3480,7 +3480,7 @@ export type DeepPartial = T extends Builtin type KeysOfUnion = T extends T ? keyof T : never; export type Exact = P extends Builtin ? P - : P & { [K in keyof P]: Exact } & Record>, never>; + : P & { [K in keyof P]: Exact } & { [K in Exclude>]: never }; function longToNumber(long: Long): number { if (long.gt(Number.MAX_SAFE_INTEGER)) { diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index c019360444..6336b16d88 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -25,7 +25,11 @@ import { ConnectionError, TrackInvalidError, UnexpectedConnectionState } from '. import { EngineEvent } from './events'; import PCTransport from './PCTransport'; import { ReconnectContext, ReconnectPolicy } from './ReconnectPolicy'; -import { isWeb, sleep } from './utils'; +import LocalTrack from './track/LocalTrack'; +import LocalVideoTrack, { SimulcastTrackInfo } from './track/LocalVideoTrack'; +import { TrackPublishOptions, VideoCodec } from './track/options'; +import { Track } from './track/Track'; +import { isWeb, sleep, supportsAddTrack, supportsTransceiver } from './utils'; const lossyDataChannel = '_lossy'; const reliableDataChannel = '_reliable'; @@ -475,6 +479,142 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } }; + private setPreferredCodec( + transceiver: RTCRtpTransceiver, + kind: Track.Kind, + videoCodec: VideoCodec, + ) { + if (!('getCapabilities' in RTCRtpSender)) { + return; + } + const cap = RTCRtpSender.getCapabilities(kind); + if (!cap) return; + log.debug('get capabilities', cap); + const matched: RTCRtpCodecCapability[] = []; + const partialMatched: RTCRtpCodecCapability[] = []; + const unmatched: RTCRtpCodecCapability[] = []; + cap.codecs.forEach((c) => { + const codec = c.mimeType.toLowerCase(); + if (codec === 'audio/opus') { + matched.push(c); + return; + } + const matchesVideoCodec = codec === `video/${videoCodec}`; + if (!matchesVideoCodec) { + unmatched.push(c); + return; + } + // for h264 codecs that have sdpFmtpLine available, use only if the + // profile-level-id is 42e01f for cross-browser compatibility + if (videoCodec === 'h264') { + if (c.sdpFmtpLine && c.sdpFmtpLine.includes('profile-level-id=42e01f')) { + matched.push(c); + } else { + partialMatched.push(c); + } + return; + } + + matched.push(c); + }); + + if ('setCodecPreferences' in transceiver) { + transceiver.setCodecPreferences(matched.concat(partialMatched, unmatched)); + } + } + + async createSender( + track: LocalTrack, + opts: TrackPublishOptions, + encodings?: RTCRtpEncodingParameters[], + ) { + if (supportsTransceiver()) { + return this.createTransceiverRTCRtpSender(track, opts, encodings); + } + if (supportsAddTrack()) { + log.debug('using add-track fallback'); + return this.createRTCRtpSender(track.mediaStreamTrack); + } + throw new UnexpectedConnectionState('Required webRTC APIs not supported on this device'); + } + + async createSimulcastSender( + track: LocalVideoTrack, + simulcastTrack: SimulcastTrackInfo, + opts: TrackPublishOptions, + encodings?: RTCRtpEncodingParameters[], + ) { + // store RTCRtpSender + // @ts-ignore + if (supportsTransceiver()) { + return this.createSimulcastTransceiverSender(track, simulcastTrack, opts, encodings); + } + if (supportsAddTrack()) { + log.debug('using add-track fallback'); + return this.createRTCRtpSender(track.mediaStreamTrack); + } + + throw new UnexpectedConnectionState('Cannot stream on this device'); + } + + private async createTransceiverRTCRtpSender( + track: LocalTrack, + opts: TrackPublishOptions, + encodings?: RTCRtpEncodingParameters[], + ) { + if (!this.publisher) { + throw new UnexpectedConnectionState('publisher is closed'); + } + + const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' }; + if (encodings) { + transceiverInit.sendEncodings = encodings; + } + // addTransceiver for react-native is async. web is synchronous, but await won't effect it. + const transceiver = await this.publisher.pc.addTransceiver( + track.mediaStreamTrack, + transceiverInit, + ); + if (track.kind === Track.Kind.Video && opts.videoCodec) { + this.setPreferredCodec(transceiver, track.kind, opts.videoCodec); + track.codec = opts.videoCodec; + } + return transceiver.sender; + } + + private async createSimulcastTransceiverSender( + track: LocalVideoTrack, + simulcastTrack: SimulcastTrackInfo, + opts: TrackPublishOptions, + encodings?: RTCRtpEncodingParameters[], + ) { + if (!this.publisher) { + throw new UnexpectedConnectionState('publisher is closed'); + } + const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' }; + if (encodings) { + transceiverInit.sendEncodings = encodings; + } + // addTransceiver for react-native is async. web is synchronous, but await won't effect it. + const transceiver = await this.publisher.pc.addTransceiver( + simulcastTrack.mediaStreamTrack, + transceiverInit, + ); + if (!opts.videoCodec) { + return; + } + this.setPreferredCodec(transceiver, track.kind, opts.videoCodec); + track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender); + return transceiver.sender; + } + + private async createRTCRtpSender(track: MediaStreamTrack) { + if (!this.publisher) { + throw new UnexpectedConnectionState('publisher is closed'); + } + return this.publisher.pc.addTrack(track); + } + // websocket reconnect behavior. if websocket is interrupted, and the PeerConnection // continues to work, we can reconnect to websocket to continue the session // after a number of retries, we'll close and give up permanently diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index 1585a21e05..098b4a2d7c 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -30,7 +30,6 @@ import { ScreenSharePresets, TrackPublishOptions, VideoCaptureOptions, - VideoCodec, } from '../track/options'; import { Track } from '../track/Track'; import { constraintsForOptions, mergeDefaultOptions } from '../track/utils'; @@ -552,19 +551,9 @@ export default class LocalParticipant extends Participant { throw new UnexpectedConnectionState('publisher is closed'); } log.debug(`publishing ${track.kind} with encodings`, { encodings, trackInfo: ti }); - const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' }; - if (encodings) { - transceiverInit.sendEncodings = encodings; - } - // addTransceiver for react-native is async. web is synchronous, but await won't effect it. - const transceiver = await this.engine.publisher.pc.addTransceiver( - track.mediaStreamTrack, - transceiverInit, - ); - if (track.kind === Track.Kind.Video && opts.videoCodec) { - this.setPreferredCodec(transceiver, track.kind, opts.videoCodec); - track.codec = opts.videoCodec; - } + + // store RTPSender + track.sender = await this.engine.createSender(track, opts, encodings); if (track.codec === 'av1' && encodings && encodings[0]?.maxBitrate) { this.engine.publisher.setTrackCodecBitrate( @@ -576,8 +565,6 @@ export default class LocalParticipant extends Participant { this.engine.negotiate(); - // store RTPSender - track.sender = transceiver.sender; if (track instanceof LocalVideoTrack) { track.startMonitor(this.engine.client); } else if (track instanceof LocalAudioTrack) { @@ -652,20 +639,11 @@ export default class LocalParticipant extends Participant { const ti = await this.engine.addTrack(req); - if (!this.engine.publisher) { - throw new UnexpectedConnectionState('publisher is closed'); - } const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' }; if (encodings) { transceiverInit.sendEncodings = encodings; } - // addTransceiver for react-native is async. web is synchronous, but await won't effect it. - const transceiver = await this.engine.publisher.pc.addTransceiver( - simulcastTrack.mediaStreamTrack, - transceiverInit, - ); - this.setPreferredCodec(transceiver, track.kind, videoCodec); - track.setSimulcastTrackSender(videoCodec, transceiver.sender); + await this.engine.createSimulcastSender(track, simulcastTrack, opts, encodings); this.engine.negotiate(); log.debug(`published ${videoCodec} for track ${track.sid}`, { encodings, trackInfo: ti }); @@ -1000,50 +978,6 @@ export default class LocalParticipant extends Participant { return publication; } - private setPreferredCodec( - transceiver: RTCRtpTransceiver, - kind: Track.Kind, - videoCodec: VideoCodec, - ) { - if (!('getCapabilities' in RTCRtpSender)) { - return; - } - const cap = RTCRtpSender.getCapabilities(kind); - if (!cap) return; - log.debug('get capabilities', cap); - const matched: RTCRtpCodecCapability[] = []; - const partialMatched: RTCRtpCodecCapability[] = []; - const unmatched: RTCRtpCodecCapability[] = []; - cap.codecs.forEach((c) => { - const codec = c.mimeType.toLowerCase(); - if (codec === 'audio/opus') { - matched.push(c); - return; - } - const matchesVideoCodec = codec === `video/${videoCodec}`; - if (!matchesVideoCodec) { - unmatched.push(c); - return; - } - // for h264 codecs that have sdpFmtpLine available, use only if the - // profile-level-id is 42e01f for cross-browser compatibility - if (videoCodec === 'h264') { - if (c.sdpFmtpLine && c.sdpFmtpLine.includes('profile-level-id=42e01f')) { - matched.push(c); - } else { - partialMatched.push(c); - } - return; - } - - matched.push(c); - }); - - if ('setCodecPreferences' in transceiver) { - transceiver.setCodecPreferences(matched.concat(partialMatched, unmatched)); - } - } - /** @internal */ publishedTracksInfo(): TrackPublishedResponse[] { const infos: TrackPublishedResponse[] = []; diff --git a/src/room/utils.ts b/src/room/utils.ts index 1b6f71b9a0..dd259bcd4c 100644 --- a/src/room/utils.ts +++ b/src/room/utils.ts @@ -15,6 +15,28 @@ export async function sleep(duration: number): Promise { return new Promise((resolve) => setTimeout(resolve, duration)); } +/** @internal */ +export function supportsTransceiver() { + return 'addTransceiver' in RTCPeerConnection; +} + +/** @internal */ +export function supportsAddTrack() { + return 'addTrack' in RTCPeerConnection; +} + +export function supportsAdaptiveStream() { + return typeof ResizeObserver !== undefined && typeof IntersectionObserver !== undefined; +} + +export function supportsDynacast() { + return supportsTransceiver(); +} + +export function isDeviceSupported() { + return supportsTransceiver() || supportsAddTrack(); +} + export function isFireFox(): boolean { if (!isWeb()) return false; return navigator.userAgent.indexOf('Firefox') !== -1;