diff --git a/.changeset/eight-dryers-drop.md b/.changeset/eight-dryers-drop.md new file mode 100644 index 0000000000..e9e717e24f --- /dev/null +++ b/.changeset/eight-dryers-drop.md @@ -0,0 +1,5 @@ +--- +'livekit-client': patch +--- + +Add experimental option to pipe attached audio tracks through webaudio API diff --git a/src/options.ts b/src/options.ts index 25d9df832e..aa05648689 100644 --- a/src/options.ts +++ b/src/options.ts @@ -49,6 +49,11 @@ export interface InternalRoomOptions { */ stopLocalTrackOnUnpublish: boolean; + /** + * policy to use when attempting to reconnect + */ + reconnectPolicy: ReconnectPolicy; + /** * @internal * experimental flag, introduce a delay before sending signaling messages @@ -56,9 +61,11 @@ export interface InternalRoomOptions { expSignalLatency?: number; /** - * policy to use when attempting to reconnect + * @internal + * @experimental + * experimental flag, mix all audio tracks in web audio */ - reconnectPolicy: ReconnectPolicy; + expWebAudioMix: boolean; } /** diff --git a/src/room/Room.ts b/src/room/Room.ts index b16d1baae6..4caa0bf5e7 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -957,6 +957,9 @@ class Room extends (EventEmitter as new () => TypedEmitter) const ctx = getNewAudioContext(); if (ctx) { this.audioContext = ctx; + if (this.options.expWebAudioMix) { + this.participants.forEach((participant) => participant.setAudioContext(this.audioContext)); + } } } @@ -965,7 +968,10 @@ class Room extends (EventEmitter as new () => TypedEmitter) if (info) { participant = RemoteParticipant.fromParticipantInfo(this.engine.client, info); } else { - participant = new RemoteParticipant(this.engine.client, id, ''); + participant = new RemoteParticipant(this.engine.client, id, '', undefined, undefined); + } + if (this.options.expWebAudioMix) { + participant.setAudioContext(this.audioContext); } return participant; } diff --git a/src/room/defaults.ts b/src/room/defaults.ts index b26fb8d35a..6adf78f2dd 100644 --- a/src/room/defaults.ts +++ b/src/room/defaults.ts @@ -34,6 +34,7 @@ export const roomOptionDefaults: InternalRoomOptions = { dynacast: false, stopLocalTrackOnUnpublish: true, reconnectPolicy: new DefaultReconnectPolicy(), + expWebAudioMix: false, } as const; export const roomConnectOptionDefaults: InternalRoomConnectOptions = { diff --git a/src/room/participant/RemoteParticipant.ts b/src/room/participant/RemoteParticipant.ts index bcf1b4fdd6..d785eb12ab 100644 --- a/src/room/participant/RemoteParticipant.ts +++ b/src/room/participant/RemoteParticipant.ts @@ -22,6 +22,8 @@ export default class RemoteParticipant extends Participant { private volume?: number; + private audioContext?: AudioContext; + /** @internal */ static fromParticipantInfo(signalClient: SignalClient, pi: ParticipantInfo): RemoteParticipant { return new RemoteParticipant(signalClient, pi.sid, pi.identity, pi.name, pi.metadata); @@ -167,7 +169,7 @@ export default class RemoteParticipant extends Participant { if (isVideo) { track = new RemoteVideoTrack(mediaTrack, sid, receiver, adaptiveStreamSettings); } else { - track = new RemoteAudioTrack(mediaTrack, sid, receiver); + track = new RemoteAudioTrack(mediaTrack, sid, receiver, this.audioContext); } // set track info @@ -297,6 +299,13 @@ export default class RemoteParticipant extends Participant { } } + /** + * @internal + */ + setAudioContext(ctx: AudioContext | undefined) { + this.audioContext = ctx; + } + /** @internal */ emit( event: E, diff --git a/src/room/track/RemoteAudioTrack.ts b/src/room/track/RemoteAudioTrack.ts index 4df063b0de..1569a68112 100644 --- a/src/room/track/RemoteAudioTrack.ts +++ b/src/room/track/RemoteAudioTrack.ts @@ -1,14 +1,30 @@ import { AudioReceiverStats, computeBitrate } from '../stats'; import RemoteTrack from './RemoteTrack'; import { Track } from './Track'; +import log from '../../logger'; export default class RemoteAudioTrack extends RemoteTrack { private prevStats?: AudioReceiverStats; private elementVolume: number | undefined; - constructor(mediaTrack: MediaStreamTrack, sid: string, receiver?: RTCRtpReceiver) { + private audioContext?: AudioContext; + + private gainNode?: GainNode; + + private sourceNode?: MediaStreamAudioSourceNode; + + private webAudioPluginNodes: AudioNode[]; + + constructor( + mediaTrack: MediaStreamTrack, + sid: string, + receiver?: RTCRtpReceiver, + audioContext?: AudioContext, + ) { super(mediaTrack, sid, Track.Kind.Audio, receiver); + this.audioContext = audioContext; + this.webAudioPluginNodes = []; } /** @@ -16,7 +32,11 @@ export default class RemoteAudioTrack extends RemoteTrack { */ setVolume(volume: number) { for (const el of this.attachedElements) { - el.volume = volume; + if (this.audioContext) { + this.gainNode?.gain.setTargetAtTime(volume, 0, 0.1); + } else { + el.volume = volume; + } } this.elementVolume = volume; } @@ -40,6 +60,7 @@ export default class RemoteAudioTrack extends RemoteTrack { attach(): HTMLMediaElement; attach(element: HTMLMediaElement): HTMLMediaElement; attach(element?: HTMLMediaElement): HTMLMediaElement { + const needsNewWebAudioConnection = this.attachedElements.length === 0; if (!element) { element = super.attach(); } else { @@ -48,9 +69,90 @@ export default class RemoteAudioTrack extends RemoteTrack { if (this.elementVolume) { element.volume = this.elementVolume; } + if (this.audioContext && needsNewWebAudioConnection) { + log.debug('using audio context mapping'); + this.connectWebAudio(this.audioContext, element); + element.volume = 0; + element.muted = true; + } return element; } + /** + * Detaches from all attached elements + */ + detach(): HTMLMediaElement[]; + + /** + * Detach from a single element + * @param element + */ + detach(element: HTMLMediaElement): HTMLMediaElement; + detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] { + let detached: HTMLMediaElement | HTMLMediaElement[]; + if (!element) { + detached = super.detach(); + this.disconnectWebAudio(); + } else { + detached = super.detach(element); + // if there are still any attached elements after detaching, connect webaudio to the first element that's left + if (this.audioContext && this.attachedElements.length > 0) { + if (this.attachedElements.length > 0) { + this.connectWebAudio(this.audioContext, this.attachedElements[0]); + } + } + } + return detached; + } + + /** + * @internal + * @experimental + */ + setAudioContext(audioContext: AudioContext) { + this.audioContext = audioContext; + if (this.attachedElements.length > 0) { + this.connectWebAudio(audioContext, this.attachedElements[0]); + } + } + + /** + * @internal + * @experimental + * @param {AudioNode[]} nodes - An array of WebAudio nodes. These nodes should not be connected to each other when passed, as the sdk will take care of connecting them in the order of the array. + */ + setWebAudioPlugins(nodes: AudioNode[]) { + this.webAudioPluginNodes = nodes; + if (this.attachedElements.length > 0 && this.audioContext) { + this.connectWebAudio(this.audioContext, this.attachedElements[0]); + } + } + + private connectWebAudio(context: AudioContext, element: HTMLMediaElement) { + this.disconnectWebAudio(); + // @ts-ignore attached elements always have a srcObject set + this.sourceNode = context.createMediaStreamSource(element.srcObject); + let lastNode: AudioNode = this.sourceNode; + this.webAudioPluginNodes.forEach((node) => { + lastNode.connect(node); + lastNode = node; + }); + this.gainNode = context.createGain(); + lastNode.connect(this.gainNode); + this.gainNode.connect(context.destination); + + if (this.elementVolume) { + this.gainNode.gain.setTargetAtTime(this.elementVolume, 0, 0.1); + } + } + + private disconnectWebAudio() { + this.gainNode?.disconnect(); + this.sourceNode?.disconnect(); + this.gainNode = undefined; + this.sourceNode = undefined; + } + protected monitorReceiver = async () => { if (!this.receiver) { this._currentBitrate = 0; diff --git a/src/room/track/utils.ts b/src/room/track/utils.ts index 8cf87f134b..d1a6a76010 100644 --- a/src/room/track/utils.ts +++ b/src/room/track/utils.ts @@ -108,6 +108,6 @@ export function getNewAudioContext(): AudioContext | void { // @ts-ignore const AudioContext = window.AudioContext || window.webkitAudioContext; if (AudioContext) { - return new AudioContext(); + return new AudioContext({ latencyHint: 'interactive' }); } }