Add experimental option to pipe attached audio tracks through webaudi…

…o API (#446)

* add option to pipe attached audio tracks through webaudio api

* use logger

* detach logic

* reconnect if still attached

* set nodes to undefined

* cleanup

* add method to set additional audionodes that plugin between track and destination

* changeset

* only reconnect webaudio if needed

.changeset/eight-dryers-drop.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Add experimental option to pipe attached audio tracks through webaudio API

src/options.ts

@@ -49,16 +49,23 @@ export interface InternalRoomOptions {
    */
   stopLocalTrackOnUnpublish: boolean;
 
+  /**
+   * policy to use when attempting to reconnect
+   */
+  reconnectPolicy: ReconnectPolicy;
+
   /**
    * @internal
    * experimental flag, introduce a delay before sending signaling messages
    */
   expSignalLatency?: number;
 
   /**
-   * policy to use when attempting to reconnect
+   * @internal
+   * @experimental
+   * experimental flag, mix all audio tracks in web audio
    */
-  reconnectPolicy: ReconnectPolicy;
+  expWebAudioMix: boolean;
 }
 
 /**

src/room/Room.ts

@@ -957,6 +957,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     const ctx = getNewAudioContext();
     if (ctx) {
       this.audioContext = ctx;
+      if (this.options.expWebAudioMix) {
+        this.participants.forEach((participant) => participant.setAudioContext(this.audioContext));
+      }
     }
   }
 
@@ -965,7 +968,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     if (info) {
       participant = RemoteParticipant.fromParticipantInfo(this.engine.client, info);
     } else {
-      participant = new RemoteParticipant(this.engine.client, id, '');
+      participant = new RemoteParticipant(this.engine.client, id, '', undefined, undefined);
+    }
+    if (this.options.expWebAudioMix) {
+      participant.setAudioContext(this.audioContext);
     }
     return participant;
   }

src/room/defaults.ts

@@ -34,6 +34,7 @@ export const roomOptionDefaults: InternalRoomOptions = {
   dynacast: false,
   stopLocalTrackOnUnpublish: true,
   reconnectPolicy: new DefaultReconnectPolicy(),
+  expWebAudioMix: false,
 } as const;
 
 export const roomConnectOptionDefaults: InternalRoomConnectOptions = {

src/room/participant/RemoteParticipant.ts

@@ -22,6 +22,8 @@ export default class RemoteParticipant extends Participant {
 
   private volume?: number;
 
+  private audioContext?: AudioContext;
+
   /** @internal */
   static fromParticipantInfo(signalClient: SignalClient, pi: ParticipantInfo): RemoteParticipant {
     return new RemoteParticipant(signalClient, pi.sid, pi.identity, pi.name, pi.metadata);
@@ -167,7 +169,7 @@ export default class RemoteParticipant extends Participant {
     if (isVideo) {
       track = new RemoteVideoTrack(mediaTrack, sid, receiver, adaptiveStreamSettings);
     } else {
-      track = new RemoteAudioTrack(mediaTrack, sid, receiver);
+      track = new RemoteAudioTrack(mediaTrack, sid, receiver, this.audioContext);
     }
 
     // set track info
@@ -297,6 +299,13 @@ export default class RemoteParticipant extends Participant {
     }
   }
 
+  /**
+   * @internal
+   */
+  setAudioContext(ctx: AudioContext | undefined) {
+    this.audioContext = ctx;
+  }
+
   /** @internal */
   emit<E extends keyof ParticipantEventCallbacks>(
     event: E,

src/room/track/RemoteAudioTrack.ts

@@ -1,22 +1,42 @@
 import { AudioReceiverStats, computeBitrate } from '../stats';
 import RemoteTrack from './RemoteTrack';
 import { Track } from './Track';
+import log from '../../logger';
 
 export default class RemoteAudioTrack extends RemoteTrack {
   private prevStats?: AudioReceiverStats;
 
   private elementVolume: number | undefined;
 
-  constructor(mediaTrack: MediaStreamTrack, sid: string, receiver?: RTCRtpReceiver) {
+  private audioContext?: AudioContext;
+
+  private gainNode?: GainNode;
+
+  private sourceNode?: MediaStreamAudioSourceNode;
+
+  private webAudioPluginNodes: AudioNode[];
+
+  constructor(
+    mediaTrack: MediaStreamTrack,
+    sid: string,
+    receiver?: RTCRtpReceiver,
+    audioContext?: AudioContext,
+  ) {
     super(mediaTrack, sid, Track.Kind.Audio, receiver);
+    this.audioContext = audioContext;
+    this.webAudioPluginNodes = [];
   }
 
   /**
    * sets the volume for all attached audio elements
    */
   setVolume(volume: number) {
     for (const el of this.attachedElements) {
-      el.volume = volume;
+      if (this.audioContext) {
+        this.gainNode?.gain.setTargetAtTime(volume, 0, 0.1);
+      } else {
+        el.volume = volume;
+      }
     }
     this.elementVolume = volume;
   }
@@ -40,6 +60,7 @@ export default class RemoteAudioTrack extends RemoteTrack {
   attach(): HTMLMediaElement;
   attach(element: HTMLMediaElement): HTMLMediaElement;
   attach(element?: HTMLMediaElement): HTMLMediaElement {
+    const needsNewWebAudioConnection = this.attachedElements.length === 0;
     if (!element) {
       element = super.attach();
     } else {
@@ -48,9 +69,90 @@ export default class RemoteAudioTrack extends RemoteTrack {
     if (this.elementVolume) {
       element.volume = this.elementVolume;
     }
+    if (this.audioContext && needsNewWebAudioConnection) {
+      log.debug('using audio context mapping');
+      this.connectWebAudio(this.audioContext, element);
+      element.volume = 0;
+      element.muted = true;
+    }
     return element;
   }
 
+  /**
+   * Detaches from all attached elements
+   */
+  detach(): HTMLMediaElement[];
+
+  /**
+   * Detach from a single element
+   * @param element
+   */
+  detach(element: HTMLMediaElement): HTMLMediaElement;
+  detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] {
+    let detached: HTMLMediaElement | HTMLMediaElement[];
+    if (!element) {
+      detached = super.detach();
+      this.disconnectWebAudio();
+    } else {
+      detached = super.detach(element);
+      // if there are still any attached elements after detaching, connect webaudio to the first element that's left
+      if (this.audioContext && this.attachedElements.length > 0) {
+        if (this.attachedElements.length > 0) {
+          this.connectWebAudio(this.audioContext, this.attachedElements[0]);
+        }
+      }
+    }
+    return detached;
+  }
+
+  /**
+   * @internal
+   * @experimental
+   */
+  setAudioContext(audioContext: AudioContext) {
+    this.audioContext = audioContext;
+    if (this.attachedElements.length > 0) {
+      this.connectWebAudio(audioContext, this.attachedElements[0]);
+    }
+  }
+
+  /**
+   * @internal
+   * @experimental
+   * @param {AudioNode[]} nodes - An array of WebAudio nodes. These nodes should not be connected to each other when passed, as the sdk will take care of connecting them in the order of the array.
+   */
+  setWebAudioPlugins(nodes: AudioNode[]) {
+    this.webAudioPluginNodes = nodes;
+    if (this.attachedElements.length > 0 && this.audioContext) {
+      this.connectWebAudio(this.audioContext, this.attachedElements[0]);
+    }
+  }
+
+  private connectWebAudio(context: AudioContext, element: HTMLMediaElement) {
+    this.disconnectWebAudio();
+    // @ts-ignore attached elements always have a srcObject set
+    this.sourceNode = context.createMediaStreamSource(element.srcObject);
+    let lastNode: AudioNode = this.sourceNode;
+    this.webAudioPluginNodes.forEach((node) => {
+      lastNode.connect(node);
+      lastNode = node;
+    });
+    this.gainNode = context.createGain();
+    lastNode.connect(this.gainNode);
+    this.gainNode.connect(context.destination);
+
+    if (this.elementVolume) {
+      this.gainNode.gain.setTargetAtTime(this.elementVolume, 0, 0.1);
+    }
+  }
+
+  private disconnectWebAudio() {
+    this.gainNode?.disconnect();
+    this.sourceNode?.disconnect();
+    this.gainNode = undefined;
+    this.sourceNode = undefined;
+  }
+
   protected monitorReceiver = async () => {
     if (!this.receiver) {
       this._currentBitrate = 0;

src/room/track/utils.ts

@@ -108,6 +108,6 @@ export function getNewAudioContext(): AudioContext | void {
   // @ts-ignore
   const AudioContext = window.AudioContext || window.webkitAudioContext;
   if (AudioContext) {
-    return new AudioContext();
+    return new AudioContext({ latencyHint: 'interactive' });
   }
 }