Network Audio Fixes (pt 2)

examples/worlds/Multiplayer/ideas/PingPongMulti.tsx

@@ -0,0 +1,15 @@
+import { useNetwork } from "spacesvr";
+import { useEffect } from "react";
+
+export default function PingPongMulti() {
+  const { voice, setVoice } = useNetwork();
+
+  useEffect(() => {
+    setTimeout(() => {
+      console.log("setting to ", !voice);
+      setVoice(!voice);
+    }, 5000);
+  }, [setVoice, voice]);
+
+  return null;
+}

examples/worlds/Multiplayer/index.tsx

@@ -1,12 +1,14 @@
 import { StandardReality, Background, Model } from "spacesvr";
 import LightSwitch from "./ideas/LightSwitch";
+import PingPongMulti from "./ideas/PingPongMulti";
 
 export default function Multiplayer() {
   return (
     <StandardReality
       playerProps={{ pos: [5, 1, 0], rot: Math.PI }}
       networkProps={{ autoconnect: true, voice: true }}
     >
+      {/*<PingPongMulti />*/}
       <Background color={0xffffff} />
       <fog attach="fog" args={[0xffffff, 10, 90]} />
       <ambientLight />

src/layers/Network/ideas/NetworkedEntities/index.tsx

@@ -49,13 +49,11 @@ export default function NetworkedEntities() {
       },
     }));
 
-    const snapshot: Snapshot = {
+    SI.vault.add({
       id: Math.random().toString(),
       time: new Date().getTime(),
       state,
-    };
-
-    SI.vault.add(snapshot);
+    });
   });
 
   // send own player data
@@ -77,23 +75,27 @@ export default function NetworkedEntities() {
     let i = 0;
     for (const entityState of snapshot.state) {
       const { x, y, z, q } = entityState;
-      obj.position.x = x as number;
-      obj.position.y = y as number;
-      obj.position.z = z as number;
+      obj.position.set(x as number, y as number, z as number);
       obj.position.y -= 0.2; // they were floating before, idk where the constant comes from really
       const quat = q as Quat;
-      obj.quaternion.x = quat.x;
-      obj.quaternion.y = quat.y;
-      obj.quaternion.z = quat.z;
-      obj.quaternion.w = quat.w;
+      obj.quaternion.set(
+        quat.x as number,
+        quat.y as number,
+        quat.z as number,
+        quat.w as number
+      );
       obj.updateMatrix();
       mesh.current.setMatrixAt(i, obj.matrix);
 
-      const audio = entities[i]?.posAudio;
-      if (audio) {
-        obj.matrix.decompose(audio.position, audio.quaternion, audio.scale);
-        audio.updateMatrix();
-        audio.rotateY(Math.PI); // for some reason it's flipped
+      const posAudio = entities[i]?.posAudio;
+      if (posAudio) {
+        obj.matrix.decompose(
+          posAudio.position,
+          posAudio.quaternion,
+          posAudio.scale
+        );
+        posAudio.rotation.y += Math.PI; // for some reason it's flipped
+        posAudio.updateMatrix();
       }
 
       i++;
@@ -107,11 +109,15 @@ export default function NetworkedEntities() {
   }
 
   return (
-    <group>
+    <group name="spacesvr-entities">
       {entities.map(
         (entity) =>
           entity.posAudio && (
-            <primitive key={entity.posAudio.uuid} object={entity.posAudio} />
+            <primitive
+              key={entity.posAudio.uuid}
+              object={entity.posAudio}
+              matrixAutoUpdate={false}
+            />
           )
       )}
       <instancedMesh

src/layers/Network/ideas/NetworkedEntities/logic/entity.ts

@@ -12,7 +12,7 @@ type Entity = {
 };
 
 export const useEntities = (): Entity[] => {
-  const { connections, connected, voiceStreams } = useNetwork();
+  const { connections, connected, mediaConnections } = useNetwork();
   const { paused } = useEnvironment();
 
   const listener = useListener();
@@ -23,80 +23,74 @@ export const useEntities = (): Entity[] => {
 
   const entities = useMemo<Entity[]>(() => [], []);
 
-  const sameIds = (ids1: string[], ids2: string[]) =>
-    ids1.sort().join(",") === ids2.sort().join(",");
+  const needsAudio = (e: Entity) => mediaConnections.has(e.id) && !e.posAudio;
 
   // check for a change in player list, re-render if there is a change
-  const connectionIds = useRef<string[]>([]);
-  const voiceIds = useRef<string[]>([]);
-  useLimitedFrame(6, () => {
+  useLimitedFrame(5, () => {
     if (!connected) return;
 
-    // check for changes in connections
-    if (!sameIds(connectionIds.current, Array.from(connections.keys()))) {
-      connectionIds.current = Array.from(connections.keys());
+    // changed flag to trigger re-render at the end
+    let changed = false;
 
-      // remove entities that are no longer connected
-      entities.map((e) => {
-        if (!connectionIds.current.includes(e.id)) {
-          entities.splice(entities.indexOf(e), 1);
-        }
-      });
-
-      // add in new entities
-      for (const id of connectionIds.current) {
-        if (!entities.some((e) => e.id === id)) {
-          entities.push({ id, posAudio: undefined });
+    // remove old entities
+    entities.map((e) => {
+      if (!connections.has(e.id)) {
+        if (e.posAudio) {
+          e.posAudio.remove();
+          e.posAudio = undefined;
         }
+        entities.splice(entities.indexOf(e), 1);
+        changed = true;
       }
+    });
 
-      rerender();
+    // add in new entities
+    for (const id of Array.from(connections.keys())) {
+      if (!entities.some((e) => e.id === id)) {
+        entities.push({ id, posAudio: undefined });
+        changed = true;
+      }
     }
 
     // dont run until first time unpaused to make sure audio context is running from first press
-    if (
-      !firstPaused &&
-      !sameIds(voiceIds.current, Array.from(voiceStreams.keys()))
-    ) {
-      voiceIds.current = Array.from(voiceStreams.keys());
-
-      // remove voice streams that are no longer connected
+    if (!firstPaused) {
+      // remove media connections streams that are no longer connected
       entities.map((e) => {
-        if (!voiceIds.current.includes(e.id)) {
+        if (!mediaConnections.has(e.id)) {
           e.posAudio?.remove();
           e.posAudio = undefined;
+          changed = true;
         }
       });
 
-      // add in new voice streams
-      for (const id of voiceIds.current) {
-        const entity = entities.find((e) => e.id === id);
-        if (!entity) continue;
-
-        const stream = voiceStreams.get(id)!;
-        if (!stream) continue;
+      entities.filter(needsAudio).map((e) => {
+        // add in new media connections if the stream is active
+        const mediaConn = mediaConnections.get(e.id);
+        if (!mediaConn) return;
+        if (!mediaConn.remoteStream) return;
 
         const audioElem = document.createElement("audio");
-        audioElem.srcObject = stream;
+        audioElem.srcObject = mediaConn.remoteStream; // remote is incoming, local is own voice
         audioElem.muted = true;
         audioElem.autoplay = true;
         audioElem.loop = true;
         //@ts-ignore
         audioElem.playsInline = true;
 
         const posAudio = new PositionalAudio(listener);
-        posAudio.userData.peerId = id;
-        posAudio.setMediaStreamSource(stream);
+        posAudio.userData.peerId = e.id;
+        posAudio.setMediaStreamSource(audioElem.srcObject);
         posAudio.setRefDistance(2);
-        posAudio.setDirectionalCone(200, 290, 0.2);
-        posAudio.setVolume(0.6);
+        posAudio.setDirectionalCone(200, 290, 0.35);
 
         // posAudio.add(new PositionalAudioHelper(posAudio, 1));
-        entity.posAudio = posAudio;
-      }
+        e.posAudio = posAudio;
 
-      rerender();
+        changed = true;
+      });
     }
+
+    if (changed) rerender();
   });
 
   return entities;

src/layers/Network/logic/connection.ts

@@ -1,19 +1,19 @@
 import { useEffect, useMemo, useState } from "react";
-import { DataConnection, Peer } from "peerjs";
+import { DataConnection, MediaConnection, Peer } from "peerjs";
 import { isLocalNetwork } from "./local";
 import { LocalSignaller } from "./signallers/LocalSignaller";
 import { MuseSignaller } from "./signallers/MuseSignaller";
 import { useWaving } from "./wave";
 import { Signaller, SignallerConfig } from "./signallers";
 import { Channels, useChannels } from "./channels";
-import { useVoice } from "./voice";
+import { useVoiceConnections } from "./voice";
 import { getMuseIceServers } from "./ice";
 
 export type ConnectionState = {
   connected: boolean;
   connect: (config?: ConnectionConfig) => Promise<void>;
   connections: Map<string, DataConnection>;
-  voiceStreams: Map<string, MediaStream>;
+  mediaConnections: Map<string, MediaConnection>;
   disconnect: () => void;
   voice: boolean;
   setVoice: (v: boolean) => void;
@@ -43,6 +43,10 @@ export const useConnection = (
         console.log("connection closed with peer");
         connections.delete(conn.peer);
       });
+      conn.on("error", () => {
+        console.log("connection closed with peer");
+        connections.delete(conn.peer);
+      });
       channels.greet(conn);
       connections.set(conn.peer, conn);
     });
@@ -125,16 +129,16 @@ export const useConnection = (
 
   const [voice, setVoice] = useState(!!externalConfig.voice);
   useEffect(() => setVoice(!!externalConfig.voice), [externalConfig.voice]);
-  const voiceStreams = useVoice(voice, peer, connections);
+  const mediaConnections = useVoiceConnections(voice, peer, connections);
 
   return {
     connected,
     connect,
     disconnect,
     connections,
-    voiceStreams,
     useChannel: channels.useChannel,
     voice,
     setVoice,
+    mediaConnections,
   };
 };

src/layers/Network/logic/mic.ts

@@ -0,0 +1,54 @@
+import { useEffect, useState } from "react";
+import { useEnvironment } from "../../Environment";
+
+/**
+ * WHen enabled, will ask user for mic permissions and return the local microphone stream
+ * @param enabled
+ */
+export const useMicrophone = (enabled = true): MediaStream | undefined => {
+  const { paused } = useEnvironment();
+  const [firstPaused, setFirstPaused] = useState(true);
+  useEffect(() => setFirstPaused(paused && firstPaused), [paused, firstPaused]);
+
+  function iOS() {
+    return (
+      [
+        "iPad Simulator",
+        "iPhone Simulator",
+        "iPod Simulator",
+        "iPad",
+        "iPhone",
+        "iPod",
+      ].includes(navigator.platform) ||
+      // iPad on iOS 13 detection
+      (navigator.userAgent.includes("Mac") && "ontouchend" in document)
+    );
+  }
+
+  const [localStream, setLocalStream] = useState<MediaStream>();
+
+  // attempt to request permission for microphone, only try once
+  const [attempted, setAttempted] = useState(false);
+  useEffect(() => {
+    // https://bugs.webkit.org/show_bug.cgi?id=230902#c47
+    if (!enabled || attempted || (iOS() && firstPaused)) return;
+
+    setAttempted(true);
+
+    navigator.getUserMedia =
+      navigator.getUserMedia ||
+      navigator.webkitGetUserMedia ||
+      navigator.mozGetUserMedia ||
+      navigator.msGetUserMedia;
+
+    navigator.getUserMedia(
+      { audio: true },
+      (str) => setLocalStream(str),
+      (err) => {
+        console.error(err);
+      }
+    );
+  }, [attempted, enabled, firstPaused]);
+
+  return localStream;
+};