Add audio handling in the echo example (#58)

Author: LVala

examples/echo/example.exs

@@ -8,7 +8,13 @@ defmodule Peer do
 
   require Logger
 
-  alias ExWebRTC.{IceCandidate, PeerConnection, SessionDescription, RTPTransceiver}
+  alias ExWebRTC.{
+    IceCandidate,
+    PeerConnection,
+    MediaStreamTrack,
+    SessionDescription,
+    RTPTransceiver
+  }
 
   @ice_servers [
     %{urls: "stun:stun.l.google.com:19302"}
@@ -31,7 +37,17 @@ defmodule Peer do
 
         {:ok, pc} = PeerConnection.start_link(ice_servers: @ice_servers)
 
-        {:ok, %{conn: conn, stream: stream, peer_connection: pc, track_id: nil}}
+        state = %{
+          conn: conn,
+          stream: stream,
+          peer_connection: pc,
+          out_audio_track_id: nil,
+          out_video_track_id: nil,
+          in_audio_track_id: nil,
+          in_video_track_id: nil
+        }
+
+        {:ok, state}
 
       other ->
         Logger.error("Couldn't connect to the signalling server: #{inspect(other)}")
@@ -70,7 +86,7 @@ defmodule Peer do
 
   @impl true
   def handle_info({:ex_webrtc, _pid, msg}, state) do
-    handle_webrtc_message(msg, state)
+    state = handle_webrtc_message(msg, state)
     {:noreply, state}
   end
 
@@ -90,15 +106,17 @@ defmodule Peer do
     msg = %{"type" => "answer", "sdp" => answer.sdp}
     :gun.ws_send(state.conn, state.stream, {:text, Jason.encode!(msg)})
 
-    track = ExWebRTC.MediaStreamTrack.new(:video)
-    {:ok, _} = PeerConnection.add_track(pc, track)
+    video_track = MediaStreamTrack.new(:video)
+    audio_track = MediaStreamTrack.new(:audio)
+    {:ok, _} = PeerConnection.add_track(pc, video_track)
+    {:ok, _} = PeerConnection.add_track(pc, audio_track)
     {:ok, offer} = PeerConnection.create_offer(pc)
     :ok = PeerConnection.set_local_description(pc, offer)
     Logger.info("Sent SDP offer: #{inspect(offer.sdp)}")
     msg = %{"type" => "offer", "sdp" => offer.sdp}
     :gun.ws_send(state.conn, state.stream, {:text, Jason.encode!(msg)})
 
-    %{state | track_id: track.id}
+    %{state | out_audio_track_id: audio_track.id, out_video_track_id: video_track.id}
   end
 
   defp handle_ws_message(%{"type" => "answer", "sdp" => sdp}, state) do
@@ -138,19 +156,36 @@ defmodule Peer do
 
     msg = %{"type" => "ice", "data" => candidate}
     :gun.ws_send(state.conn, state.stream, {:text, Jason.encode!(msg)})
+    state
+  end
+
+  defp handle_webrtc_message({:track, track}, state) do
+    %MediaStreamTrack{kind: kind, id: id} = track
+
+    case kind do
+      :audio -> %{state | in_audio_track_id: id}
+      :video -> %{state | in_video_track_id: id}
+    end
   end
 
-  defp handle_webrtc_message({:rtp, _mid, _packet}, %{track_id: nil}) do
-    Logger.warning("Received RTP, but out transceiver has not beed created")
+  defp handle_webrtc_message({:rtp, id, packet}, %{in_audio_track_id: id} = state) do
+    PeerConnection.send_rtp(state.peer_connection, state.out_audio_track_id, packet)
+    state
+  end
+
+  defp handle_webrtc_message({:rtp, id, packet}, %{in_video_track_id: id} = state) do
+    PeerConnection.send_rtp(state.peer_connection, state.out_video_track_id, packet)
+    state
   end
 
-  defp handle_webrtc_message({:rtp, _mid, packet}, state) do
-    Logger.info("Received RTP: #{inspect(packet)}")
-    PeerConnection.send_rtp(state.peer_connection, state.track_id, packet)
+  defp handle_webrtc_message({:rtcp, packet}, state) do
+    Logger.info("Received RCTP: #{inspect(packet)}")
+    state
   end
 
-  defp handle_webrtc_message(msg, _state) do
-    Logger.warning("Received unknown ex_webrtc message: #{inspect(msg)}")
+  defp handle_webrtc_message(msg, state) do
+    Logger.warning("Received other ex_webrtc message: #{inspect(msg)}")
+    state
   end
 end
 

examples/echo/example.html

@@ -10,6 +10,7 @@
     <main>
         <h1>Elixir WebRTC Echo Example</h1>  
     </main>
+    <video id="videoPlayer" autoplay controls></video>
 	<script src="example.js"></script>
   </body>
 </html>

examples/echo/example.js

@@ -1,6 +1,10 @@
 const pcConfig = { 'iceServers': [{ 'urls': 'stun:stun.l.google.com:19302' },] };
 
 const start_connection = async (ws) => {
+  // setup default MediaStream for the player
+  const videoPlayer = document.getElementById("videoPlayer");
+  videoPlayer.srcObject = new MediaStream();
+
   const pc = new RTCPeerConnection(pcConfig);
 
   pc.onconnectionstatechange = _ => console.log("Connection state changed:", pc.connectionState);
@@ -9,12 +13,7 @@ const start_connection = async (ws) => {
   pc.onicegatheringstatechange = _ => console.log("ICE gathering state changed:", pc.iceGatheringState);
   pc.onsignalingstatechange = _ => console.log("Signaling state changed:", pc.signalingState);
   pc.ontrack = event => {
-    const videoPlayer = document.createElement("video");
-    videoPlayer.srcObject = event.streams[0];
-    videoPlayer.onloadedmetadata = () => {
-      videoPlayer.play();
-    };
-    document.body.appendChild(videoPlayer);
+    videoPlayer.srcObject.addTrack(event.track);
   };
   pc.onicecandidate = event => {
     console.log("New local ICE candidate:", event.candidate);
@@ -24,14 +23,7 @@ const start_connection = async (ws) => {
     }
   };
 
-  const localStream = await navigator.mediaDevices.getUserMedia({ video: true });
-  const localVideoPlayer = document.createElement("video");
-  localVideoPlayer.srcObject = localStream;
-  localVideoPlayer.onloadedmetadata = () => {
-    localVideoPlayer.play();
-  };
-  document.body.appendChild(localVideoPlayer);
-
+  const localStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
   for (const track of localStream.getTracks()) {
     pc.addTrack(track, localStream);
   }

lib/ex_webrtc/peer_connection.ex

@@ -655,6 +655,16 @@ defmodule ExWebRTC.PeerConnection do
     end
   end
 
+  @impl true
+  def handle_info({:dtls_transport, _pid, {:rtcp, data}}, state) do
+    case ExRTCP.CompoundPacket.decode(data) do
+      {:ok, packets} -> notify(state.owner, {:rtcp, packets})
+      {:error, _res} -> Logger.info("Failed to decode RTCP packet")
+    end
+
+    {:noreply, state}
+  end
+
   @impl true
   def handle_info(msg, state) do
     Logger.info("OTHER MSG #{inspect(msg)}")

lib/ex_webrtc/peer_connection/demuxer.ex

@@ -67,14 +67,14 @@ defmodule ExWebRTC.PeerConnection.Demuxer do
 
   defp update_ssrc_mapping(%__MODULE__{mid_ext_id: id} = demuxer, %Packet{ssrc: ssrc} = packet) do
     mid =
-      Enum.find_value(packet.extensions, fn
-        %Extension{id: ^id} = ext ->
+      case Packet.fetch_extension(packet, id) do
+        {:ok, %Extension{id: ^id} = ext} ->
           {:ok, mid_ext} = SourceDescription.from_raw(ext)
           mid_ext.text
 
-        _ ->
+        :error ->
           nil
-      end)
+      end
 
     case Map.fetch(demuxer.ssrc_to_mid, ssrc) do
       {:ok, last_mid} when mid != nil and mid != last_mid ->

lib/ex_webrtc/rtp/opus_payloader.ex

@@ -10,6 +10,6 @@ defmodule ExWebRTC.RTP.OpusPayloader do
   """
   @spec payload(binary()) :: ExRTP.Packet.t()
   def payload(packet) when packet != <<>> do
-    ExRTP.Packet.new(packet, 0, 0, 0, 0)
+    ExRTP.Packet.new(packet)
   end
 end

lib/ex_webrtc/rtp/vp8_payloader.ex

@@ -36,11 +36,11 @@ defmodule ExWebRTC.RTP.VP8Payloader do
 
     [first_rtp_payload | next_rtp_payloads] = rtp_payloads
 
-    first_rtp_packet = ExRTP.Packet.new(@first_chunk_descriptor <> first_rtp_payload, 0, 0, 0, 0)
+    first_rtp_packet = ExRTP.Packet.new(@first_chunk_descriptor <> first_rtp_payload)
 
     next_rtp_packets =
       for rtp_payload <- next_rtp_payloads do
-        ExRTP.Packet.new(@next_chunk_descriptor <> rtp_payload, 0, 0, 0, 0)
+        ExRTP.Packet.new(@next_chunk_descriptor <> rtp_payload)
       end
 
     rtp_packets = [first_rtp_packet | next_rtp_packets]

lib/ex_webrtc/rtp_sender.ex

@@ -78,7 +78,7 @@ defmodule ExWebRTC.RTPSender do
 
     packet =
       packet
-      |> ExRTP.Packet.set_extension(:two_byte, [mid_ext])
+      |> ExRTP.Packet.add_extension(mid_ext)
       |> ExRTP.Packet.encode()
 
     sender = %{sender | last_seq_num: next_seq_num}

mix.exs

@@ -53,7 +53,7 @@ defmodule ExWebRTC.MixProject do
       {:ex_ice, "~> 0.4.0"},
       {:ex_dtls, "~> 0.15.0"},
       {:ex_libsrtp, "~> 0.7.1"},
-      {:ex_rtp, "~> 0.2.0"},
+      {:ex_rtp, "~> 0.3.0"},
       {:ex_rtcp, "~> 0.1.0"},
       {:crc, "~> 0.10"},
 

mix.lock

@@ -16,7 +16,7 @@
   "ex_ice": {:hex, :ex_ice, "0.4.0", "fdce80373e8bc519be99825b970f7388694bcbe47ad1be99a57bd3b00223a22b", [:mix], [{:ex_stun, "~> 0.1.0", [hex: :ex_stun, repo: "hexpm", optional: false]}], "hexpm", "447a3ba75ebf482b0c07768d942a312e40f047f29d34160b220dd56a59647aa2"},
   "ex_libsrtp": {:hex, :ex_libsrtp, "0.7.2", "211bd89c08026943ce71f3e2c0231795b99cee748808ed3ae7b97cd8d2450b6b", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.3", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:unifex, "~> 1.1", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "2e20645d0d739a4ecdcf8d4810a0c198120c8a2f617f2b75b2e2e704d59f492a"},
   "ex_rtcp": {:hex, :ex_rtcp, "0.1.0", "2e02d23fc6ccc7e00aed13358ffdbcb23e34d3a7f35c66cadaa54447383ecae4", [:mix], [], "hexpm", "1c9a7e636f3950fbcefedce31f3e4ca60b84ea80ad519789f9d215167c60cb2b"},
-  "ex_rtp": {:hex, :ex_rtp, "0.2.0", "b5a7eb093a2177dd8faa5c2cf82cbd3722bed8676dbff81c3ae3e30a32a8e407", [:mix], [], "hexpm", "a48080edeca99d90aca5a2a8129bc9d46548f1055d957d775d751fb390f9988c"},
+  "ex_rtp": {:hex, :ex_rtp, "0.3.0", "d18d0de39875958902816284f79c8cd51ec83d39bdd14edf6a5b069926268a43", [:mix], [], "hexpm", "cec8398237095b02438842cfc74487c3cbaeb7fe29e4c62ae11457a0ddd99754"},
   "ex_sdp": {:hex, :ex_sdp, "0.14.0", "4f6a3fed22ec312c6ad89def77ad0a32f81094cb688c845c5fbb0887f35f5f63", [:mix], [{:bunch, "~> 1.3", [hex: :bunch, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}], "hexpm", "bc3de2b015b8eee859ac6e48b8045bf08bfbd8dcdb075db76c19fc1365a9cf90"},
   "ex_stun": {:hex, :ex_stun, "0.1.0", "252474bf4c8519fbf4bc0fbfc6a1b846a634b1478c65dbbfb4b6ab4e33c2a95a", [:mix], [], "hexpm", "629fc8be45b624a92522f81d85ba001877b1f0745889a2419bdb678790d7480c"},
   "excoveralls": {:hex, :excoveralls, "0.17.1", "83fa7906ef23aa7fc8ad7ee469c357a63b1b3d55dd701ff5b9ce1f72442b2874", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "95bc6fda953e84c60f14da4a198880336205464e75383ec0f570180567985ae0"},

test/ex_webrtc/peer_connection/demuxer_test.exs

@@ -19,7 +19,7 @@ defmodule ExWebRTC.PeerConnection.DemuxerTest do
 
   @packet Packet.encode(@deserialized_packet)
   @packet_mid @deserialized_packet
-              |> Packet.set_extension(:two_byte, [%Extension{id: 15, data: @mid}])
+              |> Packet.add_extension(%Extension{id: 15, data: @mid})
               |> Packet.encode()
 
   @demuxer %Demuxer{mid_ext_id: 15}

test/ex_webrtc/peer_connection_test.exs

@@ -803,13 +803,13 @@ defmodule ExWebRTC.PeerConnectionTest do
 
     # send data
     payload = <<3, 2, 5>>
-    packet = ExRTP.Packet.new(payload, 111, 50_000, 3_000, 5_000)
+    packet = ExRTP.Packet.new(payload)
     :ok = PeerConnection.send_rtp(pc1, track1.id, packet)
 
     assert_receive {:ex_webrtc, ^pc2, {:rtp, ^id2, %ExRTP.Packet{payload: ^payload}}}
 
     payload = <<7, 8, 9>>
-    packet = ExRTP.Packet.new(payload, 111, 50_000, 3_000, 5_000)
+    packet = ExRTP.Packet.new(payload)
     :ok = PeerConnection.send_rtp(pc2, track2.id, packet)
 
     assert_receive {:ex_webrtc, ^pc1, {:rtp, ^id1, %ExRTP.Packet{payload: ^payload}}}

test/ex_webrtc/rtp/vp8_depayloader_test.exs

@@ -12,7 +12,7 @@ defmodule ExWebRTC.RTP.VP8DepayloaderTest do
     vp8_payload = %VP8Payload{n: 0, s: 1, pid: 0, payload: data}
     vp8_payload = VP8Payload.serialize(vp8_payload)
 
-    packet = ExRTP.Packet.new(vp8_payload, 0, 0, 0, 0, marker: true)
+    packet = ExRTP.Packet.new(vp8_payload, marker: true)
 
     assert {:ok, ^data, %{current_frame: nil, current_timestamp: nil} = depayloader} =
              VP8Depayloader.write(depayloader, packet)
@@ -21,7 +21,7 @@ defmodule ExWebRTC.RTP.VP8DepayloaderTest do
     vp8_payload = %VP8Payload{n: 0, s: 0, pid: 0, payload: data}
     vp8_payload = VP8Payload.serialize(vp8_payload)
 
-    packet = ExRTP.Packet.new(vp8_payload, 0, 0, 0, 0)
+    packet = ExRTP.Packet.new(vp8_payload)
 
     assert {:ok, %{current_frame: nil, current_timestamp: nil} = depayloader} =
              VP8Depayloader.write(depayloader, packet)
@@ -30,7 +30,7 @@ defmodule ExWebRTC.RTP.VP8DepayloaderTest do
     vp8_payload = %VP8Payload{n: 0, s: 1, pid: 0, payload: data}
     vp8_payload = VP8Payload.serialize(vp8_payload)
 
-    packet = ExRTP.Packet.new(vp8_payload, 0, 0, 0, 0)
+    packet = ExRTP.Packet.new(vp8_payload)
 
     assert {:ok, %{current_frame: ^data, current_timestamp: 0} = depayloader} =
              VP8Depayloader.write(depayloader, packet)
@@ -39,7 +39,7 @@ defmodule ExWebRTC.RTP.VP8DepayloaderTest do
     vp8_payload = %VP8Payload{n: 0, s: 1, pid: 0, payload: data2}
     vp8_payload = VP8Payload.serialize(vp8_payload)
 
-    packet = ExRTP.Packet.new(vp8_payload, 0, 0, 3000, 0)
+    packet = ExRTP.Packet.new(vp8_payload, timestamp: 3000)
 
     assert {:ok, %{current_frame: ^data2, current_timestamp: 3000} = depayloader} =
              VP8Depayloader.write(depayloader, packet)
@@ -49,7 +49,7 @@ defmodule ExWebRTC.RTP.VP8DepayloaderTest do
     vp8_payload = %VP8Payload{n: 0, s: 0, pid: 0, payload: data2}
     vp8_payload = VP8Payload.serialize(vp8_payload)
 
-    packet = ExRTP.Packet.new(vp8_payload, 0, 0, 6000, 0)
+    packet = ExRTP.Packet.new(vp8_payload, timestamp: 6000)
 
     assert {:ok, %{current_frame: nil, current_timestamp: nil}} =
              VP8Depayloader.write(depayloader, packet)

test/ex_webrtc/rtp_sender_test.exs

@@ -27,7 +27,7 @@ defmodule ExWebRTC.RTPSenderTest do
     sender = RTPSender.new(track, codec, rtp_hdr_exts, "1", @ssrc)
     sender = %RTPSender{sender | last_seq_num: 10_000}
 
-    packet = ExRTP.Packet.new(<<>>, 0, 0, 0, 0)
+    packet = ExRTP.Packet.new(<<>>)
 
     {packet, sender} = RTPSender.send(sender, packet)
 
@@ -45,7 +45,7 @@ defmodule ExWebRTC.RTPSenderTest do
 
     # check sequence number rollover and marker flag
     sender = %RTPSender{sender | last_seq_num: @max_seq_num}
-    packet = ExRTP.Packet.new(<<>>, 0, 1, 0, 0, marker: true)
+    packet = ExRTP.Packet.new(<<>>, sequence_number: 1, marker: true)
     {packet, _sender} = RTPSender.send(sender, packet)
     {:ok, packet} = ExRTP.Packet.decode(packet)
     assert packet.sequence_number == 0