Add support for VP8 simulcast (#198)

Author: mickel8

guides/advanced/simulcast.md

@@ -86,7 +86,7 @@ This process is known as munging.
 alias ExWebRTC.PeerConnection
 alias ExWebRTC.RTP.{H264, Munger}
 
-m = Munger.new(90_000)
+m = Munger.new(:h264, 90_000)
 ```
 
 2. When a packet from an encoding that we want to forward arrives, rewrite its sequnce number and timestamp:

lib/ex_webrtc/rtp/munger.ex

@@ -2,6 +2,7 @@ defmodule ExWebRTC.RTP.Munger do
   @moduledoc """
   RTP Munger allows for converting RTP packet timestamps and sequence numbers
   to a common domain.
+  It also rewrites parts of the RTP payload that may require a similar behaviour.
 
   This is useful when e.g. changing between Simulcast layers - the sender sends
   three separate RTP streams (also called layers or encodings), but the receiver can receive only a
@@ -12,7 +13,7 @@ defmodule ExWebRTC.RTP.Munger do
   # and this is a GenServer
 
   def init() do
-    {:ok, %{munger: Munger.new(90_000), layer: "h"}}
+    {:ok, %{munger: Munger.new(:h264, 90_000), layer: "h"}}
   end
 
   def handle_info({:ex_webrtc, _from, {:rtp, _id, rid, packet}}, state) do
@@ -34,6 +35,8 @@ defmodule ExWebRTC.RTP.Munger do
   """
 
   alias ExRTP.Packet
+  alias ExWebRTC.RTPCodecParameters
+  alias ExWebRTC.RTP.VP8
 
   @max_rtp_ts 0xFFFFFFFF
   @max_rtp_sn 0xFFFF
@@ -47,14 +50,16 @@ defmodule ExWebRTC.RTP.Munger do
   # * `sn_offset` - offset for sequence numbers
   # * `ts_offset` - offset for timestamps
   # * `update?` - flag telling if the next munged packets belongs to a new encoding
+  # * `vp8_munger` - VP8 munger, only used when RTP packets contain VP8 codec
   @opaque t() :: %__MODULE__{
             clock_rate: non_neg_integer(),
             rtp_sn: non_neg_integer() | nil,
             rtp_ts: non_neg_integer() | nil,
             wc_ts: integer() | nil,
             sn_offset: integer(),
             ts_offset: integer(),
-            update?: boolean()
+            update?: boolean(),
+            vp8_munger: VP8.Munger.t() | nil
           }
 
   @enforce_keys [:clock_rate]
@@ -64,19 +69,31 @@ defmodule ExWebRTC.RTP.Munger do
               :wc_ts,
               sn_offset: 0,
               ts_offset: 0,
-              update?: false
+              update?: false,
+              vp8_munger: nil
             ] ++ @enforce_keys
 
   @doc """
   Creates a new `t:ExWebRTC.RTP.Munger.t/0`.
 
   `clock_rate` is the clock rate of the codec carried in munged RTP packets.
   """
-  @spec new(non_neg_integer()) :: t()
-  def new(clock_rate) do
+  @spec new(:h264 | :vp8 | RTPCodecParameters.t(), non_neg_integer()) :: t()
+  def new(:h264, clock_rate) do
     %__MODULE__{clock_rate: clock_rate}
   end
 
+  def new(:vp8, clock_rate) do
+    %__MODULE__{clock_rate: clock_rate, vp8_munger: VP8.Munger.new()}
+  end
+
+  def new(%RTPCodecParameters{} = codec_params) do
+    case codec_params.mime_type do
+      "video/H264" -> new(:h264, codec_params.clock_rate)
+      "video/VP8" -> new(:vp8, codec_params.clock_rate)
+    end
+  end
+
   @doc """
   Informs the munger that the next packet passed to `munge/2` will come
   from a different RTP stream.
@@ -90,17 +107,30 @@ defmodule ExWebRTC.RTP.Munger do
   @spec munge(t(), Packet.t()) :: {Packet.t(), t()}
   def munge(%{rtp_sn: nil} = munger, packet) do
     # first packet ever munged
+    vp8_munger = munger.vp8_munger && VP8.Munger.init(munger.vp8_munger, packet.payload)
+
     munger = %__MODULE__{
       munger
       | rtp_sn: packet.sequence_number,
         rtp_ts: packet.timestamp,
-        wc_ts: get_wc_ts(packet)
+        wc_ts: get_wc_ts(packet),
+        vp8_munger: vp8_munger
     }
 
     {packet, munger}
   end
 
   def munge(munger, packet) when munger.update? do
+    {vp8_munger, rtp_payload} =
+      if munger.vp8_munger do
+        vp8_munger = VP8.Munger.update(munger.vp8_munger, packet.payload)
+        VP8.Munger.munge(vp8_munger, packet.payload)
+      else
+        {munger.vp8_munger, packet.payload}
+      end
+
+    packet = %ExRTP.Packet{packet | payload: rtp_payload}
+
     wc_ts = get_wc_ts(packet)
 
     native_in_sec = System.convert_time_unit(1, :second, :native)
@@ -124,7 +154,8 @@ defmodule ExWebRTC.RTP.Munger do
       | rtp_sn: new_packet.sequence_number,
         rtp_ts: new_packet.timestamp,
         wc_ts: wc_ts,
-        update?: false
+        update?: false,
+        vp8_munger: vp8_munger
     }
 
     {new_packet, munger}
@@ -135,6 +166,15 @@ defmodule ExWebRTC.RTP.Munger do
     # the first packet after the encoding update
     # as these might conflict with packets from the previous layer
     # and we should change on a keyframe anyways
+    {vp8_munger, rtp_payload} =
+      if munger.vp8_munger do
+        VP8.Munger.munge(munger.vp8_munger, packet.payload)
+      else
+        {munger.vp8_munger, packet.payload}
+      end
+
+    packet = %ExRTP.Packet{packet | payload: rtp_payload}
+
     wc_ts = get_wc_ts(packet)
 
     new_packet = adjust_packet(munger, packet)
@@ -148,10 +188,11 @@ defmodule ExWebRTC.RTP.Munger do
           munger
           | rtp_sn: new_packet.sequence_number,
             rtp_ts: new_packet.timestamp,
-            wc_ts: wc_ts
+            wc_ts: wc_ts,
+            vp8_munger: vp8_munger
         }
       else
-        munger
+        %__MODULE__{munger | vp8_munger: vp8_munger}
       end
 
     {new_packet, munger}

lib/ex_webrtc/rtp/vp8.ex

@@ -0,0 +1,26 @@
+defmodule ExWebRTC.RTP.VP8 do
+  @moduledoc """
+  Utilities for RTP packets carrying VP8 encoded payload.
+  """
+
+  alias ExRTP.Packet
+  alias ExWebRTC.RTP.VP8
+
+  @doc """
+  Checks whether RTP payload contains VP8 keyframe.
+  """
+  @spec keyframe?(Packet.t()) :: boolean()
+  def keyframe?(%Packet{payload: rtp_payload}) do
+    # RTP payload contains VP8 keyframe when P bit in VP8 payload header is set to 0
+    # besides this S bit (start of VP8 partition) and PID (partition index)
+    # have to be 1 and 0 respectively
+    # for more information refer to RFC 7741 Sections 4.2 and 4.3
+
+    with {:ok, vp8_payload} <- VP8.Payload.parse(rtp_payload),
+         <<_size0::3, _h::1, _ver::3, p::1, _size1::8, _size2::8, _rest::binary>> <- rtp_payload do
+      vp8_payload.s == 1 and vp8_payload.pid == 0 and p == 0
+    else
+      _err -> false
+    end
+  end
+end

lib/ex_webrtc/rtp/vp8/munger.ex

@@ -0,0 +1,117 @@
+defmodule ExWebRTC.RTP.VP8.Munger do
+  @moduledoc false
+  # Module responsible for rewriting VP8 RTP payload fields
+  # to provide transparent switch between simulcast encodings.
+  import Bitwise
+
+  alias ExWebRTC.RTP.VP8
+
+  @type t() :: %__MODULE__{
+          pic_id_used: boolean(),
+          last_pic_id: integer(),
+          pic_id_offset: integer(),
+          tl0picidx_used: boolean(),
+          last_tl0picidx: integer(),
+          tl0picidx_offset: integer(),
+          keyidx_used: boolean(),
+          last_keyidx: integer(),
+          keyidx_offset: integer()
+        }
+
+  defstruct pic_id_used: false,
+            last_pic_id: 0,
+            pic_id_offset: 0,
+            tl0picidx_used: false,
+            last_tl0picidx: 0,
+            tl0picidx_offset: 0,
+            keyidx_used: false,
+            last_keyidx: 0,
+            keyidx_offset: 0
+
+  @spec new() :: t()
+  def new() do
+    %__MODULE__{}
+  end
+
+  @spec init(t(), binary()) :: t()
+  def init(vp8_munger, rtp_payload) do
+    {:ok, vp8_payload} = VP8.Payload.parse(rtp_payload)
+
+    last_pic_id = vp8_payload.picture_id || 0
+    last_tl0picidx = vp8_payload.tl0picidx || 0
+    last_keyidx = vp8_payload.keyidx || 0
+
+    %__MODULE__{
+      vp8_munger
+      | pic_id_used: vp8_payload.picture_id != nil,
+        last_pic_id: last_pic_id,
+        tl0picidx_used: vp8_payload.tl0picidx != nil,
+        last_tl0picidx: last_tl0picidx,
+        keyidx_used: vp8_payload.keyidx != nil,
+        last_keyidx: last_keyidx
+    }
+  end
+
+  @spec update(t(), binary()) :: t()
+  def update(vp8_munger, rtp_payload) do
+    {:ok, vp8_payload} = VP8.Payload.parse(rtp_payload)
+
+    %VP8.Payload{
+      keyidx: keyidx,
+      picture_id: pic_id,
+      tl0picidx: tl0picidx
+    } = vp8_payload
+
+    pic_id_offset = (vp8_munger.pic_id_used && pic_id - vp8_munger.last_pic_id - 1) || 0
+
+    tl0picidx_offset =
+      (vp8_munger.tl0picidx_used && tl0picidx - vp8_munger.last_tl0picidx - 1) || 0
+
+    keyidx_offset = (vp8_munger.keyidx_used && keyidx - vp8_munger.last_keyidx - 1) || 0
+
+    %__MODULE__{
+      vp8_munger
+      | pic_id_offset: pic_id_offset,
+        tl0picidx_offset: tl0picidx_offset,
+        keyidx_offset: keyidx_offset
+    }
+  end
+
+  @spec munge(t(), binary()) :: {t(), binary()}
+  def munge(vp8_munger, <<>> = rtp_payload), do: {vp8_munger, rtp_payload}
+
+  def munge(vp8_munger, rtp_payload) do
+    {:ok, vp8_payload} = VP8.Payload.parse(rtp_payload)
+
+    %VP8.Payload{
+      keyidx: keyidx,
+      picture_id: pic_id,
+      tl0picidx: tl0picidx
+    } = vp8_payload
+
+    munged_pic_id = pic_id && rem(pic_id + (1 <<< 15) - vp8_munger.pic_id_offset, 1 <<< 15)
+
+    munged_tl0picidx =
+      tl0picidx && rem(tl0picidx + (1 <<< 8) - vp8_munger.tl0picidx_offset, 1 <<< 8)
+
+    munged_keyidx = keyidx && rem(keyidx + (1 <<< 5) - vp8_munger.keyidx_offset, 1 <<< 5)
+
+    vp8_payload =
+      %VP8.Payload{
+        vp8_payload
+        | keyidx: munged_keyidx,
+          picture_id: munged_pic_id,
+          tl0picidx: munged_tl0picidx
+      }
+      |> VP8.Payload.serialize()
+
+    vp8_munger = %__MODULE__{
+      vp8_munger
+      | last_pic_id: munged_pic_id,
+        last_tl0picidx: munged_tl0picidx,
+        last_keyidx: munged_keyidx
+    }
+
+    {vp8_munger, vp8_payload}
+  end
+end

test/ex_webrtc/rtp/munger_test.exs

@@ -11,7 +11,7 @@ defmodule ExWebRTC.RTP.MungerTest do
   @packet Packet.new(<<0::128*8>>)
 
   test "assigns sequence numbers properly" do
-    munger = Munger.new(@clock_rate)
+    munger = Munger.new(:h264, @clock_rate)
 
     l1_packet = %{@packet | sequence_number: 100}
     {^l1_packet, munger} = Munger.munge(munger, l1_packet)
@@ -45,7 +45,7 @@ defmodule ExWebRTC.RTP.MungerTest do
   end
 
   test "handles input sequence number rollover" do
-    munger = Munger.new(@clock_rate)
+    munger = Munger.new(:h264, @clock_rate)
 
     l1_packet = %{@packet | sequence_number: 100}
     {^l1_packet, munger} = Munger.munge(munger, l1_packet)
@@ -77,7 +77,7 @@ defmodule ExWebRTC.RTP.MungerTest do
   end
 
   test "handles output sequence number rollover" do
-    munger = Munger.new(@clock_rate)
+    munger = Munger.new(:h264, @clock_rate)
 
     l1_packet = %{@packet | sequence_number: @max_sn - 2}
     {^l1_packet, munger} = Munger.munge(munger, l1_packet)
@@ -98,7 +98,7 @@ defmodule ExWebRTC.RTP.MungerTest do
   end
 
   test "assigns timestamps properly" do
-    munger = Munger.new(@clock_rate)
+    munger = Munger.new(:h264, @clock_rate)
 
     l1_packet = %{@packet | sequence_number: 100, timestamp: 5000}
     {^l1_packet, munger} = Munger.munge(munger, l1_packet)
@@ -134,7 +134,7 @@ defmodule ExWebRTC.RTP.MungerTest do
   end
 
   test "handles input timestamp rollover" do
-    munger = Munger.new(@clock_rate)
+    munger = Munger.new(:h264, @clock_rate)
 
     l1_packet = %{@packet | sequence_number: 100, timestamp: 5000}
     {^l1_packet, munger} = Munger.munge(munger, l1_packet)
@@ -152,7 +152,7 @@ defmodule ExWebRTC.RTP.MungerTest do
   end
 
   test "handles output timestamp rollover" do
-    munger = Munger.new(@clock_rate)
+    munger = Munger.new(:h264, @clock_rate)
 
     l1_packet = %{@packet | sequence_number: 100, timestamp: @max_ts}
     {^l1_packet, munger} = Munger.munge(munger, l1_packet)