completed testing/fixing for audio frame data callbacks

added code for video frame data callbacks

Author: robin-raymond

idl/AudioBuffers.idl

@@ -34,7 +34,7 @@ namespace org
 {
   namespace webRtc
   {
-    [special]
+    [special, disposable]
     interface AudioData
     {
       /// <summary>

idl/MediaStreamTrack.idl

@@ -34,6 +34,22 @@ namespace org
 {
   namespace webRtc
   {
+    /// <summary>
+    /// The event is fired when an audio buffer is ready for processing.
+    /// <summary>
+    [disposable]
+    interface VideoFrameBufferEvent
+    {
+      [constructor, delete, default]
+      void VideoFrameBufferEvent();
+
+      /// <summary>
+      /// Gets the video buffer for further processing.
+      /// <summary>
+      [getter]
+      VideoFrameBuffer buffer;
+    };
+
     /// <summary>
     /// A MediaStreamTrack object represents a media source in the User Agent.
     /// An example source is a device connected to the User Agent. Other
@@ -129,6 +145,12 @@ namespace org
       [event]
       void onFrameRateChanged(float frameRate);
 
+      /// <summary>
+      /// Fires events when an video buffer is available.
+      /// </summary>
+      [event]
+      void onVideoFrame(VideoFrameBufferEvent buffer);
+
     };
   }
 }

idl/VideoBuffers.idl

@@ -0,0 +1,187 @@
+/*
+
+ Copyright (c) 2017, Optical Tone Ltd.
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ The views and conclusions contained in the software and documentation are those
+ of the authors and should not be interpreted as representing official policies,
+ either expressed or implied, of the FreeBSD Project.
+ 
+ */
+
+
+namespace org
+{
+  namespace webRtc
+  {
+    [special, disposable]
+    interface VideoData
+    {
+      [constructor, default, delete]
+      void VideoData();
+
+      [getter]
+      bool is8BitColorSpace;
+
+      [getter]
+      bool is16BitColorSpace;
+
+      /// <summary>
+      /// Gets or sets the video 8 bit color space data.
+      /// <summary>
+      [getter]
+      std::list<uint8> data8bit;
+
+      /// <summary>
+      /// Gets or sets the video 16 bit color space data.
+      /// <summary>
+      [getter]
+      std::list<uint16> data16bit;
+    };
+
+    [disposable]
+    interface VideoFrameBuffer
+    {
+      [constructor, default, delete]
+      void VideoFrameBuffer();
+
+      /// <summary>
+      /// Gets the video frame type.
+      /// </summary>
+      [getter]
+      VideoFrameBufferType type;
+
+      /// <summary>
+      /// Gets the width of the video frame.
+      /// <summary>
+      [getter]
+      int width;
+
+      /// <summary>
+      /// Gets the width of the video frame.
+      /// <summary>
+      [getter]
+      int height;
+
+      /// <summary>
+      /// Convert to a new (or return an existing) I420 frame buffer (if possible).
+      /// <summary>
+      VideoFramePlanarYuvBuffer toI420();
+
+      /// <summary>
+      /// Gets the YUV frame if the buffer type is YUV format.
+      /// <summary>
+      [getter]
+      VideoFramePlanarYuvBuffer yuvFrame;
+
+      /// <summary>
+      /// Gets the I420A frame if the buffer type is YUVA.
+      /// <summary>
+      [getter]
+      VideoFramePlanarYuvaBuffer yuvaFrame;
+
+      /// <summary>
+      /// Gets a native frame if the buffer type is native.
+      /// <summary>
+      [getter]
+      VideoFrameNativeBuffer nativeFrame;
+    };
+
+    interface VideoFramePlanarYuvBuffer : VideoFrameBuffer
+    {
+      [constructor, default, delete]
+      void VideoFramePlanarYuvBuffer();
+
+      /// <summary>
+      /// Gets the chroma width of the video frame.
+      /// <summary>
+      [getter]
+      int chromaWidth;
+
+      /// <summary>
+      /// Gets the chroma width of the video frame.
+      /// <summary>
+      [getter]
+      int chromaHeight;
+
+      /// <summary>
+      /// Gets the number of Y steps (in terms of VideoData) between
+      /// successive rows for the given plane.
+      /// <summary>
+      [getter]
+      int strideY;
+
+      /// <summary>
+      /// Gets the number of U steps (in terms of VideoData) between
+      /// successive rows for the given plane.
+      /// <summary>
+      [getter]
+      int strideU;
+
+      /// <summary>
+      /// Gets the number of V steps (in terms of VideoData) between
+      /// successive rows for the given plane.
+      /// <summary>
+      [getter]
+      int strideV;
+
+      [getter]
+      VideoData y;
+
+      [getter]
+      VideoData u;
+
+      [getter]
+      VideoData v;
+    };
+
+    interface VideoFramePlanarYuvaBuffer : VideoFramePlanarYuvBuffer
+    {
+      [constructor, default, delete]
+      void VideoFramePlanarYuvaBuffer();
+
+      /// <summary>
+      /// Gets the number of A steps (in terms of VideoData) between
+      /// successive rows for the given plane.
+      /// <summary>
+      [getter]
+      int strideA;
+
+      [getter]
+      VideoData a;
+    };
+
+    interface VideoFrameNativeBuffer : VideoFrameBuffer
+    {
+      [constructor, default, delete]
+      void VideoFrameNativeBuffer();
+
+      /// <summary>
+      /// Gets the native sample from the video frame.
+      /// <summary>
+      [getter]
+      MediaSample sample;
+
+    };
+  }
+}

idl/VideoCapturer.idl

@@ -208,10 +208,10 @@ namespace org
       VideoCaptureState state;
 
       /// <summary>
-      /// Event indicates new captured sample available.
+      /// Event fires when a new video frame buffer is available.
       /// </summary>
       [event]
-      void onVideoSampleReceived(MediaSample sample);
+      void onVideoFrame(VideoFrameBufferEvent event);
 
     };
   }

idl/WebRtc.idl

@@ -94,7 +94,13 @@ namespace org
     interface RTCCertificate;
     interface RTCKeyParams;
     interface VideoCapturer;
+    interface VideoData;
     interface WebRtcFactory;
+    interface VideoFrameBuffer;
+    interface VideoFrameBufferEvent;
+    interface VideoFramePlanarYuvBuffer;
+    interface VideoFramePlanarYuvaBuffer;
+    interface VideoFrameNativeBuffer;
 
     typedef zs::PromiseRejectionReason<RTCError> PromiseWithRejectionRTCError;
     typedef zs::PromiseWith<RTCSessionDescription> PromiseWithDescription;
@@ -144,7 +150,6 @@ namespace org
     /// <summary>
     /// A message queue for receiving delegate events.
     /// </summary>
-    [special]
     interface EventQueue
     {
       /// <summary>
@@ -153,12 +158,6 @@ namespace org
       [constructor, delete]
       void EventQueue();
 
-      /// <summary>
-      /// Constructs an event queue object from a native queue type.
-      /// </summary>
-      [constructor, default, altname(EventQueueWithNativeQueue)]
-      void EventQueue(Any queue);
-
       /// <summary>
       /// Gets or creates an event queue by queue name. If the queue is not
       /// found then a new thread queue will be created.
@@ -181,12 +180,6 @@ namespace org
       /// </summary>
       [static]
       EventQueue getDefaultForUi();
-
-      /// <summary>
-      /// Gets a native queue from an event queue object.
-      /// </summary>
-      [getter]
-      Any queue;
     };
 
     [special, static, platform(webhidden)]
@@ -214,10 +207,16 @@ namespace org
       EventQueue queue;
 
       /// <summary>
-      /// Gets or sets the default queue for processing audio frame events.
-      /// If not specified then the GUI queue is used.
+      /// Gets or sets the default queue for processing audio capture frame
+      /// events. If not specified then the GUI queue is used.
+      /// </summary>
+      EventQueue audioCaptureFrameProcessingQueue;
+
+      /// <summary>
+      /// Gets or sets the default queue for processing audio render frame
+      /// events. If not specified then the GUI queue is used.
       /// </summary>
-      EventQueue audioFrameProcessingQueue;
+      EventQueue audioRenderFrameProcessingQueue;
 
       /// <summary>
       /// Gets or sets the default queue for processing video frame

idl/WebRtcFactory.idl

@@ -100,14 +100,20 @@ namespace org
     {
 
       /// <summary>
-      /// Gets or sets if audio capturing is be enabled.
+      /// Gets or sets if audio capturing is enabled.
       /// </summary>
       bool audioCapturingEnabled = true;
 
       /// <summary>
       /// Gets or sets if audio rendering is enabled.
       /// </summary>
       bool audioRenderingEnabled = true;
+
+      /// <summary>
+      /// Gets or sets if audio post-capture and pre-render events should be
+      /// enabled.
+      /// </summary>
+      bool enableAudioBufferEvents = false;
     };
 
     /// <summary>

idl/config.json

@@ -51,6 +51,7 @@
         "SessionDescription.idl",
         "PeerConnection.idl",
         "AudioBuffers.idl",
+        "VideoBuffers.idl",
         "WebRtcFactory.idl",
       ]
     }

idl/enums.idl

@@ -950,6 +950,15 @@ namespace org
       CustomRenderProcessingRuntimeSetting
     };
 
+    enum VideoFrameBufferType
+    {
+      Native,
+      I420,
+      I420A,
+      I444,
+      I010
+    };
+
   }
 }
 

windows/wrapper/impl_org_webRtc_AudioBuffer.cpp

@@ -84,7 +84,7 @@ wrapper::org::webRtc::AudioDataPtr wrapper::impl::org::webRtc::AudioBuffer::chan
   ZS_ASSERT(native_);
   if (!native_)
     return {};
-  return UseAudioData::toWrapper(native_->channels()[channel], native_->num_channels());
+  return UseAudioData::toWrapper(native_->channels()[channel], native_->num_frames());
 }
 
 //------------------------------------------------------------------------------