Handle scaling frames in RTCVideoEncoder

This CL adds support for scaling incoming video frames in RTCVideoEncoder.
Earlier, we expected WebRTC to only send frames that have the same size
as the initialized size. However, after last changes we can have different
sizes.

I replaced libyuv::I420Copy with libyuv::I420Scale, but note that it still
does only a copy when the gven input and output sizes are the same:
https://cs.chromium.org/chromium/src/third_party/libyuv/source/scale.cc?rcl=0&l=1400

BUG=630577
TEST=Tested Hangouts call on veyron_jerry. Added RTCVideoEncoderUnittest.

Review-Url: https://codereview.chromium.org/2182183007
Cr-Commit-Position: refs/heads/master@{#409286}

content/content_tests.gypi

@@ -783,6 +783,7 @@
       'renderer/media/audio_track_recorder_unittest.cc',
       'renderer/media/canvas_capture_handler_unittest.cc',
       'renderer/media/gpu/rtc_video_decoder_unittest.cc',
+      'renderer/media/gpu/rtc_video_encoder_unittest.cc',
       'renderer/media/html_video_element_capturer_source_unittest.cc',
       'renderer/media/media_recorder_handler_unittest.cc',
       'renderer/media/media_stream_audio_processor_unittest.cc',

content/renderer/media/gpu/rtc_video_encoder.cc

@@ -585,22 +585,27 @@ void RTCVideoEncoder::Impl::EncodeOneFrame() {
                         media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
-    // Do a strided copy of the input frame to match the input requirements for
-    // the encoder.
-    // TODO(sheu): support zero-copy from WebRTC.  http://crbug.com/269312
-    if (libyuv::I420Copy(next_frame->video_frame_buffer()->DataY(),
-                         next_frame->video_frame_buffer()->StrideY(),
-                         next_frame->video_frame_buffer()->DataU(),
-                         next_frame->video_frame_buffer()->StrideU(),
-                         next_frame->video_frame_buffer()->DataV(),
-                         next_frame->video_frame_buffer()->StrideV(),
-                         frame->data(media::VideoFrame::kYPlane),
-                         frame->stride(media::VideoFrame::kYPlane),
-                         frame->data(media::VideoFrame::kUPlane),
-                         frame->stride(media::VideoFrame::kUPlane),
-                         frame->data(media::VideoFrame::kVPlane),
-                         frame->stride(media::VideoFrame::kVPlane),
-                         next_frame->width(), next_frame->height())) {
+
+    // Do a strided copy and scale (if necessary) the input frame to match
+    // the input requirements for the encoder.
+    // TODO(sheu): Support zero-copy from WebRTC. http://crbug.com/269312
+    // TODO(magjed): Downscale with kFilterBox in an image pyramid instead.
+    if (libyuv::I420Scale(next_frame->video_frame_buffer()->DataY(),
+                          next_frame->video_frame_buffer()->StrideY(),
+                          next_frame->video_frame_buffer()->DataU(),
+                          next_frame->video_frame_buffer()->StrideU(),
+                          next_frame->video_frame_buffer()->DataV(),
+                          next_frame->video_frame_buffer()->StrideV(),
+                          next_frame->width(), next_frame->height(),
+                          frame->visible_data(media::VideoFrame::kYPlane),
+                          frame->stride(media::VideoFrame::kYPlane),
+                          frame->visible_data(media::VideoFrame::kUPlane),
+                          frame->stride(media::VideoFrame::kUPlane),
+                          frame->visible_data(media::VideoFrame::kVPlane),
+                          frame->stride(media::VideoFrame::kVPlane),
+                          frame->visible_rect().width(),
+                          frame->visible_rect().height(),
+                          libyuv::kFilterBox)) {
       LogAndNotifyError(FROM_HERE, "Failed to copy buffer",
                         media::VideoEncodeAccelerator::kPlatformFailureError);
       return;

content/renderer/media/gpu/rtc_video_encoder_unittest.cc

@@ -0,0 +1,180 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stdint.h>
+
+#include "base/bind.h"
+#include "base/memory/ptr_util.h"
+#include "base/single_thread_task_runner.h"
+#include "base/synchronization/waitable_event.h"
+#include "base/threading/thread.h"
+#include "content/renderer/media/gpu/rtc_video_encoder.h"
+#include "media/renderers/mock_gpu_video_accelerator_factories.h"
+#include "media/video/mock_video_encode_accelerator.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "third_party/libyuv/include/libyuv/planar_functions.h"
+#include "third_party/webrtc/modules/video_coding/include/video_codec_interface.h"
+
+using ::testing::_;
+using ::testing::Invoke;
+using ::testing::Return;
+using ::testing::SaveArg;
+using ::testing::Values;
+using ::testing::WithArgs;
+
+namespace content {
+
+namespace {
+
+const int kInputFrameFillY = 12;
+const int kInputFrameFillU = 23;
+const int kInputFrameFillV = 34;
+const unsigned short kInputFrameHeight = 234;
+const unsigned short kInputFrameWidth = 345;
+const unsigned short kStartBitrate = 100;
+
+}  // anonymous namespace
+
+class RTCVideoEncoderTest
+    : public ::testing::TestWithParam<webrtc::VideoCodecType> {
+ public:
+  RTCVideoEncoderTest()
+      : mock_gpu_factories_(
+            new media::MockGpuVideoAcceleratorFactories(nullptr)),
+        encoder_thread_("vea_thread"),
+        idle_waiter_(base::WaitableEvent::ResetPolicy::AUTOMATIC,
+                     base::WaitableEvent::InitialState::NOT_SIGNALED) {}
+
+  void SetUp() override {
+    DVLOG(3) << __FUNCTION__;
+    ASSERT_TRUE(encoder_thread_.Start());
+    mock_vea_ = new media::MockVideoEncodeAccelerator();
+
+    EXPECT_CALL(*mock_gpu_factories_.get(), GetTaskRunner())
+        .WillRepeatedly(Return(encoder_thread_.task_runner()));
+    EXPECT_CALL(*mock_gpu_factories_.get(), DoCreateVideoEncodeAccelerator())
+        .WillRepeatedly(Return(mock_vea_));
+    EXPECT_CALL(*mock_vea_, Initialize(_, _, _, _, _))
+        .WillOnce(Invoke(this, &RTCVideoEncoderTest::Initialize));
+    EXPECT_CALL(*mock_vea_, UseOutputBitstreamBuffer(_)).Times(3);
+    EXPECT_CALL(*mock_vea_, Destroy()).Times(1);
+  }
+
+  void TearDown() override {
+    DVLOG(3) << __FUNCTION__;
+    EXPECT_TRUE(encoder_thread_.IsRunning());
+    RunUntilIdle();
+    rtc_encoder_->Release();
+    RunUntilIdle();
+    encoder_thread_.Stop();
+  }
+
+  void RunUntilIdle() {
+    DVLOG(3) << __FUNCTION__;
+    encoder_thread_.task_runner()->PostTask(
+        FROM_HERE, base::Bind(&base::WaitableEvent::Signal,
+                              base::Unretained(&idle_waiter_)));
+    idle_waiter_.Wait();
+  }
+
+  void CreateEncoder(webrtc::VideoCodecType codec_type) {
+    DVLOG(3) << __FUNCTION__;
+    rtc_encoder_ = base::WrapUnique(
+        new RTCVideoEncoder(codec_type, mock_gpu_factories_.get()));
+  }
+
+  // media::VideoEncodeAccelerator implementation.
+  bool Initialize(media::VideoPixelFormat input_format,
+                  const gfx::Size& input_visible_size,
+                  media::VideoCodecProfile output_profile,
+                  uint32_t initial_bitrate,
+                  media::VideoEncodeAccelerator::Client* client) {
+    DVLOG(3) << __FUNCTION__;
+    client->RequireBitstreamBuffers(0, input_visible_size,
+                                    input_visible_size.GetArea());
+    return true;
+  }
+
+  webrtc::VideoCodec GetDefaultCodec() {
+    webrtc::VideoCodec codec = {};
+    memset(&codec, 0, sizeof(codec));
+    codec.width = kInputFrameWidth;
+    codec.height = kInputFrameHeight;
+    codec.codecType = webrtc::kVideoCodecVP8;
+    codec.startBitrate = kStartBitrate;
+    return codec;
+  }
+
+  void FillFrameBuffer(rtc::scoped_refptr<webrtc::I420Buffer> frame) {
+    CHECK(libyuv::I420Rect(frame->MutableDataY(), frame->StrideY(),
+                           frame->MutableDataU(), frame->StrideU(),
+                           frame->MutableDataV(), frame->StrideV(), 0, 0,
+                           frame->width(), frame->height(), kInputFrameFillY,
+                           kInputFrameFillU, kInputFrameFillV) == 0);
+  }
+
+  void VerifyEncodedFrame(const scoped_refptr<media::VideoFrame>& frame,
+                          bool force_keyframe) {
+    DVLOG(3) << __FUNCTION__;
+    EXPECT_EQ(kInputFrameWidth, frame->visible_rect().width());
+    EXPECT_EQ(kInputFrameHeight, frame->visible_rect().height());
+    EXPECT_EQ(kInputFrameFillY,
+              frame->visible_data(media::VideoFrame::kYPlane)[0]);
+    EXPECT_EQ(kInputFrameFillU,
+              frame->visible_data(media::VideoFrame::kUPlane)[0]);
+    EXPECT_EQ(kInputFrameFillV,
+              frame->visible_data(media::VideoFrame::kVPlane)[0]);
+  }
+
+ protected:
+  media::MockVideoEncodeAccelerator* mock_vea_;
+  std::unique_ptr<RTCVideoEncoder> rtc_encoder_;
+
+ private:
+  std::unique_ptr<media::MockGpuVideoAcceleratorFactories> mock_gpu_factories_;
+  base::Thread encoder_thread_;
+  base::WaitableEvent idle_waiter_;
+};
+
+TEST_P(RTCVideoEncoderTest, CreateAndInitSucceeds) {
+  const webrtc::VideoCodecType codec_type = GetParam();
+  CreateEncoder(codec_type);
+  webrtc::VideoCodec codec = GetDefaultCodec();
+  codec.codecType = codec_type;
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, rtc_encoder_->InitEncode(&codec, 1, 12345));
+}
+
+TEST_F(RTCVideoEncoderTest, EncodeScaledFrame) {
+  const webrtc::VideoCodecType codec_type = webrtc::kVideoCodecVP8;
+  CreateEncoder(codec_type);
+  webrtc::VideoCodec codec = GetDefaultCodec();
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, rtc_encoder_->InitEncode(&codec, 1, 12345));
+
+  EXPECT_CALL(*mock_vea_, Encode(_, _))
+      .Times(2)
+      .WillRepeatedly(Invoke(this, &RTCVideoEncoderTest::VerifyEncodedFrame));
+
+  const rtc::scoped_refptr<webrtc::I420Buffer> buffer =
+      webrtc::I420Buffer::Create(kInputFrameWidth, kInputFrameHeight);
+  FillFrameBuffer(buffer);
+  std::vector<webrtc::FrameType> frame_types;
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            rtc_encoder_->Encode(
+                webrtc::VideoFrame(buffer, 0, 0, webrtc::kVideoRotation_0),
+                nullptr, &frame_types));
+
+  const rtc::scoped_refptr<webrtc::I420Buffer> upscaled_buffer =
+      webrtc::I420Buffer::Create(2 * kInputFrameWidth, 2 * kInputFrameHeight);
+  FillFrameBuffer(upscaled_buffer);
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            rtc_encoder_->Encode(webrtc::VideoFrame(upscaled_buffer, 0, 0,
+                                                    webrtc::kVideoRotation_0),
+                                 nullptr, &frame_types));
+}
+
+INSTANTIATE_TEST_CASE_P(CodecProfiles,
+                        RTCVideoEncoderTest,
+                        Values(webrtc::kVideoCodecVP8,
+                               webrtc::kVideoCodecH264));
+}  // namespace content

media/BUILD.gn

@@ -764,6 +764,8 @@ static_library("test_support") {
     "renderers/mock_gpu_video_accelerator_factories.h",
     "video/mock_video_decode_accelerator.cc",
     "video/mock_video_decode_accelerator.h",
+    "video/mock_video_encode_accelerator.cc",
+    "video/mock_video_encode_accelerator.h",
   ]
   public_deps = [
     ":media",

media/media.gyp

@@ -1600,6 +1600,8 @@
         'renderers/mock_gpu_video_accelerator_factories.h',
         'video/mock_video_decode_accelerator.cc',
         'video/mock_video_decode_accelerator.h',
+        'video/mock_video_encode_accelerator.cc',
+        'video/mock_video_encode_accelerator.h',
       ],
     },
     {

media/renderers/mock_gpu_video_accelerator_factories.cc

@@ -101,6 +101,9 @@ MockGpuVideoAcceleratorFactories::AllocateGpuMemoryBuffer(
 
 std::unique_ptr<base::SharedMemory>
 MockGpuVideoAcceleratorFactories::CreateSharedMemory(size_t size) {
+  std::unique_ptr<base::SharedMemory> shared_memory(new base::SharedMemory);
+  if (shared_memory->CreateAndMapAnonymous(size))
+    return shared_memory;
   return nullptr;
 }
 

media/video/mock_video_encode_accelerator.cc

@@ -0,0 +1,23 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/video/mock_video_encode_accelerator.h"
+
+namespace media {
+
+using ::testing::Invoke;
+
+MockVideoEncodeAccelerator::MockVideoEncodeAccelerator() {
+  // Delete |this| when Destroy() is called.
+  ON_CALL(*this, Destroy())
+      .WillByDefault(Invoke(this, &MockVideoEncodeAccelerator::DeleteThis));
+}
+
+MockVideoEncodeAccelerator::~MockVideoEncodeAccelerator() {}
+
+void MockVideoEncodeAccelerator::DeleteThis() {
+  delete this;
+}
+
+}  // namespace media

media/video/mock_video_encode_accelerator.h

@@ -0,0 +1,43 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_VIDEO_MOCK_VIDEO_ENCODE_ACCELERATOR_H_
+#define MEDIA_VIDEO_MOCK_VIDEO_ENCODE_ACCELERATOR_H_
+
+#include "media/video/video_encode_accelerator.h"
+
+#include "base/macros.h"
+#include "testing/gmock/include/gmock/gmock.h"
+
+namespace media {
+
+class MockVideoEncodeAccelerator : public VideoEncodeAccelerator {
+ public:
+  MockVideoEncodeAccelerator();
+  virtual ~MockVideoEncodeAccelerator();
+
+  MOCK_METHOD0(GetSupportedProfiles,
+               VideoEncodeAccelerator::SupportedProfiles());
+  MOCK_METHOD5(Initialize,
+               bool(VideoPixelFormat input_format,
+                    const gfx::Size& input_visible_size,
+                    VideoCodecProfile output_profile,
+                    uint32_t initial_bitrate,
+                    VideoEncodeAccelerator::Client* client));
+  MOCK_METHOD2(Encode,
+               void(const scoped_refptr<VideoFrame>& frame,
+                    bool force_keyframe));
+  MOCK_METHOD1(UseOutputBitstreamBuffer, void(const BitstreamBuffer& buffer));
+  MOCK_METHOD2(RequestEncodingParametersChange,
+               void(uint32_t bitrate, uint32_t framerate));
+  MOCK_METHOD0(Destroy, void());
+
+ private:
+  void DeleteThis();
+  DISALLOW_COPY_AND_ASSIGN(MockVideoEncodeAccelerator);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_VIDEO_MOCK_VIDEO_ENCODE_ACCELERATOR_H_