Skip to content

Commit

Permalink
Tab Capture: Backing store readbacks to YV12 VideoFrames.
Browse files Browse the repository at this point in the history 
Goal here is performance; in tab capture there are
significant benefits to doing the readback as YV12:
it's less data to copy back, and it's cheap to do
on the GPU.

VideoFrame is used because it's the most capable YV12
container.

[render_widget_host.h] 
Add a new flavor of CopyFromBackingStore, called 
CopyFromBackingStoreToVideoFrame. The semantics are 
slightly different from the RGBA copy in ways that are 
video-appropriate: aspect ratio is preserved via 
letterboxing, meaning the output is returned at the target's
allocated size, no matter what (whereas CopyFromBackingStore
returns whatever it wants, treating |dst_size| as a hint).

Callers may only call CopyFromBackingStoreToVideoFrame
after checking the result of CanCopyToVideoFrame(). 
Support is only on Windows now, and only while accelerated
compositing is active. But the interface defined
here should make it possible to implement VideoFrame
readbacks on other platforms without having to touch
the callers.

[video_capture_controller.h]
Amend the interface to allow a VideoFrame to
be passed in, as an alternative to void*.
The buffer-based interface was inadequate for
our needs since stride was not handled. Using
VideoFrame allows strides to be accomodated,
and paves the way for the interface to
pre-reserve the VideoFrames.

[web_contents_video_capture_device.cc]
Start using CopyFromBackingStoreToVideoFrame. Handling
both copy flavors requires a bifurcation of much
of the processing pipeline. When dealing with a VideoFrame,
the Render stage can is bypassed completely.

[accelerated_surface_win.h]
Implementation of VideoFrame YV12 readback, using
AcceleratedSurfaceTransformer's d3d-accelerated routines.

BUG=161537

Review URL: https://chromiumcodereview.appspot.com/12090109

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@181785 0039d316-1c4b-4281-b951-d872f2087c98
  • Loading branch information
nick@chromium.org committed Feb 11, 2013
1 parent a6b73c6 commit c02cccc
Show file tree
Hide file tree
Showing 32 changed files with 922 additions and 235 deletions.
134 changes: 117 additions & 17 deletions content/browser/renderer_host/media/video_capture_controller.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
#include "content/browser/renderer_host/media/media_stream_manager.h"
#include "content/browser/renderer_host/media/video_capture_manager.h"
#include "content/public/browser/browser_thread.h"
#include "media/base/video_frame.h"
#include "media/base/video_util.h"
#include "media/base/yuv_convert.h"

#if !defined(OS_IOS) && !defined(OS_ANDROID)
Expand Down Expand Up @@ -245,13 +247,10 @@ void VideoCaptureController::ReturnBuffer(
}
}

///////////////////////////////////////////////////////////////////////////////
// Implements VideoCaptureDevice::EventHandler.
// OnIncomingCapturedFrame is called the thread running the capture device.
// I.e.- DirectShow thread on windows and v4l2_thread on Linux.
void VideoCaptureController::OnIncomingCapturedFrame(const uint8* data,
int length,
base::Time timestamp) {
bool VideoCaptureController::ReserveSharedMemory(int* buffer_id_out,
uint8** yplane,
uint8** uplane,
uint8** vplane) {
int buffer_id = 0;
base::SharedMemory* dib = NULL;
{
Expand All @@ -269,25 +268,39 @@ void VideoCaptureController::OnIncomingCapturedFrame(const uint8* data,
}
}

if (!dib) {
return;
}
if (!dib)
return false;

*buffer_id_out = buffer_id;
CHECK_GE(dib->created_size(),
static_cast<size_t>(frame_info_.width * frame_info_.height * 3) / 2);
uint8* target = static_cast<uint8*>(dib->memory());
CHECK(dib->created_size() >= static_cast<size_t> (frame_info_.width *
frame_info_.height * 3) /
2);
uint8* yplane = target;
uint8* uplane = target + frame_info_.width * frame_info_.height;
uint8* vplane = uplane + (frame_info_.width * frame_info_.height) / 4;
*yplane = target;
*uplane = *yplane + frame_info_.width * frame_info_.height;
*vplane = *uplane + (frame_info_.width * frame_info_.height) / 4;
return true;
}

// Implements VideoCaptureDevice::EventHandler.
// OnIncomingCapturedFrame is called the thread running the capture device.
// I.e.- DirectShow thread on windows and v4l2_thread on Linux.
void VideoCaptureController::OnIncomingCapturedFrame(const uint8* data,
int length,
base::Time timestamp) {
int buffer_id = 0;
uint8* yplane = NULL;
uint8* uplane = NULL;
uint8* vplane = NULL;
if (!ReserveSharedMemory(&buffer_id, &yplane, &uplane, &vplane))
return;

// Do color conversion from the camera format to I420.
switch (frame_info_.color) {
case media::VideoCaptureCapability::kColorUnknown: // Color format not set.
break;
case media::VideoCaptureCapability::kI420: {
DCHECK(!chopped_width_ && !chopped_height_);
memcpy(target, data, (frame_info_.width * frame_info_.height * 3) / 2);
memcpy(yplane, data, (frame_info_.width * frame_info_.height * 3) / 2);
break;
}
case media::VideoCaptureCapability::kYV12: {
Expand Down Expand Up @@ -365,6 +378,93 @@ void VideoCaptureController::OnIncomingCapturedFrame(const uint8* data,
this, buffer_id, timestamp));
}

// OnIncomingCapturedVideoFrame is called the thread running the capture device.
void VideoCaptureController::OnIncomingCapturedVideoFrame(
media::VideoFrame* frame,
base::Time timestamp) {
// Validate the inputs.
gfx::Size target_size = gfx::Size(frame_info_.width, frame_info_.height);
if (frame->coded_size() != target_size)
return; // Only exact copies are supported.
if (!(frame->format() == media::VideoFrame::I420 ||
frame->format() == media::VideoFrame::YV12 ||
frame->format() == media::VideoFrame::RGB32)) {
NOTREACHED() << "Unsupported format passed to OnIncomingCapturedVideoFrame";
return;
}

// Carve out a shared memory buffer.
int buffer_id = 0;
uint8* yplane = NULL;
uint8* uplane = NULL;
uint8* vplane = NULL;
if (!ReserveSharedMemory(&buffer_id, &yplane, &uplane, &vplane))
return;

scoped_refptr<media::VideoFrame> target_as_frame(
media::VideoFrame::WrapExternalYuvData(
media::VideoFrame::YV12, // Actually I420, but it's equivalent here.
target_size, gfx::Rect(target_size), target_size,
frame_info_.width, // y stride
frame_info_.width / 2, // v stride
frame_info_.width / 2, // u stride
yplane,
uplane,
vplane,
base::TimeDelta(),
base::Bind(&base::DoNothing)));

const int kYPlane = media::VideoFrame::kYPlane;
const int kUPlane = media::VideoFrame::kUPlane;
const int kVPlane = media::VideoFrame::kVPlane;
const int kRGBPlane = media::VideoFrame::kRGBPlane;

// Do color conversion from the camera format to I420.
switch (frame->format()) {
case media::VideoFrame::INVALID:
case media::VideoFrame::YV16:
case media::VideoFrame::EMPTY:
case media::VideoFrame::NATIVE_TEXTURE: {
NOTREACHED();
break;
}
case media::VideoFrame::I420:
case media::VideoFrame::YV12: {
DCHECK(!chopped_width_ && !chopped_height_);
media::CopyYPlane(frame->data(kYPlane),
frame->stride(kYPlane),
frame->rows(kYPlane),
target_as_frame);
media::CopyUPlane(frame->data(kUPlane),
frame->stride(kUPlane),
frame->rows(kUPlane),
target_as_frame);
media::CopyVPlane(frame->data(kVPlane),
frame->stride(kVPlane),
frame->rows(kVPlane),
target_as_frame);
break;
}
case media::VideoFrame::RGB32: {
media::ConvertRGB32ToYUV(frame->data(kRGBPlane),
target_as_frame->data(kYPlane),
target_as_frame->data(kUPlane),
target_as_frame->data(kVPlane),
target_size.width(),
target_size.height(),
frame->stride(kRGBPlane),
target_as_frame->stride(kYPlane),
target_as_frame->stride(kUPlane));
break;
}
}

BrowserThread::PostTask(BrowserThread::IO,
FROM_HERE,
base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
this, buffer_id, timestamp));
}

void VideoCaptureController::OnError() {
BrowserThread::PostTask(BrowserThread::IO,
FROM_HERE,
Expand Down
14 changes: 13 additions & 1 deletion content/browser/renderer_host/media/video_capture_controller.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ class CONTENT_EXPORT VideoCaptureController
virtual void OnIncomingCapturedFrame(const uint8* data,
int length,
base::Time timestamp) OVERRIDE;
virtual void OnIncomingCapturedVideoFrame(media::VideoFrame* frame,
base::Time timestamp) OVERRIDE;
virtual void OnError() OVERRIDE;
virtual void OnFrameInfo(
const media::VideoCaptureCapability& info) OVERRIDE;
Expand All @@ -93,22 +95,32 @@ class CONTENT_EXPORT VideoCaptureController
// Send frame info and init buffers to |client|.
void SendFrameInfoAndBuffers(ControllerClient* client, int buffer_size);

// Helpers.
// Find a client of |id| and |handler| in |clients|.
ControllerClient* FindClient(
const VideoCaptureControllerID& id,
VideoCaptureControllerEventHandler* handler,
const ControllerClients& clients);

// Find a client of |session_id| in |clients|.
ControllerClient* FindClient(
int session_id,
const ControllerClients& clients);

// Decide what to do after kStopping state. Dependent on events, controller
// can stay in kStopping state, or go to kStopped, or restart capture.
void PostStopping();

// Check if any DIB is used by client.
bool ClientHasDIB();

// DIB reservation. Locate an available DIB object, reserve it, and provide
// the caller the reserved DIB's buffer_id as well as pointers to its color
// planes. Returns true if successful.
bool ReserveSharedMemory(int* buffer_id_out,
uint8** yplane,
uint8** uplane,
uint8** vplane);

// Lock to protect free_dibs_ and owned_dibs_.
base::Lock lock_;

Expand Down
5 changes: 4 additions & 1 deletion content/browser/renderer_host/media/video_capture_manager_unittest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,11 @@ class MockFrameObserver : public media::VideoCaptureDevice::EventHandler {
public:
virtual void OnError() OVERRIDE {}
void OnFrameInfo(const media::VideoCaptureCapability& info) {}
virtual void OnIncomingCapturedFrame(const uint8* data, int length,
virtual void OnIncomingCapturedFrame(const uint8* data,
int length,
base::Time timestamp) OVERRIDE {}
virtual void OnIncomingCapturedVideoFrame(media::VideoFrame* frame,
base::Time timestamp) OVERRIDE {}
};

// Test class
Expand Down
Loading

0 comments on commit c02cccc

Please sign in to comment.