FileVideoCaptureDevice: support Mac NV12 GMB.

This capturer didn't previously support GMB capturing. This patch
changes this by supplying a new GpuMemoryBufferTrackerMac
supporting IOSurface generation.

With capture & display in HD at 60 fps, we routinely
run out of buffers. This patch increases the max number of
outstanding buffers for macOS.

Because the FileVideoCaptureDevice writes to GMBs using mode
gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE which
previously implied kIOSurfaceLockReadOnly locking mode, a new
buffer usage constant gfx::BufferUsage::SCANOUT_VEA_CPU_READ is
introduced which is used for buffers shipped from video capture. This
fixes the related TODO in gpu_memory_buffer_impl_io_surface.cc by
mapping SCANOUT_VEA_CPU_READ to kIOSurfaceLockReadOnly.

Bug: 1134165, 1130101
Change-Id: I4d2d1624214659bd232a6fe064dc1de9362c5958
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2487341
Commit-Queue: Markus Handell <handellm@google.com>
Reviewed-by: Hirokazu Honda <hiroh@chromium.org>
Reviewed-by: ccameron <ccameron@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Reviewed-by: Robert Sesek <rsesek@chromium.org>
Reviewed-by: Daniele Castagna <dcastagna@chromium.org>
Cr-Commit-Position: refs/heads/master@{#820611}

gpu/ipc/common/gpu_memory_buffer_impl_io_surface.cc

@@ -18,19 +18,22 @@ namespace {
 
 // The maximum number of times to dump before throttling (to avoid sending
 // thousands of crash dumps).
+
 const int kMaxCrashDumps = 10;
 
 uint32_t LockFlags(gfx::BufferUsage usage) {
   switch (usage) {
     case gfx::BufferUsage::GPU_READ_CPU_READ_WRITE:
-      return kIOSurfaceLockAvoidSync;
     case gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE:
+      // The AvoidSync call has the property that it will not preserve the
+      // previous contents of the buffer if those contents were written by a
+      // GPU.
+      return kIOSurfaceLockAvoidSync;
+    case gfx::BufferUsage::SCANOUT_VEA_CPU_READ:
       // This constant is used for buffers used by video capture. On macOS,
       // these buffers are only ever written to in the capture process,
-      // directly as IOSurfaces. Once they are sent to other processes, no CPU
-      // writes are performed.
-      // TODO(https://crbug.com/1130101): A more accurate usage constant may be
-      // appropriate.
+      // directly as IOSurfaces.
+      // Once they are sent to other processes, no CPU writes are performed.
       return kIOSurfaceLockReadOnly;
     case gfx::BufferUsage::GPU_READ:
     case gfx::BufferUsage::SCANOUT:

gpu/ipc/common/gpu_memory_buffer_impl_shared_memory.cc

@@ -163,6 +163,7 @@ bool GpuMemoryBufferImplSharedMemory::IsUsageSupported(gfx::BufferUsage usage) {
     case gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE:
     case gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE:
     case gfx::BufferUsage::SCANOUT_VDA_WRITE:
+    case gfx::BufferUsage::SCANOUT_VEA_CPU_READ:
     case gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE:
       return false;
   }

gpu/ipc/common/gpu_memory_buffer_impl_test_template.h

@@ -99,6 +99,7 @@ TYPED_TEST_P(GpuMemoryBufferImplTest, CreateFromHandle) {
         gfx::BufferUsage::SCANOUT_CPU_READ_WRITE,
         gfx::BufferUsage::SCANOUT_VDA_WRITE,
         gfx::BufferUsage::GPU_READ_CPU_READ_WRITE,
+        gfx::BufferUsage::SCANOUT_VEA_CPU_READ,
         gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE,
     };
     for (auto usage : usages) {
@@ -140,6 +141,7 @@ TYPED_TEST_P(GpuMemoryBufferImplTest, CreateFromHandleSmallBuffer) {
         gfx::BufferUsage::SCANOUT_CPU_READ_WRITE,
         gfx::BufferUsage::SCANOUT_VDA_WRITE,
         gfx::BufferUsage::GPU_READ_CPU_READ_WRITE,
+        gfx::BufferUsage::SCANOUT_VEA_CPU_READ,
         gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE,
     };
     for (auto usage : usages) {
@@ -317,6 +319,7 @@ TYPED_TEST_P(GpuMemoryBufferImplTest, SerializeAndDeserialize) {
         gfx::BufferUsage::SCANOUT_CPU_READ_WRITE,
         gfx::BufferUsage::SCANOUT_VDA_WRITE,
         gfx::BufferUsage::GPU_READ_CPU_READ_WRITE,
+        gfx::BufferUsage::SCANOUT_VEA_CPU_READ,
         gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE,
     };
     for (auto usage : usages) {

gpu/ipc/common/gpu_memory_buffer_support.cc

@@ -96,6 +96,7 @@ bool GpuMemoryBufferSupport::IsNativeGpuMemoryBufferConfigurationSupported(
     case gfx::BufferUsage::SCANOUT_VDA_WRITE:
     case gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE:
     case gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE:
+    case gfx::BufferUsage::SCANOUT_VEA_CPU_READ:
     case gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE:
       return false;
   }
@@ -116,6 +117,7 @@ bool GpuMemoryBufferSupport::IsNativeGpuMemoryBufferConfigurationSupported(
     case gfx::BufferUsage::SCANOUT_VDA_WRITE:
     case gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE:
     case gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE:
+    case gfx::BufferUsage::SCANOUT_VEA_CPU_READ:
     case gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE:
       return false;
   }
@@ -145,6 +147,7 @@ bool GpuMemoryBufferSupport::IsNativeGpuMemoryBufferConfigurationSupported(
     case gfx::BufferUsage::SCANOUT_VDA_WRITE:
     case gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE:
     case gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE:
+    case gfx::BufferUsage::SCANOUT_VEA_CPU_READ:
     case gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE:
       return false;
   }

gpu/ipc/host/gpu_memory_buffer_support.cc

@@ -40,6 +40,7 @@ GpuMemoryBufferConfigurationSet GetNativeGpuMemoryBufferConfigurations(
       gfx::BufferUsage::SCANOUT_CPU_READ_WRITE,
       gfx::BufferUsage::SCANOUT_VDA_WRITE,
       gfx::BufferUsage::GPU_READ_CPU_READ_WRITE,
+      gfx::BufferUsage::SCANOUT_VEA_CPU_READ,
       gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE,
   };
 

gpu/ipc/service/gpu_memory_buffer_factory_test_template.h

@@ -69,6 +69,7 @@ TYPED_TEST_P(GpuMemoryBufferFactoryTest, CreateGpuMemoryBuffer) {
         gfx::BufferUsage::SCANOUT_CPU_READ_WRITE,
         gfx::BufferUsage::SCANOUT_VDA_WRITE,
         gfx::BufferUsage::GPU_READ_CPU_READ_WRITE,
+        gfx::BufferUsage::SCANOUT_VEA_CPU_READ,
         gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE,
     };
     for (auto usage : usages) {

media/capture/BUILD.gn

@@ -166,6 +166,8 @@ component("capture_lib") {
 
   if (is_mac) {
     sources += [
+      "video/mac/gpu_memory_buffer_tracker_mac.cc",
+      "video/mac/gpu_memory_buffer_tracker_mac.h",
       "video/mac/pixel_buffer_pool_mac.cc",
       "video/mac/pixel_buffer_pool_mac.h",
       "video/mac/pixel_buffer_transferer_mac.cc",

media/capture/video/file_video_capture_device.cc

@@ -9,6 +9,7 @@
 
 #include "base/bind.h"
 #include "base/location.h"
+#include "base/logging.h"
 #include "base/macros.h"
 #include "base/single_thread_task_runner.h"
 #include "base/strings/string_number_conversions.h"
@@ -453,6 +454,7 @@ void FileVideoCaptureDevice::OnCaptureTask() {
         VideoCaptureDevice::Client::ReserveResult::kSucceeded) {
       client_->OnFrameDropped(
           ConvertReservationFailureToFrameDropReason(reserve_result));
+      DVLOG(2) << __func__ << " frame was dropped.";
       return;
     }
     ScopedNV12GpuMemoryBufferMapping scoped_mapping(std::move(gmb));
@@ -469,7 +471,6 @@ void FileVideoCaptureDevice::OnCaptureTask() {
         src_v_plane, buffer_size.width() / 2, scoped_mapping.y_plane(),
         scoped_mapping.y_stride(), scoped_mapping.uv_plane(),
         scoped_mapping.uv_stride(), buffer_size.width(), buffer_size.height());
-
     VideoCaptureFormat modified_format = capture_format_;
     // When GpuMemoryBuffer is used, the frame data is opaque to the CPU for
     // most of the time.  Currently the only supported underlying format is

media/capture/video/mac/gpu_memory_buffer_tracker_mac.cc

@@ -0,0 +1,76 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/capture/video/mac/gpu_memory_buffer_tracker_mac.h"
+
+#include "base/logging.h"
+
+namespace media {
+
+GpuMemoryBufferTrackerMac::GpuMemoryBufferTrackerMac() {}
+
+GpuMemoryBufferTrackerMac::~GpuMemoryBufferTrackerMac() {}
+
+bool GpuMemoryBufferTrackerMac::Init(const gfx::Size& dimensions,
+                                     VideoPixelFormat format,
+                                     const mojom::PlaneStridesPtr& strides) {
+  if (format != PIXEL_FORMAT_NV12) {
+    NOTREACHED() << "Unsupported VideoPixelFormat "
+                 << VideoPixelFormatToString(format);
+    return false;
+  }
+  if (IOSurfaceRef io_surface =
+          CreateIOSurface(dimensions, gfx::BufferFormat::YUV_420_BIPLANAR,
+                          /*should_clear=*/false)) {
+    io_surface_.reset(io_surface, base::scoped_policy::ASSUME);
+    DVLOG(2) << __func__ << " id " << IOSurfaceGetID(io_surface_);
+    return true;
+  } else {
+    LOG(ERROR) << "Unable to create IOSurface!";
+    return false;
+  }
+}
+
+bool GpuMemoryBufferTrackerMac::IsReusableForFormat(
+    const gfx::Size& dimensions,
+    VideoPixelFormat format,
+    const mojom::PlaneStridesPtr& strides) {
+  gfx::Size surface_size(IOSurfaceGetWidth(io_surface_),
+                         IOSurfaceGetHeight(io_surface_));
+  return format == PIXEL_FORMAT_NV12 && dimensions == surface_size;
+}
+
+uint32_t GpuMemoryBufferTrackerMac::GetMemorySizeInBytes() {
+  return IOSurfaceGetAllocSize(io_surface_);
+}
+
+std::unique_ptr<VideoCaptureBufferHandle>
+GpuMemoryBufferTrackerMac::GetMemoryMappedAccess() {
+  NOTREACHED() << "Unsupported operation";
+  return std::make_unique<NullHandle>();
+}
+
+base::UnsafeSharedMemoryRegion
+GpuMemoryBufferTrackerMac::DuplicateAsUnsafeRegion() {
+  NOTREACHED() << "Unsupported operation";
+  return base::UnsafeSharedMemoryRegion();
+}
+
+mojo::ScopedSharedBufferHandle
+GpuMemoryBufferTrackerMac::DuplicateAsMojoBuffer() {
+  NOTREACHED() << "Unsupported operation";
+  return mojo::ScopedSharedBufferHandle();
+}
+
+gfx::GpuMemoryBufferHandle
+GpuMemoryBufferTrackerMac::GetGpuMemoryBufferHandle() {
+  DVLOG(2) << __func__ << " id " << IOSurfaceGetID(io_surface_);
+  gfx::GpuMemoryBufferHandle gmb_handle;
+  gmb_handle.type = gfx::GpuMemoryBufferType::IO_SURFACE_BUFFER;
+  gmb_handle.id.id = -1;
+  gmb_handle.io_surface = io_surface_;
+  return gmb_handle;
+}
+
+}  // namespace media

media/capture/video/mac/gpu_memory_buffer_tracker_mac.h

@@ -0,0 +1,41 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_CAPTURE_VIDEO_MAC_GPU_MEMORY_BUFFER_TRACKER_MAC_H_
+#define MEDIA_CAPTURE_VIDEO_MAC_GPU_MEMORY_BUFFER_TRACKER_MAC_H_
+
+#include "media/capture/video/video_capture_buffer_tracker.h"
+#include "ui/gfx/geometry/size.h"
+#include "ui/gfx/mac/io_surface.h"
+
+namespace media {
+
+class CAPTURE_EXPORT GpuMemoryBufferTrackerMac final
+    : public VideoCaptureBufferTracker {
+ public:
+  GpuMemoryBufferTrackerMac();
+  ~GpuMemoryBufferTrackerMac() override;
+
+  // VideoCaptureBufferTracker
+  bool Init(const gfx::Size& dimensions,
+            VideoPixelFormat format,
+            const mojom::PlaneStridesPtr& strides) override;
+  bool IsReusableForFormat(const gfx::Size& dimensions,
+                           VideoPixelFormat format,
+                           const mojom::PlaneStridesPtr& strides) override;
+  uint32_t GetMemorySizeInBytes() override;
+  std::unique_ptr<VideoCaptureBufferHandle> GetMemoryMappedAccess() override;
+  base::UnsafeSharedMemoryRegion DuplicateAsUnsafeRegion() override;
+  mojo::ScopedSharedBufferHandle DuplicateAsMojoBuffer() override;
+  gfx::GpuMemoryBufferHandle GetGpuMemoryBufferHandle() override;
+
+ private:
+  base::ScopedCFTypeRef<IOSurfaceRef> io_surface_;
+
+  DISALLOW_COPY_AND_ASSIGN(GpuMemoryBufferTrackerMac);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_CAPTURE_VIDEO_MAC_GPU_MEMORY_BUFFER_TRACKER_MAC_H_

media/capture/video/video_capture_buffer_pool_impl.cc

@@ -242,6 +242,8 @@ VideoCaptureBufferPoolImpl::ReserveForProducerInternal(
     if (tracker_to_drop == trackers_.end()) {
       // We're out of space, and can't find an unused tracker to reallocate.
       *buffer_id = kInvalidId;
+      DLOG(ERROR) << __func__
+                  << " max buffer count exceeded count_ = " << count_;
       return VideoCaptureDevice::Client::ReserveResult::kMaxBufferCountExceeded;
     }
     *buffer_id_to_drop = tracker_to_drop->first;

media/capture/video/video_capture_buffer_tracker_factory_impl.cc

@@ -7,12 +7,17 @@
 #include <memory>
 #include "build/chromeos_buildflags.h"
 
+#include "build/build_config.h"
 #include "media/capture/video/shared_memory_buffer_tracker.h"
 
 #if BUILDFLAG(IS_ASH)
 #include "media/capture/video/chromeos/gpu_memory_buffer_tracker.h"
 #endif
 
+#if defined(OS_MAC)
+#include "media/capture/video/mac/gpu_memory_buffer_tracker_mac.h"
+#endif
+
 namespace media {
 
 std::unique_ptr<VideoCaptureBufferTracker>
@@ -22,6 +27,8 @@ VideoCaptureBufferTrackerFactoryImpl::CreateTracker(
     case VideoCaptureBufferType::kGpuMemoryBuffer:
 #if BUILDFLAG(IS_ASH)
       return std::make_unique<GpuMemoryBufferTracker>();
+#elif defined(OS_MAC)
+      return std::make_unique<GpuMemoryBufferTrackerMac>();
 #else
       return nullptr;
 #endif

media/capture/video/video_capture_device_client.cc

@@ -558,8 +558,10 @@ VideoCaptureDeviceClient::ReserveOutputBuffer(const gfx::Size& frame_size,
       receiver_->OnBufferRetired(buffer_id_to_drop);
     }
   }
-  if (reservation_result_code != ReserveResult::kSucceeded)
+  if (reservation_result_code != ReserveResult::kSucceeded) {
+    DVLOG(2) << __func__ << " reservation failed";
     return reservation_result_code;
+  }
 
   DCHECK_NE(VideoCaptureBufferPool::kInvalidId, buffer_id);
 

services/video_capture/device_media_to_mojo_adapter.cc

@@ -7,6 +7,7 @@
 #include "base/bind.h"
 #include "base/check.h"
 #include "base/command_line.h"
+#include "build/build_config.h"
 #include "build/chromeos_buildflags.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/capture/capture_switches.h"
@@ -177,7 +178,11 @@ int DeviceMediaToMojoAdapter::max_buffer_pool_buffer_count() {
   // those frames get dropped.
   static int kMaxBufferCount = 3;
 
-#if BUILDFLAG(IS_ASH)
+#if defined(OS_MAC)
+  // On macOS, we allow a few more buffers as it's routinely observed that it
+  // runs out of three when just displaying 60 FPS media in a video element.
+  kMaxBufferCount = 10;
+#elif BUILDFLAG(IS_ASH)
   // On Chrome OS with MIPI cameras running on HAL v3, there can be three
   // concurrent streams of camera pipeline depth ~6. We allow at most 30 buffers
   // here to take into account the delay caused by the consumer (e.g. display or

third_party/blink/renderer/platform/video_capture/video_capture_impl.cc

@@ -685,8 +685,7 @@ void VideoCaptureImpl::OnBufferReady(
             gpu_memory_buffer_support_->CreateGpuMemoryBufferImplFromHandle(
                 buffer_context->TakeGpuMemoryBufferHandle(),
                 gfx::Size(info->coded_size), gfx_format,
-                gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE,
-                base::DoNothing());
+                gfx::BufferUsage::SCANOUT_VEA_CPU_READ, base::DoNothing());
         buffer_context->SetGpuMemoryBuffer(std::move(gmb));
       }
       CHECK(buffer_context->GetGpuMemoryBuffer());
@@ -697,8 +696,7 @@ void VideoCaptureImpl::OnBufferReady(
               buffer_context->GetGpuMemoryBuffer()->CloneHandle(),
               buffer_context->GetGpuMemoryBuffer()->GetSize(),
               buffer_context->GetGpuMemoryBuffer()->GetFormat(),
-              gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE,
-              base::DoNothing());
+              gfx::BufferUsage::SCANOUT_VEA_CPU_READ, base::DoNothing());
 
       media_task_runner_->PostTask(
           FROM_HERE,

ui/gfx/buffer_types.h

@@ -48,6 +48,7 @@ enum class BufferUsage {
   SCANOUT_CPU_READ_WRITE,
   SCANOUT_VDA_WRITE,
   GPU_READ_CPU_READ_WRITE,
+  SCANOUT_VEA_CPU_READ,
   SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE,
 
   LAST = SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE

ui/gfx/buffer_usage_util.cc

@@ -22,6 +22,8 @@ const char* BufferUsageToString(BufferUsage usage) {
       return "SCANOUT_VDA_WRITE";
     case BufferUsage::GPU_READ_CPU_READ_WRITE:
       return "GPU_READ_CPU_READ_WRITE";
+    case BufferUsage::SCANOUT_VEA_CPU_READ:
+      return "SCANOUT_VEA_CPU_READ";
     case BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE:
       return "SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE";
   }

ui/gfx/linux/client_native_pixmap_dmabuf.cc

@@ -191,6 +191,7 @@ bool ClientNativePixmapDmaBuf::IsConfigurationSupported(
     case gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE:
       // R_8 is used as the underlying pixel format for BLOB buffers.
       return format == gfx::BufferFormat::R_8;
+    case gfx::BufferUsage::SCANOUT_VEA_CPU_READ:
     case gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE:
       return format == gfx::BufferFormat::YVU_420 ||
              format == gfx::BufferFormat::YUV_420_BIPLANAR;

ui/gfx/linux/client_native_pixmap_factory_dmabuf.cc

@@ -59,6 +59,7 @@ class ClientNativePixmapFactoryDmabuf : public ClientNativePixmapFactory {
       case gfx::BufferUsage::GPU_READ_CPU_READ_WRITE:
       case gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE:
       case gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE:
+      case gfx::BufferUsage::SCANOUT_VEA_CPU_READ:
       case gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE:
         return ClientNativePixmapDmaBuf::ImportFromDmabuf(std::move(handle),
                                                           size, format);

ui/gfx/linux/gbm_util.cc

@@ -27,6 +27,9 @@ uint32_t BufferUsageToGbmFlags(gfx::BufferUsage usage) {
              GBM_BO_USE_HW_VIDEO_DECODER;
     case gfx::BufferUsage::GPU_READ_CPU_READ_WRITE:
       return GBM_BO_USE_LINEAR | GBM_BO_USE_TEXTURING;
+    case gfx::BufferUsage::SCANOUT_VEA_CPU_READ:
+      return GBM_BO_USE_LINEAR | GBM_BO_USE_SCANOUT | GBM_BO_USE_TEXTURING |
+             GBM_BO_USE_HW_VIDEO_ENCODER;
     case gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE:
       return GBM_BO_USE_LINEAR | GBM_BO_USE_CAMERA_WRITE | GBM_BO_USE_SCANOUT |
              GBM_BO_USE_TEXTURING | GBM_BO_USE_HW_VIDEO_ENCODER;

ui/gfx/mojom/buffer_types.mojom

@@ -34,6 +34,7 @@ enum BufferUsage {
   SCANOUT_CPU_READ_WRITE,
   SCANOUT_VDA_WRITE,
   GPU_READ_CPU_READ_WRITE,
+  SCANOUT_VEA_CPU_READ,
   SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE,
 };