Vulkan backed Skia representation of the SharedImageVideo.

1. Implement vulkan representation of the SharedImageVideo. 2. Skia promise image requires sampler conversion information ycbcr in the vulkan mode. This information is needed during the promise image creation time which happens before BeginReadAccess. This is done by passing this info to promise image via VideoFrame->StreamVideoDrawQuad->ResourceMetadata. 3. Add ycbcr info to the video frame. Update required mojom and struct traits. 4. Add ycbcr info to the StreamVideoDrawQuad and update required mojom and struct traits. Bug: 900963 Change-Id: Ie4537604efe446492377d3615ed7c3cfde8c2130 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1582774 Commit-Queue: vikas soni <vikassoni@chromium.org> Reviewed-by: Daniel Cheng <dcheng@chromium.org> Reviewed-by: Frank Liberato <liberato@chromium.org> Reviewed-by: Eric Karl <ericrk@chromium.org> Cr-Commit-Position: refs/heads/master@{#664802}
qweri0p · May 30, 2019 · 85bf8ed · 85bf8ed
1 parent ea8bab4
commit 85bf8ed
Show file tree

Hide file tree

Showing 28 changed files with 466 additions and 60 deletions.
diff --git a/components/viz/common/BUILD.gn b/components/viz/common/BUILD.gn
@@ -250,6 +250,7 @@ viz_component("common") {
     "//gpu/command_buffer/client:gles2_interface",
     "//gpu/command_buffer/client:raster",
     "//gpu/command_buffer/client:raster_interface",
+    "//gpu/ipc/common:common",
     "//gpu/vulkan:buildflags",
     "//mojo/public/cpp/base",
     "//mojo/public/cpp/system",

diff --git a/components/viz/common/quads/DEPS b/components/viz/common/quads/DEPS
@@ -10,6 +10,7 @@ include_rules = [
   "+third_party/skia",
   "+ui/gl/dc_renderer_layer_params.h",
   "+ui/latency",
+  "+gpu/ipc/common/vulkan_ycbcr_info.h",
 ]
 
 specific_include_rules = {

diff --git a/components/viz/common/quads/stream_video_draw_quad.cc b/components/viz/common/quads/stream_video_draw_quad.cc
@@ -12,22 +12,28 @@
 namespace viz {
 
 StreamVideoDrawQuad::StreamVideoDrawQuad() = default;
+StreamVideoDrawQuad::~StreamVideoDrawQuad() = default;
+StreamVideoDrawQuad::StreamVideoDrawQuad(const StreamVideoDrawQuad& quad) =
+    default;
 
-void StreamVideoDrawQuad::SetNew(const SharedQuadState* shared_quad_state,
-                                 const gfx::Rect& rect,
-                                 const gfx::Rect& visible_rect,
-                                 bool needs_blending,
-                                 unsigned resource_id,
-                                 gfx::Size resource_size_in_pixels,
-                                 const gfx::PointF& uv_top_left,
-                                 const gfx::PointF& uv_bottom_right) {
+void StreamVideoDrawQuad::SetNew(
+    const SharedQuadState* shared_quad_state,
+    const gfx::Rect& rect,
+    const gfx::Rect& visible_rect,
+    bool needs_blending,
+    unsigned resource_id,
+    gfx::Size resource_size_in_pixels,
+    const gfx::PointF& uv_top_left,
+    const gfx::PointF& uv_bottom_right,
+    const base::Optional<gpu::VulkanYCbCrInfo>& ycbcr_info) {
   DrawQuad::SetAll(shared_quad_state, DrawQuad::Material::kStreamVideoContent,
                    rect, visible_rect, needs_blending);
   resources.ids[kResourceIdIndex] = resource_id;
   overlay_resources.size_in_pixels[kResourceIdIndex] = resource_size_in_pixels;
   resources.count = 1;
   this->uv_top_left = uv_top_left;
   this->uv_bottom_right = uv_bottom_right;
+  this->ycbcr_info = ycbcr_info;
 }
 
 void StreamVideoDrawQuad::SetAll(const SharedQuadState* shared_quad_state,

diff --git a/components/viz/common/quads/stream_video_draw_quad.h b/components/viz/common/quads/stream_video_draw_quad.h
@@ -11,6 +11,7 @@
 
 #include "components/viz/common/quads/draw_quad.h"
 #include "components/viz/common/viz_common_export.h"
+#include "gpu/ipc/common/vulkan_ycbcr_info.h"
 #include "ui/gfx/geometry/point_f.h"
 
 namespace viz {
@@ -20,15 +21,19 @@ class VIZ_COMMON_EXPORT StreamVideoDrawQuad : public DrawQuad {
   static const size_t kResourceIdIndex = 0;
 
   StreamVideoDrawQuad();
-
-  void SetNew(const SharedQuadState* shared_quad_state,
-              const gfx::Rect& rect,
-              const gfx::Rect& visible_rect,
-              bool needs_blending,
-              unsigned resource_id,
-              gfx::Size resource_size_in_pixels,
-              const gfx::PointF& uv_top_left,
-              const gfx::PointF& uv_bottom_right);
+  ~StreamVideoDrawQuad() override;
+  StreamVideoDrawQuad(const StreamVideoDrawQuad& quad);
+
+  void SetNew(
+      const SharedQuadState* shared_quad_state,
+      const gfx::Rect& rect,
+      const gfx::Rect& visible_rect,
+      bool needs_blending,
+      unsigned resource_id,
+      gfx::Size resource_size_in_pixels,
+      const gfx::PointF& uv_top_left,
+      const gfx::PointF& uv_bottom_right,
+      const base::Optional<gpu::VulkanYCbCrInfo>& ycbcr_info = base::nullopt);
 
   void SetAll(const SharedQuadState* shared_quad_state,
               const gfx::Rect& rect,
@@ -48,6 +53,9 @@ class VIZ_COMMON_EXPORT StreamVideoDrawQuad : public DrawQuad {
   };
   OverlayResources overlay_resources;
 
+  // Sampler conversion information which is used in vulkan context for android.
+  base::Optional<gpu::VulkanYCbCrInfo> ycbcr_info;
+
   static const StreamVideoDrawQuad* MaterialCast(const DrawQuad*);
 
   ResourceId resource_id() const { return resources.ids[kResourceIdIndex]; }

diff --git a/components/viz/service/BUILD.gn b/components/viz/service/BUILD.gn
@@ -198,6 +198,7 @@ viz_component("service") {
     # Note that dependency on //gpu/ipc/client is for GpuMemoryBufferImpl. This
     # dependency should not be in public_deps.
     "//gpu/ipc/client",
+    "//gpu/ipc/common:common",
     "//gpu/ipc/common:surface_handle_type",
     "//gpu/ipc/service",
     "//gpu/skia_bindings:skia_bindings",

diff --git a/components/viz/service/display/DEPS b/components/viz/service/display/DEPS
@@ -22,6 +22,7 @@ include_rules = [
   "+ui/gl/ca_renderer_layer_params.h",
   "+ui/gl/dc_renderer_layer_params.h",
   "+ui/gl/trace_util.h",
+  "+gpu/ipc/common/vulkan_ycbcr_info.h",
 ]
 
 specific_include_rules = {

diff --git a/components/viz/service/display/resource_metadata.h b/components/viz/service/display/resource_metadata.h
@@ -5,11 +5,13 @@
 #ifndef COMPONENTS_VIZ_SERVICE_DISPLAY_RESOURCE_METADATA_H_
 #define COMPONENTS_VIZ_SERVICE_DISPLAY_RESOURCE_METADATA_H_
 
+#include "base/optional.h"
 #include "components/viz/common/resources/resource_format.h"
 #include "components/viz/common/resources/resource_id.h"
 #include "components/viz/service/viz_service_export.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
 #include "gpu/command_buffer/common/sync_token.h"
+#include "gpu/ipc/common/vulkan_ycbcr_info.h"
 #include "third_party/skia/include/gpu/GrTypes.h"
 #include "ui/gfx/color_space.h"
 #include "ui/gfx/geometry/size.h"
@@ -44,6 +46,10 @@ struct VIZ_SERVICE_EXPORT ResourceMetadata {
 
   // If the SkImage should use top-left or bottom-left for (0,0) uv
   GrSurfaceOrigin origin;
+
+  // Sampler conversion information which is used in vulkan context for android
+  // video.
+  base::Optional<gpu::VulkanYCbCrInfo> ycbcr_info;
 };
 
 }  // namespace viz

diff --git a/components/viz/service/display/skia_renderer.cc b/components/viz/service/display/skia_renderer.cc
@@ -451,6 +451,11 @@ class SkiaRenderer::ScopedSkImageBuilder {
                        ResourceId resource_id,
                        SkAlphaType alpha_type = kPremul_SkAlphaType,
                        GrSurfaceOrigin origin = kTopLeft_GrSurfaceOrigin);
+  ScopedSkImageBuilder(SkiaRenderer* skia_renderer,
+                       ResourceId resource_id,
+                       base::Optional<gpu::VulkanYCbCrInfo> ycbcr_info,
+                       SkAlphaType alpha_type = kPremul_SkAlphaType,
+                       GrSurfaceOrigin origin = kTopLeft_GrSurfaceOrigin);
   ~ScopedSkImageBuilder() = default;
 
   const SkImage* sk_image() const { return sk_image_; }
@@ -466,6 +471,18 @@ SkiaRenderer::ScopedSkImageBuilder::ScopedSkImageBuilder(
     SkiaRenderer* skia_renderer,
     ResourceId resource_id,
     SkAlphaType alpha_type,
+    GrSurfaceOrigin origin)
+    : SkiaRenderer::ScopedSkImageBuilder(skia_renderer,
+                                         resource_id,
+                                         base::nullopt,
+                                         alpha_type,
+                                         origin) {}
+
+SkiaRenderer::ScopedSkImageBuilder::ScopedSkImageBuilder(
+    SkiaRenderer* skia_renderer,
+    ResourceId resource_id,
+    base::Optional<gpu::VulkanYCbCrInfo> ycbcr_info,
+    SkAlphaType alpha_type,
     GrSurfaceOrigin origin) {
   if (!resource_id)
     return;
@@ -486,6 +503,7 @@ SkiaRenderer::ScopedSkImageBuilder::ScopedSkImageBuilder(
           skia_renderer->lock_set_for_external_use_->LockResource(resource_id);
       metadata.alpha_type = alpha_type;
       metadata.origin = origin;
+      metadata.ycbcr_info = ycbcr_info;
       image = skia_renderer->skia_output_surface_->MakePromiseSkImage(metadata);
       LOG_IF(ERROR, !image) << "Failed to create the promise sk image.";
     }
@@ -1267,7 +1285,8 @@ void SkiaRenderer::DrawSolidColorQuad(const SolidColorDrawQuad* quad,
 void SkiaRenderer::DrawStreamVideoQuad(const StreamVideoDrawQuad* quad,
                                        DrawQuadParams* params) {
   DCHECK(!MustFlushBatchedQuads(quad, *params));
-  ScopedSkImageBuilder builder(this, quad->resource_id(),
+
+  ScopedSkImageBuilder builder(this, quad->resource_id(), quad->ycbcr_info,
                                kUnpremul_SkAlphaType);
   const SkImage* image = builder.sk_image();
   if (!image)

diff --git a/components/viz/service/display_embedder/skia_output_surface_impl.cc b/components/viz/service/display_embedder/skia_output_surface_impl.cc
@@ -224,8 +224,10 @@ sk_sp<SkImage> SkiaOutputSurfaceImpl::MakePromiseSkImage(
     image_context = std::make_unique<ImageContext>(metadata);
     SkColorType color_type = ResourceFormatToClosestSkColorType(
         true /* gpu_compositing */, metadata.resource_format);
+
     GrBackendFormat backend_format = GetGrBackendFormatForTexture(
-        metadata.resource_format, metadata.mailbox_holder.texture_target);
+        metadata.resource_format, metadata.mailbox_holder.texture_target,
+        metadata.ycbcr_info);
     image_context->image = recorder_->makePromiseTexture(
         backend_format, metadata.size.width(), metadata.size.height(),
         GrMipMapped::kNo, metadata.origin, color_type, metadata.alpha_type,
@@ -594,8 +596,10 @@ void SkiaOutputSurfaceImpl::ScheduleGpuTask(
 
 GrBackendFormat SkiaOutputSurfaceImpl::GetGrBackendFormatForTexture(
     ResourceFormat resource_format,
-    uint32_t gl_texture_target) {
+    uint32_t gl_texture_target,
+    base::Optional<gpu::VulkanYCbCrInfo> ycbcr_info) {
   if (!is_using_vulkan_) {
+    DCHECK(!ycbcr_info);
     // Convert internal format from GLES2 to platform GL.
     const auto* version_info = impl_on_gpu_->gl_version_info();
     unsigned int texture_storage_format = TextureStorageFormat(resource_format);
@@ -607,7 +611,20 @@ GrBackendFormat SkiaOutputSurfaceImpl::GetGrBackendFormatForTexture(
         gl_texture_target);
   } else {
 #if BUILDFLAG(ENABLE_VULKAN)
-    return GrBackendFormat::MakeVk(ToVkFormat(resource_format));
+    if (!ycbcr_info)
+      return GrBackendFormat::MakeVk(ToVkFormat(resource_format));
+
+    GrVkYcbcrConversionInfo fYcbcrConversionInfo(
+        static_cast<VkSamplerYcbcrModelConversion>(
+            ycbcr_info->suggested_ycbcr_model),
+        static_cast<VkSamplerYcbcrRange>(ycbcr_info->suggested_ycbcr_range),
+        static_cast<VkChromaLocation>(ycbcr_info->suggested_xchroma_offset),
+        static_cast<VkChromaLocation>(ycbcr_info->suggested_ychroma_offset),
+        VK_FILTER_LINEAR,  // VkFilter
+        0,                 // VkBool32 forceExplicitReconstruction,
+        ycbcr_info->external_format,
+        static_cast<VkFormatFeatureFlags>(ycbcr_info->format_features));
+    return GrBackendFormat::MakeVk(fYcbcrConversionInfo);
 #else
     NOTREACHED();
     return GrBackendFormat();

diff --git a/components/viz/service/display_embedder/skia_output_surface_impl.h b/components/viz/service/display_embedder/skia_output_surface_impl.h
@@ -16,6 +16,7 @@
 #include "components/viz/service/viz_service_export.h"
 #include "gpu/command_buffer/common/sync_token.h"
 #include "gpu/ipc/common/surface_handle.h"
+#include "gpu/ipc/common/vulkan_ycbcr_info.h"
 #include "gpu/ipc/in_process_command_buffer.h"
 #include "third_party/skia/include/core/SkDeferredDisplayListRecorder.h"
 #include "third_party/skia/include/core/SkOverdrawCanvas.h"
@@ -107,8 +108,10 @@ class VIZ_SERVICE_EXPORT SkiaOutputSurfaceImpl : public SkiaOutputSurfaceBase {
   void BufferPresented(const gfx::PresentationFeedback& feedback);
   void ScheduleGpuTask(base::OnceClosure callback,
                        std::vector<gpu::SyncToken> sync_tokens);
-  GrBackendFormat GetGrBackendFormatForTexture(ResourceFormat resource_format,
-                                               uint32_t gl_texture_target);
+  GrBackendFormat GetGrBackendFormatForTexture(
+      ResourceFormat resource_format,
+      uint32_t gl_texture_target,
+      base::Optional<gpu::VulkanYCbCrInfo> ycbcr_info = base::nullopt);
 
   uint64_t sync_fence_release_ = 0;
   GpuServiceImpl* const gpu_service_;

diff --git a/media/base/BUILD.gn b/media/base/BUILD.gn
@@ -311,6 +311,7 @@ jumbo_source_set("base") {
   deps = [
     "//base/allocator:buildflags",
     "//gpu/command_buffer/common",
+    "//gpu/ipc/common:common",
     "//skia",
     "//third_party/libyuv",
     "//third_party/widevine/cdm:headers",
@@ -319,6 +320,7 @@ jumbo_source_set("base") {
     "//ui/events:events_base",
     "//url:url",
   ]
+
   libs = []
   configs += [
     "//build/config:precompiled_headers",

diff --git a/media/base/video_frame.h b/media/base/video_frame.h
@@ -22,11 +22,13 @@
 #include "base/memory/shared_memory.h"
 #include "base/memory/shared_memory_handle.h"
 #include "base/memory/unsafe_shared_memory_region.h"
+#include "base/optional.h"
 #include "base/synchronization/lock.h"
 #include "base/thread_annotations.h"
 #include "base/unguessable_token.h"
 #include "build/build_config.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
+#include "gpu/ipc/common/vulkan_ycbcr_info.h"
 #include "media/base/video_frame_layout.h"
 #include "media/base/video_frame_metadata.h"
 #include "media/base/video_types.h"
@@ -434,6 +436,10 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
     return data_[plane];
   }
 
+  const base::Optional<gpu::VulkanYCbCrInfo>& ycbcr_info() const {
+    return ycbcr_info_;
+  }
+
   // Returns pointer to the data in the visible region of the frame, for
   // IsMappable() storage types. The returned pointer is offsetted into the
   // plane buffer specified by visible_rect().origin(). Memory is owned by
@@ -537,6 +543,11 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
   // Returns the number of bits per channel.
   size_t BitDepth() const;
 
+  // Provide the sampler conversion information for the frame.
+  void set_ycbcr_info(const base::Optional<gpu::VulkanYCbCrInfo>& ycbcr_info) {
+    ycbcr_info_ = ycbcr_info;
+  }
+
  protected:
   friend class base::RefCountedThreadSafe<VideoFrame>;
 
@@ -678,6 +689,9 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
 
   gfx::ColorSpace color_space_;
 
+  // Sampler conversion information which is used in vulkan context for android.
+  base::Optional<gpu::VulkanYCbCrInfo> ycbcr_info_;
+
   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame);
 };
 

diff --git a/media/gpu/BUILD.gn b/media/gpu/BUILD.gn
@@ -5,6 +5,7 @@
 import("//build/buildflag_header.gni")
 import("//build/config/features.gni")
 import("//build/config/ui.gni")
+import("//gpu/vulkan/features.gni")
 import("//media/gpu/args.gni")
 import("//media/media_options.gni")
 import("//testing/test.gni")
@@ -148,6 +149,9 @@ component("gpu") {
       "//services/service_manager/public/cpp:cpp",
       "//ui/gl:gl_jni_headers",
     ]
+    if (enable_vulkan) {
+      deps += [ "//gpu/vulkan:vulkan" ]
+    }
 
     # TODO(crbug.com/789435): This is needed for AVDA to access the CDM
     # directly.  Remove this dependency after VDAs are also running as part of

diff --git a/media/gpu/DEPS b/media/gpu/DEPS
@@ -13,6 +13,7 @@ include_rules = [
   "+ui/display/manager",
   "+ui/display/types",
   "+ui/platform_window",
+  "+components/viz/common/gpu/vulkan_context_provider.h",
 
   # media/gpu is not part of "media" target and should not use MEDIA_EXPORT.
   "-media/base/media_export.h",

diff --git a/media/gpu/android/image_reader_gl_owner.cc b/media/gpu/android/image_reader_gl_owner.cc
@@ -69,8 +69,10 @@ class ImageReaderGLOwner::ScopedHardwareBufferImpl
   }
 
   void SetReadFence(base::ScopedFD fence_fd, bool has_context) final {
-    DCHECK(!read_fence_.is_valid());
-    read_fence_ = std::move(fence_fd);
+    // Client can call this method multiple times for a hardware buffer. Hence
+    // all the client provided sync_fd should be merged. Eg: BeginReadAccess()
+    // can be called multiple times for a SharedImageVideo representation.
+    read_fence_ = gl::MergeFDs(std::move(read_fence_), std::move(fence_fd));
   }
 
  private: