Video Thumbnail: Decode the video frame on Android.

This CL does the following things:

1. Implements a GpuVideoAcceleratorFactories for browser process in
content/ to create MojoVideoDecoder in the browser process.

2. Decodes video frame with MojoVideoDecoder in browser process in
DownloadMediaParser. The actual decoding happens in GPU process with
Android MediaCodec, which is same as other usages for MediaCodec based
decoders.

Test is covered in DownloadMediaParserTest.

TBR=dtrainor@chromium.org

Bug: 826021
Change-Id: I242ff31ba8f205af3b03709fb78bd14f5bc33e45
Reviewed-on: https://chromium-review.googlesource.com/1212343
Commit-Queue: Xing Liu <xingliu@chromium.org>
Reviewed-by: Lei Zhang <thestig@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Reviewed-by: Antoine Labour <piman@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Min Qin <qinmin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#590431}

chrome/browser/BUILD.gn

@@ -2498,6 +2498,8 @@ jumbo_split_static_library("browser") {
       "//components/payments/content/android",
       "//components/resources:components_resources",
       "//components/toolbar",
+      "//media/mojo/clients",
+      "//media/mojo/interfaces:constants",
       "//rlz:rlz_utils",
       "//sandbox",
       "//sandbox:sandbox_buildflags",

chrome/browser/android/DEPS

@@ -3,6 +3,8 @@ include_rules = [
   "+cc/layers/layer.h",
   "+chrome_jni_registration/chrome_jni_registration.h",
   "+device/vr/buildflags/buildflags.h",
+  "+media/gpu",
+  "+media/video",
   "+sandbox/linux/seccomp-bpf/sandbox_bpf.h",
   "+sandbox/linux/seccomp-bpf-helpers",
   "+sandbox/sandbox_buildflags.h",

chrome/browser/android/download/download_media_parser.cc

@@ -10,7 +10,15 @@
 #include "base/task/post_task.h"
 #include "base/task/task_traits.h"
 #include "chrome/browser/android/download/local_media_data_source_factory.h"
+#include "content/public/browser/android/gpu_video_accelerator_factories_provider.h"
 #include "content/public/common/service_manager_connection.h"
+#include "media/base/overlay_info.h"
+#include "media/mojo/clients/mojo_video_decoder.h"
+#include "media/mojo/interfaces/constants.mojom.h"
+#include "media/mojo/interfaces/media_service.mojom.h"
+#include "media/mojo/services/media_interface_provider.h"
+#include "media/video/gpu_video_accelerator_factories.h"
+#include "services/service_manager/public/cpp/connector.h"
 
 namespace {
 
@@ -22,6 +30,12 @@ bool IsSupportedMediaMimeType(const std::string& mime_type) {
                           base::CompareCase::INSENSITIVE_ASCII);
 }
 
+void OnRequestOverlayInfo(bool decoder_requires_restart_for_overlay,
+                          const media::ProvideOverlayInfoCB& overlay_info_cb) {
+  // No android overlay associated with video thumbnail.
+  overlay_info_cb.Run(media::OverlayInfo());
+}
+
 }  // namespace
 
 DownloadMediaParser::DownloadMediaParser(int64_t size,
@@ -34,6 +48,7 @@ DownloadMediaParser::DownloadMediaParser(int64_t size,
       parse_complete_cb_(std::move(parse_complete_cb)),
       file_task_runner_(
           base::CreateSingleThreadTaskRunnerWithTraits({base::MayBlock()})),
+      decode_done_(false),
       weak_factory_(this) {}
 
 DownloadMediaParser::~DownloadMediaParser() = default;
@@ -64,16 +79,15 @@ void DownloadMediaParser::OnMediaParserCreated() {
 }
 
 void DownloadMediaParser::OnConnectionError() {
-  if (parse_complete_cb_)
-    std::move(parse_complete_cb_).Run(false);
+  OnError();
 }
 
 void DownloadMediaParser::OnMediaMetadataParsed(
     bool parse_success,
     chrome::mojom::MediaMetadataPtr metadata,
     const std::vector<metadata::AttachedImage>& attached_images) {
   if (!parse_success) {
-    std::move(parse_complete_cb_).Run(false);
+    OnError();
     return;
   }
   metadata_ = std::move(metadata);
@@ -126,10 +140,108 @@ void DownloadMediaParser::OnEncodedVideoFrameRetrieved(
   encoded_data_ = data;
   config_ = config;
 
-  // TODO(xingliu): Decode the video frame with MojoVideoDecoder.
+  content::CreateGpuVideoAcceleratorFactories(base::BindRepeating(
+      &DownloadMediaParser::OnGpuVideoAcceleratorFactoriesReady,
+      weak_factory_.GetWeakPtr()));
+}
+
+void DownloadMediaParser::OnGpuVideoAcceleratorFactoriesReady(
+    std::unique_ptr<media::GpuVideoAcceleratorFactories> factories) {
+  gpu_factories_ = std::move(factories);
+  DecodeVideoFrame();
+}
+
+void DownloadMediaParser::DecodeVideoFrame() {
+  media::mojom::VideoDecoderPtr video_decoder_ptr;
+  GetMediaInterfaceFactory()->CreateVideoDecoder(
+      mojo::MakeRequest(&video_decoder_ptr));
+
+  // Build and config the decoder.
+  DCHECK(gpu_factories_);
+  decoder_ = std::make_unique<media::MojoVideoDecoder>(
+      base::ThreadTaskRunnerHandle::Get(), gpu_factories_.get(), this,
+      std::move(video_decoder_ptr), base::BindRepeating(&OnRequestOverlayInfo),
+      gfx::ColorSpace());
+
+  decoder_->Initialize(
+      config_, false, nullptr,
+      base::BindRepeating(&DownloadMediaParser::OnVideoDecoderInitialized,
+                          weak_factory_.GetWeakPtr()),
+      base::BindRepeating(&DownloadMediaParser::OnVideoFrameDecoded,
+                          weak_factory_.GetWeakPtr()),
+      base::RepeatingClosure());
+}
+
+void DownloadMediaParser::OnVideoDecoderInitialized(bool success) {
+  if (!success) {
+    OnError();
+    return;
+  }
+
+  // Build the video buffer to decode.
+  auto buffer =
+      media::DecoderBuffer::CopyFrom(&encoded_data_[0], encoded_data_.size());
+  encoded_data_.clear();
+
+  // Decode one frame buffer, followed by eos buffer.
+  DCHECK_GE(decoder_->GetMaxDecodeRequests(), 2);
+  decoder_->Decode(
+      buffer, base::BindRepeating(&DownloadMediaParser::OnVideoBufferDecoded,
+                                  weak_factory_.GetWeakPtr()));
+  decoder_->Decode(media::DecoderBuffer::CreateEOSBuffer(),
+                   base::BindRepeating(&DownloadMediaParser::OnEosBufferDecoded,
+                                       weak_factory_.GetWeakPtr()));
+}
+
+void DownloadMediaParser::OnVideoBufferDecoded(media::DecodeStatus status) {
+  if (status != media::DecodeStatus::OK)
+    OnError();
+}
+
+void DownloadMediaParser::OnEosBufferDecoded(media::DecodeStatus status) {
+  if (status != media::DecodeStatus::OK)
+    OnError();
+
+  // Fails if no decoded video frame is generated when eos arrives.
+  if (!decode_done_)
+    OnError();
+}
+
+void DownloadMediaParser::OnVideoFrameDecoded(
+    const scoped_refptr<media::VideoFrame>& frame) {
+  DCHECK(frame);
+  DCHECK(frame->HasTextures());
+  decode_done_ = true;
+
+  // TODO(xingliu): render the |frame| in the browser process, with cc or skia
+  // or gl. The |frame| is associated with a native texture in GPU process.
   NotifyComplete();
 }
 
+media::mojom::InterfaceFactory*
+DownloadMediaParser::GetMediaInterfaceFactory() {
+  if (!media_interface_factory_) {
+    service_manager::mojom::InterfaceProviderPtr interfaces;
+    media_interface_provider_ = std::make_unique<media::MediaInterfaceProvider>(
+        mojo::MakeRequest(&interfaces));
+    media::mojom::MediaServicePtr media_service;
+    content::ServiceManagerConnection::GetForProcess()
+        ->GetConnector()
+        ->BindInterface(media::mojom::kMediaServiceName, &media_service);
+    media_service->CreateInterfaceFactory(
+        MakeRequest(&media_interface_factory_), std::move(interfaces));
+    media_interface_factory_.set_connection_error_handler(
+        base::BindOnce(&DownloadMediaParser::OnDecoderConnectionError,
+                       base::Unretained(this)));
+  }
+
+  return media_interface_factory_.get();
+}
+
+void DownloadMediaParser::OnDecoderConnectionError() {
+  OnError();
+}
+
 void DownloadMediaParser::OnMediaDataReady(
     chrome::mojom::MediaDataSource::ReadCallback callback,
     std::unique_ptr<std::string> data) {

chrome/browser/android/download/download_media_parser.h

@@ -18,14 +18,19 @@
 #include "chrome/common/media_galleries/metadata_types.h"
 #include "chrome/services/media_gallery_util/public/cpp/media_parser_provider.h"
 #include "chrome/services/media_gallery_util/public/mojom/media_parser.mojom.h"
+#include "media/base/media_log.h"
+#include "media/mojo/interfaces/interface_factory.mojom.h"
 
 namespace media {
+class GpuVideoAcceleratorFactories;
+class MediaInterfaceProvider;
+class MojoVideoDecoder;
 class VideoDecoderConfig;
 }  // namespace media
 
 // Local media files parser is used to process local media files. This object
 // lives on main thread in browser process.
-class DownloadMediaParser : public MediaParserProvider {
+class DownloadMediaParser : public MediaParserProvider, public media::MediaLog {
  public:
   using ParseCompleteCB = base::OnceCallback<void(bool)>;
 
@@ -58,6 +63,18 @@ class DownloadMediaParser : public MediaParserProvider {
                                     const std::vector<uint8_t>& data,
                                     const media::VideoDecoderConfig& config);
 
+  // Decodes the video frame.
+  void OnGpuVideoAcceleratorFactoriesReady(
+      std::unique_ptr<media::GpuVideoAcceleratorFactories>);
+  void DecodeVideoFrame();
+  void OnVideoDecoderInitialized(bool success);
+  void OnVideoBufferDecoded(media::DecodeStatus status);
+  void OnEosBufferDecoded(media::DecodeStatus status);
+  void OnVideoFrameDecoded(
+      const scoped_refptr<media::VideoFrame>& decoded_frame);
+  media::mojom::InterfaceFactory* GetMediaInterfaceFactory();
+  void OnDecoderConnectionError();
+
   // Overlays media data source read operation. Gradually read data from media
   // file.
   void OnMediaDataReady(chrome::mojom::MediaDataSource::ReadCallback callback,
@@ -83,7 +100,14 @@ class DownloadMediaParser : public MediaParserProvider {
   // This data can be large for high resolution video, should be std::move or
   // cleared whenever possible.
   std::vector<uint8_t> encoded_data_;
+
+  // Objects used to decode the video into media::VideoFrame.
   media::VideoDecoderConfig config_;
+  std::unique_ptr<media::MojoVideoDecoder> decoder_;
+  media::mojom::InterfaceFactoryPtr media_interface_factory_;
+  std::unique_ptr<media::MediaInterfaceProvider> media_interface_provider_;
+  std::unique_ptr<media::GpuVideoAcceleratorFactories> gpu_factories_;
+  bool decode_done_;
 
   base::WeakPtrFactory<DownloadMediaParser> weak_factory_;
 

content/browser/BUILD.gn

@@ -1084,6 +1084,8 @@ jumbo_source_set("browser") {
     "manifest/manifest_icon_downloader.cc",
     "manifest/manifest_manager_host.cc",
     "manifest/manifest_manager_host.h",
+    "media/android/browser_gpu_video_accelerator_factories.cc",
+    "media/android/browser_gpu_video_accelerator_factories.h",
     "media/android/browser_media_player_manager.cc",
     "media/android/browser_media_player_manager.h",
     "media/android/media_player_renderer.cc",