Bug 1497294 - P7. Add P016 and P010 surface format support. r=jgilbert

This is only used with DXVA decoder. P016 and P010 are just like NV12 but with 16 bits data..

Depends on D8246

Differential Revision: https://phabricator.services.mozilla.com/D8136

dom/canvas/ImageUtils.cpp

@@ -42,6 +42,8 @@ GetImageBitmapFormatFromSurfaceFromat(SurfaceFormat aSurfaceFormat)
   case SurfaceFormat::R5G6B5_UINT16:
   case SurfaceFormat::YUV:
   case SurfaceFormat::NV12:
+  case SurfaceFormat::P010:
+  case SurfaceFormat::P016:
   case SurfaceFormat::UNKNOWN:
   default:
     return ImageBitmapFormat::EndGuard_;

gfx/2d/Types.h

@@ -64,7 +64,13 @@ enum class SurfaceFormat : int8_t {
 
   // These ones are their own special cases.
   YUV,
-  NV12,
+  NV12,         // YUV 4:2:0 image with a plane of 8 bit Y samples followed by
+                // an interleaved U/V plane containing 8 bit 2x2 subsampled
+                // colour difference samples.
+  P016,         // Similar to NV12, but with 16 bits plane values
+  P010,         // Identical to P016 but the 6 least significant bits are 0.
+                // With DXGI in theory entirely compatible, however practice has
+                // shown that it's not the case.
   YUV422,
   HSV,
   Lab,
@@ -124,6 +130,8 @@ inline bool IsOpaque(SurfaceFormat aFormat)
   case SurfaceFormat::Depth:
   case SurfaceFormat::YUV:
   case SurfaceFormat::NV12:
+  case SurfaceFormat::P010:
+  case SurfaceFormat::P016:
   case SurfaceFormat::YUV422:
     return true;
   default:

gfx/gl/GLBlitHelperD3D.cpp

@@ -253,10 +253,12 @@ GLBlitHelper::BlitDescriptor(const layers::SurfaceDescriptorD3D10& desc,
     const gfx::IntRect clipRect(0, 0, clipSize.width, clipSize.height);
     const auto colorSpace = YUVColorSpace::BT601;
 
-    if (format != gfx::SurfaceFormat::NV12) {
+    if (format != gfx::SurfaceFormat::NV12 &&
+        format != gfx::SurfaceFormat::P010 &&
+        format != gfx::SurfaceFormat::P016) {
         gfxCriticalError() << "Non-NV12 format for SurfaceDescriptorD3D10: "
                            << uint32_t(format);
-        return false;
+      return false;
     }
 
     const auto tex = OpenSharedTexture(d3d, handle);

gfx/layers/D3D11ShareHandleImage.cpp

@@ -44,23 +44,39 @@ D3D11ShareHandleImage::AllocateTexture(D3D11RecycleAllocator* aAllocator,
         gfx::DeviceManagerDx::Get()->CanUseNV12()) {
       mTextureClient =
         aAllocator->CreateOrRecycleClient(gfx::SurfaceFormat::NV12, mSize);
+    } else if (((mSourceFormat == MFVideoFormat_P010 &&
+                 gfx::DeviceManagerDx::Get()->CanUseP010()) ||
+                (mSourceFormat == MFVideoFormat_P016 &&
+                 gfx::DeviceManagerDx::Get()->CanUseP016())) &&
+               gfxPrefs::PDMWMFUseNV12Format()) {
+      mTextureClient = aAllocator->CreateOrRecycleClient(
+        mSourceFormat == MFVideoFormat_P010 ? gfx::SurfaceFormat::P010
+                                            : gfx::SurfaceFormat::P016,
+        mSize);
     } else {
       mTextureClient =
         aAllocator->CreateOrRecycleClient(gfx::SurfaceFormat::B8G8R8A8, mSize);
     }
     if (mTextureClient) {
-      mTexture = static_cast<D3D11TextureData*>(mTextureClient->GetInternalData())->GetD3D11Texture();
+      mTexture =
+        static_cast<D3D11TextureData*>(mTextureClient->GetInternalData())
+          ->GetD3D11Texture();
       return true;
     }
     return false;
   } else {
     MOZ_ASSERT(aDevice);
     CD3D11_TEXTURE2D_DESC newDesc(DXGI_FORMAT_B8G8R8A8_UNORM,
-                                  mSize.width, mSize.height, 1, 1,
-                                  D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE);
+                                  mSize.width,
+                                  mSize.height,
+                                  1,
+                                  1,
+                                  D3D11_BIND_RENDER_TARGET |
+                                    D3D11_BIND_SHADER_RESOURCE);
     newDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
 
-    HRESULT hr = aDevice->CreateTexture2D(&newDesc, nullptr, getter_AddRefs(mTexture));
+    HRESULT hr =
+      aDevice->CreateTexture2D(&newDesc, nullptr, getter_AddRefs(mTexture));
     return SUCCEEDED(hr);
   }
 }
@@ -186,7 +202,8 @@ D3D11ShareHandleImage::GetAsSourceSurface()
 }
 
 ID3D11Texture2D*
-D3D11ShareHandleImage::GetTexture() const {
+D3D11ShareHandleImage::GetTexture() const
+{
   return mTexture;
 }
 

gfx/layers/Effects.h

@@ -252,6 +252,8 @@ CreateTexturedEffect(gfx::SurfaceFormat aFormat,
     result = new EffectRGB(aSource, isAlphaPremultiplied, aSamplingFilter);
     break;
   case gfx::SurfaceFormat::NV12:
+  case gfx::SurfaceFormat::P010:
+  case gfx::SurfaceFormat::P016:
     result = new EffectNV12(aSource, aSamplingFilter);
     break;
   case gfx::SurfaceFormat::YUV:

gfx/layers/LayersLogging.cpp

@@ -390,6 +390,8 @@ AppendToString(std::stringstream& aStream, mozilla::gfx::SurfaceFormat format,
   case SurfaceFormat::A8:        aStream << "SurfaceFormat::A8"; break;
   case SurfaceFormat::YUV:       aStream << "SurfaceFormat::YUV"; break;
   case SurfaceFormat::NV12:      aStream << "SurfaceFormat::NV12"; break;
+  case SurfaceFormat::P010:      aStream << "SurfaceFormat::P010"; break;
+  case SurfaceFormat::P016:      aStream << "SurfaceFormat::P016"; break;
   case SurfaceFormat::YUV422:    aStream << "SurfaceFormat::YUV422"; break;
   case SurfaceFormat::UNKNOWN:   aStream << "SurfaceFormat::UNKNOWN"; break;
   default:

gfx/layers/d3d11/CompositorD3D11.cpp

@@ -848,19 +848,27 @@ CompositorD3D11::DrawGeometry(const Geometry& aGeometry,
         NS_WARNING("No texture found in texture source!");
       }
 
+      D3D11_TEXTURE2D_DESC sourceDesc;
+      texture->GetDesc(&sourceDesc);
+      MOZ_DIAGNOSTIC_ASSERT(sourceDesc.Format == DXGI_FORMAT_NV12 ||
+                            sourceDesc.Format == DXGI_FORMAT_P010 ||
+                            sourceDesc.Format == DXGI_FORMAT_P016);
+
       // Might want to cache these for efficiency.
       RefPtr<ID3D11ShaderResourceView> srViewY;
       RefPtr<ID3D11ShaderResourceView> srViewCbCr;
       D3D11_SHADER_RESOURCE_VIEW_DESC srvDesc =
         CD3D11_SHADER_RESOURCE_VIEW_DESC(D3D11_SRV_DIMENSION_TEXTURE2D,
-                                         DXGI_FORMAT_R8_UNORM);
-      mDevice->CreateShaderResourceView(texture,
-                                        &srvDesc,
-                                        getter_AddRefs(srViewY));
-      srvDesc.Format = DXGI_FORMAT_R8G8_UNORM;
-      mDevice->CreateShaderResourceView(texture,
-                                        &srvDesc,
-                                        getter_AddRefs(srViewCbCr));
+                                         sourceDesc.Format == DXGI_FORMAT_NV12
+                                           ? DXGI_FORMAT_R8_UNORM
+                                           : DXGI_FORMAT_R16_UNORM);
+      mDevice->CreateShaderResourceView(
+        texture, &srvDesc, getter_AddRefs(srViewY));
+      srvDesc.Format = sourceDesc.Format == DXGI_FORMAT_NV12
+                         ? DXGI_FORMAT_R8G8_UNORM
+                         : DXGI_FORMAT_R16G16_UNORM;
+      mDevice->CreateShaderResourceView(
+        texture, &srvDesc, getter_AddRefs(srViewCbCr));
 
       ID3D11ShaderResourceView* views[] = { srViewY, srViewCbCr };
       mContext->PSSetShaderResources(TexSlot::Y, 2, views);

gfx/layers/d3d11/MLGDeviceD3D11.cpp

@@ -1748,7 +1748,9 @@ MLGDeviceD3D11::MaybeLockTexture(ID3D11Texture2D* aTexture)
 void
 MLGDeviceD3D11::SetPSTexturesNV12(uint32_t aSlot, TextureSource* aTexture)
 {
-  MOZ_ASSERT(aTexture->GetFormat() == SurfaceFormat::NV12);
+  MOZ_ASSERT(aTexture->GetFormat() == SurfaceFormat::NV12 ||
+             aTexture->GetFormat() == SurfaceFormat::P010 ||
+             aTexture->GetFormat() == SurfaceFormat::P016);
 
   TextureSourceD3D11* source = aTexture->AsSourceD3D11();
   if (!source) {
@@ -1764,26 +1766,24 @@ MLGDeviceD3D11::SetPSTexturesNV12(uint32_t aSlot, TextureSource* aTexture)
 
   MaybeLockTexture(texture);
 
+  const bool isNV12 = aTexture->GetFormat() == SurfaceFormat::NV12;
+
   RefPtr<ID3D11ShaderResourceView> views[2];
-  D3D11_SHADER_RESOURCE_VIEW_DESC desc =
-    CD3D11_SHADER_RESOURCE_VIEW_DESC(
-      D3D11_SRV_DIMENSION_TEXTURE2D,
-      DXGI_FORMAT_R8_UNORM);
-
-  HRESULT hr = mDevice->CreateShaderResourceView(
-    texture,
-    &desc,
-    getter_AddRefs(views[0]));
+  D3D11_SHADER_RESOURCE_VIEW_DESC desc = CD3D11_SHADER_RESOURCE_VIEW_DESC(
+    D3D11_SRV_DIMENSION_TEXTURE2D,
+    isNV12 ? DXGI_FORMAT_R8_UNORM : DXGI_FORMAT_R16_UNORM);
+
+  HRESULT hr =
+    mDevice->CreateShaderResourceView(texture, &desc, getter_AddRefs(views[0]));
   if (FAILED(hr) || !views[0]) {
-    gfxWarning() << "Could not bind an SRV for Y plane of NV12 texture: " << hexa(hr);
+    gfxWarning() << "Could not bind an SRV for Y plane of NV12 texture: "
+                 << hexa(hr);
     return;
   }
 
-  desc.Format = DXGI_FORMAT_R8G8_UNORM;
-  hr = mDevice->CreateShaderResourceView(
-    texture,
-    &desc,
-    getter_AddRefs(views[1]));
+  desc.Format = isNV12 ? DXGI_FORMAT_R8G8_UNORM : DXGI_FORMAT_R16G16_UNORM;
+  hr =
+    mDevice->CreateShaderResourceView(texture, &desc, getter_AddRefs(views[1]));
   if (FAILED(hr) || !views[1]) {
     gfxWarning() << "Could not bind an SRV for CbCr plane of NV12 texture: " << hexa(hr);
     return;

gfx/layers/d3d11/MLGDeviceD3D11.h

@@ -4,8 +4,8 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#ifndef mozilla_gfx_layers_d3d11_MLGDeviceD3D11_h 
-#define mozilla_gfx_layers_d3d11_MLGDeviceD3D11_h 
+#ifndef mozilla_gfx_layers_d3d11_MLGDeviceD3D11_h
+#define mozilla_gfx_layers_d3d11_MLGDeviceD3D11_h
 
 #include <d3d11_1.h>
 
@@ -319,7 +319,7 @@ class MLGDeviceD3D11 final : public MLGDevice
 
   RefPtr<ID3D11Query> mWaitForPresentQuery;
   RefPtr<ID3D11Query> mNextWaitForPresentQuery;
-  
+
   nsTHashtable<nsRefPtrHashKey<IDXGIKeyedMutex>> mLockedTextures;
   nsTHashtable<nsRefPtrHashKey<IDXGIKeyedMutex>> mLockAttemptedTextures;
 
@@ -337,4 +337,4 @@ class MLGDeviceD3D11 final : public MLGDevice
 
 struct ShaderBytes;
 
-#endif // mozilla_gfx_layers_d3d11_MLGDeviceD3D11_h 
+#endif // mozilla_gfx_layers_d3d11_MLGDeviceD3D11_h

gfx/layers/d3d11/TextureD3D11.cpp

@@ -468,6 +468,10 @@ D3D11TextureData::Create(IntSize aSize, SurfaceFormat aFormat, SourceSurface* aS
 
   if (aFormat == SurfaceFormat::NV12) {
     newDesc.Format = DXGI_FORMAT_NV12;
+  } else if (aFormat == SurfaceFormat::P010) {
+    newDesc.Format = DXGI_FORMAT_P010;
+  } else if (aFormat == SurfaceFormat::P016) {
+    newDesc.Format = DXGI_FORMAT_P016;
   }
 
   newDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
@@ -1085,7 +1089,9 @@ DXGITextureHostD3D11::NumSubTextures() const
     case gfx::SurfaceFormat::B8G8R8X8: {
       return 1;
     }
-    case gfx::SurfaceFormat::NV12: {
+    case gfx::SurfaceFormat::NV12:
+    case gfx::SurfaceFormat::P010:
+    case gfx::SurfaceFormat::P016: {
       return 2;
     }
     default: {
@@ -1169,6 +1175,19 @@ DXGITextureHostD3D11::PushDisplayItems(wr::DisplayListBuilder& aBuilder,
                              aFilter);
       break;
     }
+    case gfx::SurfaceFormat::P010:
+    case gfx::SurfaceFormat::P016: {
+      MOZ_ASSERT(aImageKeys.length() == 2);
+      aBuilder.PushNV12Image(aBounds,
+                             aClip,
+                             true,
+                             aImageKeys[0],
+                             aImageKeys[1],
+                             wr::ColorDepth::Color16,
+                             wr::ToWrYuvColorSpace(YUVColorSpace::BT601),
+                             aFilter);
+      break;
+    }
     default: {
       MOZ_ASSERT_UNREACHABLE("unexpected to be called");
     }

gfx/layers/mlgpu/RenderPassMLGPU.cpp

@@ -120,8 +120,9 @@ RenderPassMLGPU::GetPreferredPassType(FrameBuilder* aBuilder, const ItemInfo& aI
     ImageHost* host = layer->AsTexturedLayerMLGPU()->GetImageHost();
     TextureHost* texture = host->CurrentTextureHost();
     if (texture->GetReadFormat() == SurfaceFormat::YUV ||
-        texture->GetReadFormat() == SurfaceFormat::NV12)
-    {
+        texture->GetReadFormat() == SurfaceFormat::NV12 ||
+        texture->GetReadFormat() == SurfaceFormat::P010 ||
+        texture->GetReadFormat() == SurfaceFormat::P016) {
       return RenderPassType::Video;
     }
     return RenderPassType::SingleTexture;
@@ -805,6 +806,9 @@ VideoRenderPass::SetupPipeline()
     break;
   }
   case SurfaceFormat::NV12:
+  case SurfaceFormat::P010:
+  case SurfaceFormat::P016:
+    // TODO. BT601 is very unlikely to be the right value for high-def content.
     colorSpace = YUVColorSpace::BT601;
     break;
   default:
@@ -841,6 +845,8 @@ VideoRenderPass::SetupPipeline()
     break;
   }
   case SurfaceFormat::NV12:
+  case SurfaceFormat::P010:
+  case SurfaceFormat::P016:
     if (mGeometry == GeometryMode::UnitQuad)
       mDevice->SetPixelShader(PixelShaderID::TexturedQuadNV12);
     else

gfx/webrender_bindings/RenderD3D11TextureHostOGL.cpp

@@ -29,7 +29,9 @@ RenderDXGITextureHostOGL::RenderDXGITextureHostOGL(WindowsHandle aHandle,
   , mLocked(false)
 {
   MOZ_COUNT_CTOR_INHERITED(RenderDXGITextureHostOGL, RenderTextureHostOGL);
-  MOZ_ASSERT(mFormat != gfx::SurfaceFormat::NV12 ||
+  MOZ_ASSERT((mFormat != gfx::SurfaceFormat::NV12 &&
+              mFormat != gfx::SurfaceFormat::P010 &&
+              mFormat != gfx::SurfaceFormat::P016) ||
              (mSize.width % 2 == 0 && mSize.height % 2 == 0));
   MOZ_ASSERT(aHandle);
 }
@@ -53,8 +55,10 @@ RenderDXGITextureHostOGL::EnsureLockable(wr::ImageRendering aRendering)
                                    aRendering);
       // Cache new rendering filter.
       mCachedRendering = aRendering;
-      // NV12 uses two handles.
-      if (mFormat == gfx::SurfaceFormat::NV12) {
+      // NV12 and P016 uses two handles.
+      if (mFormat == gfx::SurfaceFormat::NV12 ||
+          mFormat == gfx::SurfaceFormat::P010 ||
+          mFormat == gfx::SurfaceFormat::P016) {
         ActivateBindAndTexParameteri(mGL,
                                      LOCAL_GL_TEXTURE1,
                                      LOCAL_GL_TEXTURE_EXTERNAL_OES,
@@ -100,7 +104,9 @@ RenderDXGITextureHostOGL::EnsureLockable(wr::ImageRendering aRendering)
   mStream = egl->fCreateStreamKHR(egl->Display(), nullptr);
   MOZ_ASSERT(mStream);
 
-  if (mFormat != gfx::SurfaceFormat::NV12) {
+  if (mFormat != gfx::SurfaceFormat::NV12 &&
+      mFormat != gfx::SurfaceFormat::P010 &&
+      mFormat != gfx::SurfaceFormat::P016) {
     // The non-nv12 format.
 
     mGL->fGenTextures(1, mTextureHandle);
@@ -114,7 +120,7 @@ RenderDXGITextureHostOGL::EnsureLockable(wr::ImageRendering aRendering)
     MOZ_ALWAYS_TRUE(egl->fStreamConsumerGLTextureExternalAttribsNV(egl->Display(), mStream, nullptr));
     MOZ_ALWAYS_TRUE(egl->fCreateStreamProducerD3DTextureANGLE(egl->Display(), mStream, nullptr));
   } else {
-    // The nv12 format.
+    // The nv12/p016 format.
 
     // Setup the NV12 stream consumer/producer.
     EGLAttrib consumerAttributes[] = {
@@ -236,17 +242,22 @@ RenderDXGITextureHostOGL::DeleteTextureHandle()
 GLuint
 RenderDXGITextureHostOGL::GetGLHandle(uint8_t aChannelIndex) const
 {
-  MOZ_ASSERT(mFormat != gfx::SurfaceFormat::NV12 || aChannelIndex < 2);
-  MOZ_ASSERT(mFormat == gfx::SurfaceFormat::NV12 || aChannelIndex < 1);
-
+  MOZ_ASSERT(((mFormat == gfx::SurfaceFormat::NV12 ||
+               mFormat == gfx::SurfaceFormat::P010 ||
+               mFormat == gfx::SurfaceFormat::P016) &&
+              aChannelIndex < 2) ||
+             aChannelIndex < 1);
   return mTextureHandle[aChannelIndex];
 }
 
 gfx::IntSize
 RenderDXGITextureHostOGL::GetSize(uint8_t aChannelIndex) const
 {
-  MOZ_ASSERT(mFormat != gfx::SurfaceFormat::NV12 || aChannelIndex < 2);
-  MOZ_ASSERT(mFormat == gfx::SurfaceFormat::NV12 || aChannelIndex < 1);
+  MOZ_ASSERT(((mFormat == gfx::SurfaceFormat::NV12 ||
+               mFormat == gfx::SurfaceFormat::P010 ||
+               mFormat == gfx::SurfaceFormat::P016) &&
+              aChannelIndex < 2) ||
+             aChannelIndex < 1);
 
   if (aChannelIndex == 0) {
     return mSize;