Add support for YUV images.

YUV images are handled by a dedicated shader which reads the Y, U and V planes from separate
texture samplers and computes the rgb values using the Rec601 or Rec709 color spaces.

Author: nical

webrender/res/prim_shared.glsl

@@ -595,6 +595,33 @@ Image fetch_image(int index) {
     return image;
 }
 
+// YUV color spaces
+#define YUV_REC601 1
+#define YUV_REC709 2
+
+struct YuvImage {
+    vec4 y_st_rect;
+    vec4 u_st_rect;
+    vec4 v_st_rect;
+    vec2 size;
+    int color_space;
+};
+
+YuvImage fetch_yuv_image(int index) {
+    YuvImage image;
+
+    ivec2 uv = get_fetch_uv_2(index);
+
+    image.y_st_rect = texelFetchOffset(sData64, uv, 0, ivec2(0, 0));
+    image.u_st_rect = texelFetchOffset(sData64, uv, 0, ivec2(1, 0));
+    image.v_st_rect = texelFetchOffset(sData64, uv, 0, ivec2(2, 0));
+    vec4 size_color_space = texelFetchOffset(sData64, uv, 0, ivec2(3, 0));
+    image.size = size_color_space.xy;
+    image.color_space = int(size_color_space.z);
+
+    return image;
+}
+
 struct BoxShadow {
     vec4 src_rect;
     vec4 bs_rect;

webrender/res/ps_yuv_image.fs.glsl

@@ -0,0 +1,33 @@
+#line 1
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+void main(void) {
+#ifdef WR_FEATURE_TRANSFORM
+    float alpha = 0.0;
+    vec2 pos = init_transform_fs(vLocalPos, vLocalRect, alpha);
+
+    // We clamp the texture coordinate calculation here to the local rectangle boundaries,
+    // which makes the edge of the texture stretch instead of repeat.
+    vec2 relative_pos_in_rect =
+         clamp(pos, vLocalRect.xy, vLocalRect.xy + vLocalRect.zw) - vLocalRect.xy;
+#else
+    float alpha = 1.0;;
+    vec2 relative_pos_in_rect = vLocalPos;
+#endif
+
+    alpha = min(alpha, do_clip());
+
+    vec2 st_y = vTextureOffsetY + relative_pos_in_rect / vStretchSize * vTextureSizeY;
+    vec2 st_u = vTextureOffsetU + relative_pos_in_rect / vStretchSize * vTextureSizeUv;
+    vec2 st_v = vTextureOffsetV + relative_pos_in_rect / vStretchSize * vTextureSizeUv;
+
+    float y = texture(sColor0, st_y).r;
+    float u = texture(sColor1, st_u).r;
+    float v = texture(sColor2, st_v).r;
+
+    // See the vertex shader for an explanation of where the constants come from.
+    vec3 rgb = vYuvColorMatrix * vec3(y - 0.06275, u - 0.50196, v - 0.50196);
+    oFragColor = vec4(rgb, alpha);
+}

webrender/res/ps_yuv_image.glsl

@@ -0,0 +1,19 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+flat varying vec2 vTextureOffsetY; // Offset of the y plane into the texture atlas.
+flat varying vec2 vTextureOffsetU; // Offset of the u plane into the texture atlas.
+flat varying vec2 vTextureOffsetV; // Offset of the v plane into the texture atlas.
+flat varying vec2 vTextureSizeY;   // Size of the y plane in the texture atlas.
+flat varying vec2 vTextureSizeUv;  // Size of the u and v planes in the texture atlas.
+flat varying vec2 vStretchSize;
+
+flat varying mat3 vYuvColorMatrix;
+
+#ifdef WR_FEATURE_TRANSFORM
+varying vec3 vLocalPos;
+flat varying vec4 vLocalRect;
+#else
+varying vec2 vLocalPos;
+#endif

webrender/res/ps_yuv_image.vs.glsl

@@ -0,0 +1,72 @@
+#line 1
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+void main(void) {
+    Primitive prim = load_primitive(gl_InstanceID);
+#ifdef WR_FEATURE_TRANSFORM
+    TransformVertexInfo vi = write_transform_vertex(prim.local_rect,
+                                                    prim.local_clip_rect,
+                                                    prim.layer,
+                                                    prim.tile);
+    vLocalRect = vi.clipped_local_rect;
+    vLocalPos = vi.local_pos;
+#else
+    VertexInfo vi = write_vertex(prim.local_rect,
+                                 prim.local_clip_rect,
+                                 prim.layer,
+                                 prim.tile);
+    vLocalPos = vi.local_clamped_pos - vi.local_rect.p0;
+#endif
+
+    YuvImage image = fetch_yuv_image(prim.prim_index);
+
+    vec2 y_texture_size = vec2(textureSize(sColor0, 0));
+    vec2 y_st0 = image.y_st_rect.xy / y_texture_size;
+    vec2 y_st1 = image.y_st_rect.zw / y_texture_size;
+
+    vTextureSizeY = y_st1 - y_st0;
+    vTextureOffsetY = y_st0;
+
+    vec2 uv_texture_size = vec2(textureSize(sColor1, 0));
+    vec2 u_st0 = image.u_st_rect.xy / uv_texture_size;
+    vec2 u_st1 = image.u_st_rect.zw / uv_texture_size;
+
+    vec2 v_st0 = image.v_st_rect.xy / uv_texture_size;
+    vec2 v_st1 = image.v_st_rect.zw / uv_texture_size;
+
+    // This assumes the U and V surfaces have the same size.
+    vTextureSizeUv = u_st1 - u_st0;
+    vTextureOffsetU = u_st0;
+    vTextureOffsetV = v_st0;
+
+    vStretchSize = image.size;
+
+    // The constants added to the Y, U and V components are applied in the fragment shader.
+    if (image.color_space == YUV_REC601) {
+        // From Rec601:
+        // [R]   [1.1643835616438356,  0.0,                 1.5960267857142858   ]   [Y -  16]
+        // [G] = [1.1643835616438358, -0.3917622900949137, -0.8129676472377708   ] x [U - 128]
+        // [B]   [1.1643835616438356,  2.017232142857143,   8.862867620416422e-17]   [V - 128]
+        //
+        // For the range [0,1] instead of [0,255].
+        vYuvColorMatrix = mat3(
+            1.16438,  0.0,      1.59603,
+            1.16438, -0.39176, -0.81297,
+            1.16438,  2.01723,  0.0
+        );
+    } else { // if (image.color_space == YUV_REC709)
+        // From Rec709:
+        // [R]   [1.1643835616438356,  4.2781193979771426e-17, 1.7927410714285714]   [Y -  16]
+        // [G] = [1.1643835616438358, -0.21324861427372963,   -0.532909328559444 ] x [U - 128]
+        // [B]   [1.1643835616438356,  2.1124017857142854,     0.0               ]   [V - 128]
+        //
+        // For the range [0,1] instead of [0,255]:
+        vYuvColorMatrix = mat3(
+            1.16438,  0.0,      1.79274,
+            1.16438, -0.21325, -0.53291,
+            1.16438,  2.11240,  0.0
+        );
+    }
+}

webrender/src/frame.rs

@@ -714,6 +714,14 @@ impl Frame {
                                               info.image_key,
                                               info.image_rendering);
                 }
+                SpecificDisplayItem::YuvImage(ref info) => {
+                    context.builder.add_yuv_image(LayerRect::from_untyped(&item.rect),
+                                                  &item.clip,
+                                                  info.y_image_key,
+                                                  info.u_image_key,
+                                                  info.v_image_key,
+                                                  info.color_space);
+                }
                 SpecificDisplayItem::Text(ref text_info) => {
                     context.builder.add_text(LayerRect::from_untyped(&item.rect),
                                              &item.clip,

webrender/src/internal_types.rs

@@ -440,4 +440,4 @@ pub enum LowLevelFilterOp {
 pub enum CompositionOp {
     MixBlend(MixBlendMode),
     Filter(LowLevelFilterOp),
-}
+}

webrender/src/prim_store.rs

@@ -12,7 +12,7 @@ use std::mem;
 use std::usize;
 use tiling::RenderTask;
 use util::TransformedRect;
-use webrender_traits::{AuxiliaryLists, ColorF, ImageKey, ImageRendering};
+use webrender_traits::{AuxiliaryLists, ColorF, ImageKey, ImageRendering, YuvColorSpace};
 use webrender_traits::{ClipRegion, ComplexClipRegion, ItemRange, GlyphKey};
 use webrender_traits::{FontKey, FontRenderMode, WebGLContextId};
 use webrender_traits::{device_length, DeviceIntRect, DeviceIntSize};
@@ -73,6 +73,7 @@ pub enum PrimitiveKind {
     Rectangle,
     TextRun,
     Image,
+    YuvImage,
     Border,
     Gradient,
     BoxShadow,
@@ -144,6 +145,46 @@ pub struct ImagePrimitiveGpu {
     pub tile_spacing: LayerSize,
 }
 
+#[derive(Debug)]
+pub struct YuvImagePrimitiveCpu {
+    pub y_key: ImageKey,
+    pub u_key: ImageKey,
+    pub v_key: ImageKey,
+    pub y_texture_id: SourceTexture,
+    pub u_texture_id: SourceTexture,
+    pub v_texture_id: SourceTexture,
+}
+
+#[derive(Debug, Clone)]
+pub struct YuvImagePrimitiveGpu {
+    pub y_uv0: DevicePoint,
+    pub y_uv1: DevicePoint,
+    pub u_uv0: DevicePoint,
+    pub u_uv1: DevicePoint,
+    pub v_uv0: DevicePoint,
+    pub v_uv1: DevicePoint,
+    pub size: LayerSize,
+    pub color_space: f32,
+    pub padding: f32,
+}
+
+impl YuvImagePrimitiveGpu {
+    pub fn new(size: LayerSize, color_space: YuvColorSpace) -> Self {
+        let int_color_space: u8 = unsafe { mem::transmute(color_space) };
+        YuvImagePrimitiveGpu {
+            y_uv0: DevicePoint::zero(),
+            y_uv1: DevicePoint::zero(),
+            u_uv0: DevicePoint::zero(),
+            u_uv1: DevicePoint::zero(),
+            v_uv0: DevicePoint::zero(),
+            v_uv1: DevicePoint::zero(),
+            size: size,
+            color_space: int_color_space as f32,
+            padding: 0.0,
+        }
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct BorderPrimitiveCpu {
     pub inner_rect: LayerRect,
@@ -362,6 +403,7 @@ pub enum PrimitiveContainer {
     Rectangle(RectanglePrimitive),
     TextRun(TextRunPrimitiveCpu, TextRunPrimitiveGpu),
     Image(ImagePrimitiveCpu, ImagePrimitiveGpu),
+    YuvImage(YuvImagePrimitiveCpu, YuvImagePrimitiveGpu),
     Border(BorderPrimitiveCpu, BorderPrimitiveGpu),
     Gradient(GradientPrimitiveCpu, GradientPrimitiveGpu),
     BoxShadow(BoxShadowPrimitiveGpu, Vec<LayerRect>),
@@ -372,6 +414,7 @@ pub struct PrimitiveStore {
     pub cpu_bounding_rects: Vec<Option<DeviceIntRect>>,
     pub cpu_text_runs: Vec<TextRunPrimitiveCpu>,
     pub cpu_images: Vec<ImagePrimitiveCpu>,
+    pub cpu_yuv_images: Vec<YuvImagePrimitiveCpu>,
     pub cpu_gradients: Vec<GradientPrimitiveCpu>,
     pub cpu_metadata: Vec<PrimitiveMetadata>,
     pub cpu_borders: Vec<BorderPrimitiveCpu>,
@@ -398,6 +441,7 @@ impl PrimitiveStore {
             cpu_bounding_rects: Vec::new(),
             cpu_text_runs: Vec::new(),
             cpu_images: Vec::new(),
+            cpu_yuv_images: Vec::new(),
             cpu_gradients: Vec::new(),
             cpu_borders: Vec::new(),
             gpu_geometry: GpuStore::new(),
@@ -487,6 +531,24 @@ impl PrimitiveStore {
                 self.cpu_images.push(image_cpu);
                 metadata
             }
+            PrimitiveContainer::YuvImage(image_cpu, image_gpu) => {
+                let gpu_address = self.gpu_data64.push(image_gpu);
+
+                let metadata = PrimitiveMetadata {
+                    is_opaque: true,
+                    clip_source: clip_source,
+                    clip_cache_info: clip_info,
+                    prim_kind: PrimitiveKind::YuvImage,
+                    cpu_prim_index: SpecificPrimitiveIndex(self.cpu_yuv_images.len()),
+                    gpu_prim_index: gpu_address,
+                    gpu_data_address: GpuStoreAddress(0),
+                    gpu_data_count: 0,
+                    render_task: None,
+                };
+
+                self.cpu_yuv_images.push(image_cpu);
+                metadata
+            }
             PrimitiveContainer::Border(border_cpu, border_gpu) => {
                 let gpu_address = self.gpu_data128.push(border_gpu);
 
@@ -663,6 +725,33 @@ impl PrimitiveStore {
                     }
                     image_cpu.color_texture_id = texture_id;
                 }
+                PrimitiveKind::YuvImage => {
+                    let image_cpu = &mut self.cpu_yuv_images[metadata.cpu_prim_index.0];
+                    let image_gpu: &mut YuvImagePrimitiveGpu = unsafe {
+                        mem::transmute(self.gpu_data64.get_mut(metadata.gpu_prim_index))
+                    };
+
+                    if image_cpu.y_texture_id == SourceTexture::Invalid {
+                        let y_cache_item = resource_cache.get_cached_image(image_cpu.y_key, ImageRendering::Auto);
+                        image_cpu.y_texture_id = y_cache_item.texture_id;
+                        image_gpu.y_uv0 = y_cache_item.uv0;
+                        image_gpu.y_uv1 = y_cache_item.uv1;
+                    }
+
+                    if image_cpu.u_texture_id == SourceTexture::Invalid {
+                        let u_cache_item = resource_cache.get_cached_image(image_cpu.u_key, ImageRendering::Auto);
+                        image_cpu.u_texture_id = u_cache_item.texture_id;
+                        image_gpu.u_uv0 = u_cache_item.uv0;
+                        image_gpu.u_uv1 = u_cache_item.uv1;
+                    }
+
+                    if image_cpu.v_texture_id == SourceTexture::Invalid {
+                        let v_cache_item = resource_cache.get_cached_image(image_cpu.v_key, ImageRendering::Auto);
+                        image_cpu.v_texture_id = v_cache_item.texture_id;
+                        image_gpu.v_uv0 = v_cache_item.uv0;
+                        image_gpu.v_uv1 = v_cache_item.uv1;
+                    }
+                }
             }
         }
 
@@ -851,6 +940,17 @@ impl PrimitiveStore {
                     ImagePrimitiveKind::WebGL(..) => {}
                 }
             }
+            PrimitiveKind::YuvImage => {
+                let image_cpu = &mut self.cpu_yuv_images[metadata.cpu_prim_index.0];
+                prim_needs_resolve = true;
+
+                resource_cache.request_image(image_cpu.y_key, ImageRendering::Auto);
+                resource_cache.request_image(image_cpu.u_key, ImageRendering::Auto);
+                resource_cache.request_image(image_cpu.v_key, ImageRendering::Auto);
+
+                // TODO(nical): Currently assuming no tile_spacing for yuv images.
+                metadata.is_opaque = true;
+            }
             PrimitiveKind::Gradient => {
                 let gradient = &mut self.cpu_gradients[metadata.cpu_prim_index.0];
                 if gradient.cache_dirty {
@@ -973,6 +1073,14 @@ impl From<GradientStop> for GpuBlock32 {
     }
 }
 
+impl From<YuvImagePrimitiveGpu> for GpuBlock64 {
+    fn from(data: YuvImagePrimitiveGpu) -> GpuBlock64 {
+        unsafe {
+            mem::transmute::<YuvImagePrimitiveGpu, GpuBlock64>(data)
+        }
+    }
+}
+
 impl From<ClipRect> for GpuBlock32 {
     fn from(data: ClipRect) -> GpuBlock32 {
         unsafe {