Texture support for raw Mesh3D logging

clippy.toml

@@ -75,6 +75,7 @@ doc-valid-idents = [
   "macOS",
   "NaN",
   "OBJ",
+  "OpenGL",
   "PyPI",
   "sRGB",
   "sRGBA",

crates/re_data_ui/src/data.rs

@@ -1,9 +1,7 @@
 use egui::Vec2;
 
 use re_format::format_f32;
-use re_types::components::{
-    Color, LineStrip2D, LineStrip3D, Material, MeshProperties, ViewCoordinates,
-};
+use re_types::components::{Color, LineStrip2D, LineStrip3D, MeshProperties, ViewCoordinates};
 use re_viewer_context::{UiVerbosity, ViewerContext};
 
 use super::{table_for_verbosity, DataUi};
@@ -227,38 +225,6 @@ impl DataUi for LineStrip3D {
     }
 }
 
-impl DataUi for Material {
-    fn data_ui(
-        &self,
-        ctx: &ViewerContext<'_>,
-        ui: &mut egui::Ui,
-        verbosity: UiVerbosity,
-        query: &re_data_store::LatestAtQuery,
-        store: &re_data_store::DataStore,
-    ) {
-        let show_optional_albedo_factor = |ui: &mut egui::Ui| {
-            if let Some(albedo_factor) = self.albedo_factor {
-                Color(albedo_factor).data_ui(ctx, ui, verbosity, query, store);
-            } else {
-                ui.weak("(empty)");
-            }
-        };
-
-        match verbosity {
-            UiVerbosity::Small | UiVerbosity::Reduced => {
-                show_optional_albedo_factor(ui);
-            }
-            UiVerbosity::Full | UiVerbosity::LimitHeight => {
-                egui::Grid::new("material").num_columns(2).show(ui, |ui| {
-                    ui.label("albedo_factor");
-                    show_optional_albedo_factor(ui);
-                    ui.end_row();
-                });
-            }
-        }
-    }
-}
-
 impl DataUi for MeshProperties {
     fn data_ui(
         &self,

crates/re_data_ui/src/image.rs

@@ -65,7 +65,7 @@ impl EntityDataUi for re_types::components::TensorData {
 }
 
 #[allow(clippy::too_many_arguments)]
-fn tensor_ui(
+pub fn tensor_ui(
     ctx: &ViewerContext<'_>,
     query: &re_data_store::LatestAtQuery,
     store: &re_data_store::DataStore,

crates/re_data_ui/src/lib.rs

@@ -19,13 +19,15 @@ mod entity_path;
 mod image;
 mod image_meaning;
 mod instance_path;
-pub mod item_ui;
 mod log_msg;
+mod material;
 mod pinhole;
 mod rotation3d;
 mod store_id;
 mod transform3d;
 
+pub mod item_ui;
+
 pub use crate::image::{
     show_zoomed_image_region, show_zoomed_image_region_area_outline,
     tensor_summary_ui_grid_contents,

crates/re_data_ui/src/material.rs

@@ -0,0 +1,53 @@
+use re_types::components::{Color, Material};
+use re_viewer_context::{UiVerbosity, ViewerContext};
+
+use crate::DataUi;
+
+impl DataUi for Material {
+    fn data_ui(
+        &self,
+        ctx: &ViewerContext<'_>,
+        ui: &mut egui::Ui,
+        verbosity: UiVerbosity,
+        query: &re_data_store::LatestAtQuery,
+        store: &re_data_store::DataStore,
+    ) {
+        let show_optional_albedo_factor = |ui: &mut egui::Ui| {
+            if let Some(albedo_factor) = self.albedo_factor {
+                Color(albedo_factor).data_ui(ctx, ui, verbosity, query, store);
+            } else {
+                ui.weak("(empty)");
+            }
+        };
+        let show_optional_albedo_texture = |ui: &mut egui::Ui| {
+            if let Some(albedo_texture) = &self.albedo_texture {
+                // TODO(andreas): Either we rethink this so that albedo texture is a regular image,
+                //                or we pluck apart the image preview code to be re-usable here,
+                //                so that we can show an image preview.
+                if let Some([height, width, _]) = albedo_texture.image_height_width_channels() {
+                    ui.weak(format!("[{height} x {width}]"));
+                } else {
+                    ui.weak("(not an image)");
+                }
+            } else {
+                ui.weak("(empty)");
+            }
+        };
+
+        match verbosity {
+            UiVerbosity::Small | UiVerbosity::Reduced => {
+                show_optional_albedo_factor(ui);
+            }
+            UiVerbosity::Full | UiVerbosity::LimitHeight => {
+                egui::Grid::new("material").num_columns(2).show(ui, |ui| {
+                    ui.label("albedo_factor");
+                    show_optional_albedo_factor(ui);
+                    ui.end_row();
+                    ui.label("albedo_texture");
+                    show_optional_albedo_texture(ui);
+                    ui.end_row();
+                });
+            }
+        }
+    }
+}

crates/re_renderer/src/lib.rs

@@ -79,9 +79,13 @@ pub use self::file_resolver::{
 };
 pub use self::file_server::FileServer;
 
-// Re-export used color types.
+// Re-export used color types directly.
 pub use ecolor::{Color32, Hsva, Rgba};
 
+pub mod external {
+    pub use wgpu;
+}
+
 // ---------------------------------------------------------------------------
 
 // Make Arrow integration as transparent as possible.

crates/re_space_view_spatial/src/mesh_cache.rs

@@ -9,7 +9,7 @@ use crate::mesh_loader::LoadedMesh;
 
 // ----------------------------------------------------------------------------
 
-#[derive(Debug, PartialEq, Eq, Hash)]
+#[derive(Debug, PartialEq, Eq, Hash, Clone)]
 pub struct MeshCacheKey {
     pub versioned_instance_path_hash: VersionedInstancePathHash,
     pub media_type: Option<MediaType>,
@@ -24,7 +24,12 @@ pub struct MeshCache(ahash::HashMap<MeshCacheKey, Option<Arc<LoadedMesh>>>);
 #[derive(Debug, Clone, Copy)]
 pub enum AnyMesh<'a> {
     Asset(&'a re_types::archetypes::Asset3D),
-    Mesh(&'a re_types::archetypes::Mesh3D),
+    Mesh {
+        mesh: &'a re_types::archetypes::Mesh3D,
+
+        /// If any textures are in the mesh's material, they use this hash for texture manager lookup.
+        texture_key: u64,
+    },
 }
 
 impl MeshCache {

crates/re_space_view_spatial/src/mesh_loader.rs

@@ -3,6 +3,7 @@ use re_renderer::{resource_managers::ResourceLifeTime, RenderContext, Rgba32Unmu
 use re_types::{
     archetypes::{Asset3D, Mesh3D},
     components::MediaType,
+    datatypes::TensorBuffer,
 };
 
 use crate::mesh_cache::AnyMesh;
@@ -26,7 +27,9 @@ impl LoadedMesh {
         // TODO(emilk): load CpuMesh in background thread.
         match mesh {
             AnyMesh::Asset(asset3d) => Self::load_asset3d(name, asset3d, render_ctx),
-            AnyMesh::Mesh(mesh3d) => Ok(Self::load_mesh3d(name, mesh3d, render_ctx)?),
+            AnyMesh::Mesh { mesh, texture_key } => {
+                Ok(Self::load_mesh3d(name, mesh, texture_key, render_ctx)?)
+            }
         }
     }
 
@@ -85,6 +88,7 @@ impl LoadedMesh {
     fn load_mesh3d(
         name: String,
         mesh3d: &Mesh3D,
+        texture_key: u64,
         render_ctx: &RenderContext,
     ) -> anyhow::Result<Self> {
         re_tracing::profile_function!();
@@ -94,6 +98,7 @@ impl LoadedMesh {
             mesh_properties,
             vertex_normals,
             vertex_colors,
+            vertex_texcoords,
             mesh_material,
             class_ids: _,
             instance_keys: _,
@@ -135,18 +140,35 @@ impl LoadedMesh {
             normals.iter().map(|v| v.0.into()).collect::<Vec<_>>()
         } else {
             // TODO(andreas): Calculate normals
-            // TODO(cmc): support textured raw meshes
             vec![glam::Vec3::ZERO; num_positions]
         };
 
-        let vertex_texcoords = vec![glam::Vec2::ZERO; vertex_normals.len()];
+        let vertex_texcoords = if let Some(texcoords) = vertex_texcoords {
+            re_tracing::profile_scope!("collect_texcoords");
+            texcoords.iter().map(|v| v.0.into()).collect::<Vec<_>>()
+        } else {
+            vec![glam::Vec2::ZERO; num_positions]
+        };
+
         let albedo_factor = mesh_material.as_ref().and_then(|mat| mat.albedo_factor);
 
         let bbox = {
             re_tracing::profile_scope!("bbox");
             macaw::BoundingBox::from_points(vertex_positions.iter().copied())
         };
 
+        let albedo = if let Some(albedo_texture) = mesh_material
+            .as_ref()
+            .and_then(|mat| mat.albedo_texture.as_ref())
+        {
+            mesh_texture_from_tensor_data(albedo_texture, render_ctx, texture_key)?
+        } else {
+            render_ctx
+                .texture_manager_2d
+                .white_texture_unorm_handle()
+                .clone()
+        };
+
         let mesh = re_renderer::mesh::Mesh {
             label: name.clone().into(),
             triangle_indices,
@@ -157,10 +179,7 @@ impl LoadedMesh {
             materials: smallvec::smallvec![re_renderer::mesh::Material {
                 label: name.clone().into(),
                 index_range: 0..num_indices as _,
-                albedo: render_ctx
-                    .texture_manager_2d
-                    .white_texture_unorm_handle()
-                    .clone(),
+                albedo,
                 albedo_multiplier: albedo_factor.map_or(re_renderer::Rgba::WHITE, |c| c.into()),
             }],
         };
@@ -190,3 +209,34 @@ impl LoadedMesh {
         self.bbox
     }
 }
+
+fn mesh_texture_from_tensor_data(
+    albedo_texture: &re_types::datatypes::TensorData,
+    render_ctx: &RenderContext,
+    texture_key: u64,
+) -> anyhow::Result<re_renderer::resource_managers::GpuTexture2D> {
+    let [height, width, depth] =
+        re_viewer_context::gpu_bridge::texture_height_width_channels(albedo_texture)?;
+
+    re_viewer_context::gpu_bridge::try_get_or_create_texture(render_ctx, texture_key, || {
+        let data = match (depth, &albedo_texture.buffer) {
+            (3, TensorBuffer::U8(buf)) => re_renderer::pad_rgb_to_rgba(buf, u8::MAX).into(),
+            (4, TensorBuffer::U8(buf)) => bytemuck::cast_slice(buf.as_slice()).into(),
+
+            _ => {
+                anyhow::bail!(
+                    "Only 3 and 4 channel u8 tensor data is supported currently for mesh textures."
+                );
+            }
+        };
+
+        Ok(re_renderer::resource_managers::Texture2DCreationDesc {
+            label: "mesh albedo texture from tensor data".into(),
+            data,
+            format: re_renderer::external::wgpu::TextureFormat::Rgba8UnormSrgb,
+            width,
+            height,
+        })
+    })
+    .map_err(|err| anyhow::format_err!("{err}"))
+}

crates/re_space_view_spatial/src/visualizers/meshes.rs

@@ -3,7 +3,7 @@ use re_query::{ArchetypeView, QueryError};
 use re_renderer::renderer::MeshInstance;
 use re_types::{
     archetypes::Mesh3D,
-    components::{Color, InstanceKey, Material, MeshProperties, Position3D, Vector3D},
+    components::{Color, InstanceKey, Material, MeshProperties, Position3D, Texcoord2D, Vector3D},
     Archetype, ComponentNameSet,
 };
 use re_viewer_context::{
@@ -81,6 +81,17 @@ impl Mesh3DVisualizer {
                 } else {
                     None
                 },
+                vertex_texcoords: if arch_view.has_component::<Texcoord2D>() {
+                    re_tracing::profile_scope!("vertex_texcoords");
+                    Some(
+                        arch_view
+                            .iter_optional_component::<Texcoord2D>()?
+                            .map(|comp| comp.unwrap_or(Texcoord2D::ZERO))
+                            .collect(),
+                    )
+                } else {
+                    None
+                },
                 mesh_properties: arch_view.raw_optional_mono_component::<MeshProperties>()?,
                 mesh_material: arch_view.raw_optional_mono_component::<Material>()?,
                 class_ids: None,
@@ -93,13 +104,17 @@ impl Mesh3DVisualizer {
         let outline_mask_ids = ent_context.highlight.index_outline_mask(InstanceKey::SPLAT);
 
         let mesh = ctx.cache.entry(|c: &mut MeshCache| {
+            let key = MeshCacheKey {
+                versioned_instance_path_hash: picking_instance_hash.versioned(primary_row_id),
+                media_type: None,
+            };
             c.entry(
                 &ent_path.to_string(),
-                MeshCacheKey {
-                    versioned_instance_path_hash: picking_instance_hash.versioned(primary_row_id),
-                    media_type: None,
+                key.clone(),
+                AnyMesh::Mesh {
+                    mesh: &mesh,
+                    texture_key: re_log_types::hash::Hash64::hash(&key).hash64(),
                 },
-                AnyMesh::Mesh(&mesh),
                 ctx.render_ctx,
             )
         });

crates/re_space_view_text_log/Cargo.toml

@@ -19,8 +19,9 @@ all-features = true
 re_data_store.workspace = true
 re_data_ui.workspace = true
 re_entity_db.workspace = true
-re_log.workspace = true
 re_log_types.workspace = true
+re_log.workspace = true
+re_query_cache.workspace = true
 re_renderer.workspace = true
 re_tracing.workspace = true
 re_types.workspace = true

crates/re_types/definitions/rerun/archetypes/mesh3d.fbs

@@ -41,14 +41,17 @@ table Mesh3D (
   /// An optional color for each vertex.
   vertex_colors: [rerun.components.Color] ("attr.rerun.component_optional", nullable, order: 3100);
 
+  /// An optional uv texture coordinate for each vertex.
+  vertex_texcoords: [rerun.components.Texcoord2D] ("attr.rerun.component_optional", nullable, order: 3200);
+
   /// Optional material properties for the mesh as a whole.
-  mesh_material: rerun.components.Material ("attr.rerun.component_optional", nullable, order: 3200);
+  mesh_material: rerun.components.Material ("attr.rerun.component_optional", nullable, order: 3300);
 
   /// Optional class Ids for the vertices.
   ///
   /// The class ID provides colors and labels if not specified explicitly.
-  class_ids: [rerun.components.ClassId] ("attr.rerun.component_optional", nullable, order: 3300);
+  class_ids: [rerun.components.ClassId] ("attr.rerun.component_optional", nullable, order: 3400);
 
   /// Unique identifiers for each individual vertex in the mesh.
-  instance_keys: [rerun.components.InstanceKey] ("attr.rerun.component_optional", nullable, order: 3400);
+  instance_keys: [rerun.components.InstanceKey] ("attr.rerun.component_optional", nullable, order: 3500);
 }

crates/re_types/definitions/rerun/components.fbs

@@ -25,6 +25,7 @@ include "./components/rotation3d.fbs";
 include "./components/scalar_scattering.fbs";
 include "./components/scalar.fbs";
 include "./components/tensor_data.fbs";
+include "./components/texcoord2d.fbs";
 include "./components/text_log_level.fbs";
 include "./components/text.fbs";
 include "./components/transform3d.fbs";