Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Texture support for raw Mesh3D logging #4894

Merged
merged 30 commits into from
Feb 2, 2024
Merged
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
84fd0cb
update fbs files for mesh texture support
Wumpf Jan 23, 2024
85e3991
extension fixup
Wumpf Jan 23, 2024
c401fe7
re_types tests fixup
Wumpf Jan 23, 2024
9742012
codegen
Wumpf Jan 23, 2024
ce87aee
mesh visualizer picks albedo texture now
Wumpf Jan 23, 2024
8c51435
mesh & material python extension update
Wumpf Jan 23, 2024
6e31d15
python raw mesh example now uses new albedo texture support
Wumpf Jan 23, 2024
9aa916b
update rust raw_mesh example
Wumpf Jan 23, 2024
a5fac6a
introduce new texcoord2d component
Wumpf Jan 23, 2024
a76d34f
clarify texture limitation
Wumpf Jan 23, 2024
e53b16f
crude data ui
Wumpf Jan 23, 2024
fb42ed5
spelling exception
Wumpf Jan 23, 2024
d305578
reexport wgpu in external namespace
Wumpf Jan 25, 2024
acf9f66
better texture_key for meshes
Wumpf Jan 25, 2024
bb26200
fbs doc fixes
Wumpf Jan 25, 2024
10d178a
Merge remote-tracking branch 'origin/main' into andreas/log-textured-…
Wumpf Jan 25, 2024
1570e5a
fix missing dependency - did this break on main?
Wumpf Jan 25, 2024
12b9b8d
fix python serialization for no texture present
Wumpf Jan 25, 2024
acd27b7
missing unreleased attribute
Wumpf Jan 25, 2024
fbc140d
missing newline
Wumpf Jan 25, 2024
6b3c097
fix typo in texcoord2d cpp
Wumpf Jan 25, 2024
6693722
fix and update mesh3d python tests
Wumpf Jan 25, 2024
92bf597
Merge remote-tracking branch 'origin/main' into andreas/log-textured-…
Wumpf Jan 30, 2024
9b0107b
improvements to rgba32 rust extensions
Wumpf Feb 1, 2024
e5f1d37
revert material, put albedo texture on the mesh archetype instead
Wumpf Feb 1, 2024
b630dfd
fix c++ build
Wumpf Feb 1, 2024
3ae6938
fix typo that I didn't cause
Wumpf Feb 1, 2024
4229db9
fix too dark mesh - don't convert the trimesh material if you don't h…
Wumpf Feb 1, 2024
d89fed9
fix python raw_mesh example not picking up albedo factor
teh-cmc Feb 2, 2024
cf2f654
review
teh-cmc Feb 2, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions clippy.toml
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ doc-valid-idents = [
"macOS",
"NaN",
"OBJ",
"OpenGL",
"PyPI",
"sRGB",
"sRGBA",
Expand Down
36 changes: 1 addition & 35 deletions crates/re_data_ui/src/data.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
use egui::Vec2;

use re_format::format_f32;
use re_types::components::{
Color, LineStrip2D, LineStrip3D, Material, MeshProperties, ViewCoordinates,
};
use re_types::components::{Color, LineStrip2D, LineStrip3D, MeshProperties, ViewCoordinates};
use re_viewer_context::{UiVerbosity, ViewerContext};

use super::{table_for_verbosity, DataUi};
Expand Down Expand Up @@ -227,38 +225,6 @@ impl DataUi for LineStrip3D {
}
}

impl DataUi for Material {
fn data_ui(
&self,
ctx: &ViewerContext<'_>,
ui: &mut egui::Ui,
verbosity: UiVerbosity,
query: &re_data_store::LatestAtQuery,
store: &re_data_store::DataStore,
) {
let show_optional_albedo_factor = |ui: &mut egui::Ui| {
if let Some(albedo_factor) = self.albedo_factor {
Color(albedo_factor).data_ui(ctx, ui, verbosity, query, store);
} else {
ui.weak("(empty)");
}
};

match verbosity {
UiVerbosity::Small | UiVerbosity::Reduced => {
show_optional_albedo_factor(ui);
}
UiVerbosity::Full | UiVerbosity::LimitHeight => {
egui::Grid::new("material").num_columns(2).show(ui, |ui| {
ui.label("albedo_factor");
show_optional_albedo_factor(ui);
ui.end_row();
});
}
}
}
}

impl DataUi for MeshProperties {
fn data_ui(
&self,
Expand Down
2 changes: 1 addition & 1 deletion crates/re_data_ui/src/image.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ impl EntityDataUi for re_types::components::TensorData {
}

#[allow(clippy::too_many_arguments)]
fn tensor_ui(
pub fn tensor_ui(
ctx: &ViewerContext<'_>,
query: &re_data_store::LatestAtQuery,
store: &re_data_store::DataStore,
Expand Down
4 changes: 3 additions & 1 deletion crates/re_data_ui/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,15 @@ mod entity_path;
mod image;
mod image_meaning;
mod instance_path;
pub mod item_ui;
mod log_msg;
mod material;
mod pinhole;
mod rotation3d;
mod store_id;
mod transform3d;

pub mod item_ui;

pub use crate::image::{
show_zoomed_image_region, show_zoomed_image_region_area_outline,
tensor_summary_ui_grid_contents,
Expand Down
53 changes: 53 additions & 0 deletions crates/re_data_ui/src/material.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
use re_types::components::{Color, Material};
use re_viewer_context::{UiVerbosity, ViewerContext};

use crate::DataUi;

impl DataUi for Material {
fn data_ui(
&self,
ctx: &ViewerContext<'_>,
ui: &mut egui::Ui,
verbosity: UiVerbosity,
query: &re_data_store::LatestAtQuery,
store: &re_data_store::DataStore,
) {
let show_optional_albedo_factor = |ui: &mut egui::Ui| {
if let Some(albedo_factor) = self.albedo_factor {
Color(albedo_factor).data_ui(ctx, ui, verbosity, query, store);
} else {
ui.weak("(empty)");
}
};
let show_optional_albedo_texture = |ui: &mut egui::Ui| {
if let Some(albedo_texture) = &self.albedo_texture {
// TODO(andreas): Either we rethink this so that albedo texture is a regular image,
// or we pluck apart the image preview code to be re-usable here,
// so that we can show an image preview.
Wumpf marked this conversation as resolved.
Show resolved Hide resolved
if let Some([height, width, _]) = albedo_texture.image_height_width_channels() {
ui.weak(format!("[{height} x {width}]"));
} else {
ui.weak("(not an image)");
}
} else {
ui.weak("(empty)");
}
};

match verbosity {
UiVerbosity::Small | UiVerbosity::Reduced => {
show_optional_albedo_factor(ui);
}
UiVerbosity::Full | UiVerbosity::LimitHeight => {
egui::Grid::new("material").num_columns(2).show(ui, |ui| {
ui.label("albedo_factor");
show_optional_albedo_factor(ui);
ui.end_row();
ui.label("albedo_texture");
show_optional_albedo_texture(ui);
ui.end_row();
});
}
}
}
}
6 changes: 5 additions & 1 deletion crates/re_renderer/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,13 @@ pub use self::file_resolver::{
};
pub use self::file_server::FileServer;

// Re-export used color types.
// Re-export used color types directly.
pub use ecolor::{Color32, Hsva, Rgba};

pub mod external {
pub use wgpu;
}

// ---------------------------------------------------------------------------

// Make Arrow integration as transparent as possible.
Expand Down
9 changes: 7 additions & 2 deletions crates/re_space_view_spatial/src/mesh_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use crate::mesh_loader::LoadedMesh;

// ----------------------------------------------------------------------------

#[derive(Debug, PartialEq, Eq, Hash)]
#[derive(Debug, PartialEq, Eq, Hash, Clone)]
pub struct MeshCacheKey {
pub versioned_instance_path_hash: VersionedInstancePathHash,
pub media_type: Option<MediaType>,
Expand All @@ -24,7 +24,12 @@ pub struct MeshCache(ahash::HashMap<MeshCacheKey, Option<Arc<LoadedMesh>>>);
#[derive(Debug, Clone, Copy)]
pub enum AnyMesh<'a> {
Asset(&'a re_types::archetypes::Asset3D),
Mesh(&'a re_types::archetypes::Mesh3D),
Mesh {
mesh: &'a re_types::archetypes::Mesh3D,

/// If any textures are in the mesh's material, they use this hash for texture manager lookup.
teh-cmc marked this conversation as resolved.
Show resolved Hide resolved
texture_key: u64,
},
}

impl MeshCache {
Expand Down
64 changes: 57 additions & 7 deletions crates/re_space_view_spatial/src/mesh_loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use re_renderer::{resource_managers::ResourceLifeTime, RenderContext, Rgba32Unmu
use re_types::{
archetypes::{Asset3D, Mesh3D},
components::MediaType,
datatypes::TensorBuffer,
};

use crate::mesh_cache::AnyMesh;
Expand All @@ -26,7 +27,9 @@ impl LoadedMesh {
// TODO(emilk): load CpuMesh in background thread.
match mesh {
AnyMesh::Asset(asset3d) => Self::load_asset3d(name, asset3d, render_ctx),
AnyMesh::Mesh(mesh3d) => Ok(Self::load_mesh3d(name, mesh3d, render_ctx)?),
AnyMesh::Mesh { mesh, texture_key } => {
Ok(Self::load_mesh3d(name, mesh, texture_key, render_ctx)?)
}
}
}

Expand Down Expand Up @@ -85,6 +88,7 @@ impl LoadedMesh {
fn load_mesh3d(
name: String,
mesh3d: &Mesh3D,
texture_key: u64,
render_ctx: &RenderContext,
) -> anyhow::Result<Self> {
re_tracing::profile_function!();
Expand All @@ -94,6 +98,7 @@ impl LoadedMesh {
mesh_properties,
vertex_normals,
vertex_colors,
vertex_texcoords,
mesh_material,
class_ids: _,
instance_keys: _,
Expand Down Expand Up @@ -135,18 +140,35 @@ impl LoadedMesh {
normals.iter().map(|v| v.0.into()).collect::<Vec<_>>()
} else {
// TODO(andreas): Calculate normals
// TODO(cmc): support textured raw meshes
vec![glam::Vec3::ZERO; num_positions]
};

let vertex_texcoords = vec![glam::Vec2::ZERO; vertex_normals.len()];
let vertex_texcoords = if let Some(texcoords) = vertex_texcoords {
re_tracing::profile_scope!("collect_texcoords");
texcoords.iter().map(|v| v.0.into()).collect::<Vec<_>>()
} else {
vec![glam::Vec2::ZERO; num_positions]
};

let albedo_factor = mesh_material.as_ref().and_then(|mat| mat.albedo_factor);

let bbox = {
re_tracing::profile_scope!("bbox");
macaw::BoundingBox::from_points(vertex_positions.iter().copied())
};

let albedo = if let Some(albedo_texture) = mesh_material
.as_ref()
.and_then(|mat| mat.albedo_texture.as_ref())
{
mesh_texture_from_tensor_data(albedo_texture, render_ctx, texture_key)?
} else {
render_ctx
.texture_manager_2d
.white_texture_unorm_handle()
.clone()
};

let mesh = re_renderer::mesh::Mesh {
label: name.clone().into(),
triangle_indices,
Expand All @@ -157,10 +179,7 @@ impl LoadedMesh {
materials: smallvec::smallvec![re_renderer::mesh::Material {
label: name.clone().into(),
index_range: 0..num_indices as _,
albedo: render_ctx
.texture_manager_2d
.white_texture_unorm_handle()
.clone(),
albedo,
albedo_multiplier: albedo_factor.map_or(re_renderer::Rgba::WHITE, |c| c.into()),
}],
};
Expand Down Expand Up @@ -190,3 +209,34 @@ impl LoadedMesh {
self.bbox
}
}

fn mesh_texture_from_tensor_data(
albedo_texture: &re_types::datatypes::TensorData,
render_ctx: &RenderContext,
texture_key: u64,
) -> anyhow::Result<re_renderer::resource_managers::GpuTexture2D> {
let [height, width, depth] =
re_viewer_context::gpu_bridge::texture_height_width_channels(albedo_texture)?;

re_viewer_context::gpu_bridge::try_get_or_create_texture(render_ctx, texture_key, || {
let data = match (depth, &albedo_texture.buffer) {
(3, TensorBuffer::U8(buf)) => re_renderer::pad_rgb_to_rgba(buf, u8::MAX).into(),
(4, TensorBuffer::U8(buf)) => bytemuck::cast_slice(buf.as_slice()).into(),

_ => {
anyhow::bail!(
"Only 3 and 4 channel u8 tensor data is supported currently for mesh textures."
);
}
};

Ok(re_renderer::resource_managers::Texture2DCreationDesc {
label: "mesh albedo texture from tensor data".into(),
data,
format: re_renderer::external::wgpu::TextureFormat::Rgba8UnormSrgb,
Wumpf marked this conversation as resolved.
Show resolved Hide resolved
width,
height,
})
})
.map_err(|err| anyhow::format_err!("{err}"))
}
25 changes: 20 additions & 5 deletions crates/re_space_view_spatial/src/visualizers/meshes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use re_query::{ArchetypeView, QueryError};
use re_renderer::renderer::MeshInstance;
use re_types::{
archetypes::Mesh3D,
components::{Color, InstanceKey, Material, MeshProperties, Position3D, Vector3D},
components::{Color, InstanceKey, Material, MeshProperties, Position3D, Texcoord2D, Vector3D},
Archetype, ComponentNameSet,
};
use re_viewer_context::{
Expand Down Expand Up @@ -81,6 +81,17 @@ impl Mesh3DVisualizer {
} else {
None
},
vertex_texcoords: if arch_view.has_component::<Texcoord2D>() {
re_tracing::profile_scope!("vertex_texcoords");
Some(
arch_view
.iter_optional_component::<Texcoord2D>()?
.map(|comp| comp.unwrap_or(Texcoord2D::ZERO))
.collect(),
)
} else {
None
},
mesh_properties: arch_view.raw_optional_mono_component::<MeshProperties>()?,
mesh_material: arch_view.raw_optional_mono_component::<Material>()?,
class_ids: None,
Expand All @@ -93,13 +104,17 @@ impl Mesh3DVisualizer {
let outline_mask_ids = ent_context.highlight.index_outline_mask(InstanceKey::SPLAT);

let mesh = ctx.cache.entry(|c: &mut MeshCache| {
let key = MeshCacheKey {
versioned_instance_path_hash: picking_instance_hash.versioned(primary_row_id),
media_type: None,
};
c.entry(
&ent_path.to_string(),
MeshCacheKey {
versioned_instance_path_hash: picking_instance_hash.versioned(primary_row_id),
media_type: None,
key.clone(),
AnyMesh::Mesh {
mesh: &mesh,
texture_key: re_log_types::hash::Hash64::hash(&key).hash64(),
},
AnyMesh::Mesh(&mesh),
ctx.render_ctx,
)
});
Expand Down
3 changes: 2 additions & 1 deletion crates/re_space_view_text_log/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ all-features = true
re_data_store.workspace = true
re_data_ui.workspace = true
re_entity_db.workspace = true
re_log.workspace = true
re_log_types.workspace = true
re_log.workspace = true
re_query_cache.workspace = true
re_renderer.workspace = true
re_tracing.workspace = true
re_types.workspace = true
Expand Down
9 changes: 6 additions & 3 deletions crates/re_types/definitions/rerun/archetypes/mesh3d.fbs
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,17 @@ table Mesh3D (
/// An optional color for each vertex.
vertex_colors: [rerun.components.Color] ("attr.rerun.component_optional", nullable, order: 3100);

/// An optional uv texture coordinate for each vertex.
vertex_texcoords: [rerun.components.Texcoord2D] ("attr.rerun.component_optional", nullable, order: 3200);

/// Optional material properties for the mesh as a whole.
mesh_material: rerun.components.Material ("attr.rerun.component_optional", nullable, order: 3200);
mesh_material: rerun.components.Material ("attr.rerun.component_optional", nullable, order: 3300);

/// Optional class Ids for the vertices.
///
/// The class ID provides colors and labels if not specified explicitly.
class_ids: [rerun.components.ClassId] ("attr.rerun.component_optional", nullable, order: 3300);
class_ids: [rerun.components.ClassId] ("attr.rerun.component_optional", nullable, order: 3400);

/// Unique identifiers for each individual vertex in the mesh.
instance_keys: [rerun.components.InstanceKey] ("attr.rerun.component_optional", nullable, order: 3400);
instance_keys: [rerun.components.InstanceKey] ("attr.rerun.component_optional", nullable, order: 3500);
}
1 change: 1 addition & 0 deletions crates/re_types/definitions/rerun/components.fbs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ include "./components/rotation3d.fbs";
include "./components/scalar_scattering.fbs";
include "./components/scalar.fbs";
include "./components/tensor_data.fbs";
include "./components/texcoord2d.fbs";
include "./components/text_log_level.fbs";
include "./components/text.fbs";
include "./components/transform3d.fbs";
Expand Down
Loading
Loading