Unify spatial picking & fix "3d in 2d" picking (#663)

* 3d picking ui is now almost identical to 2d picking ui
* picking can now always handle transparent objects
* move picking to its own module
* fix 2d labels not showing
* 2d boxes are no longer hovered like rects

crates/re_viewer/src/math.rs

@@ -28,6 +28,21 @@ pub fn line_segment_distance_to_point_3d([a, b]: [glam::Vec3; 2], p: glam::Vec3)
     line_segment_distance_sq_to_point_3d([a, b], p).sqrt()
 }
 
+/// Compute the distance between a ray and a line segment.
+///
+/// Returns the ray offset at which the ray is closest to the line segment.
+/// (i.e. the closest point then is at `ray.origin + ray.dir * ray_closest_t_line_segment(...)`)
+pub fn ray_closest_t_line_segment(ray: &macaw::Ray3, [a, b]: [glam::Vec3; 2]) -> f32 {
+    let (t_ray, t_segment) = ray.closest_ts(&macaw::Ray3::from_origin_dir(a, b - a));
+    if t_ray.is_nan() || t_segment < 0.0 {
+        ray.closest_t_to_point(a)
+    } else if t_segment > 1.0 {
+        ray.closest_t_to_point(b)
+    } else {
+        t_ray
+    }
+}
+
 /// Returns the distance the ray traveled of the first intersection or `f32::INFINITY` on miss.
 pub fn ray_bbox_intersect(ray: &macaw::Ray3, bbox: &macaw::BoundingBox) -> f32 {
     // from https://gamedev.stackexchange.com/a/18459
@@ -59,3 +74,51 @@ fn max(a: f32, b: f32) -> f32 {
 pub fn ease_out(t: f32) -> f32 {
     1. - (1. - t) * (1. - t)
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::math::ray_closest_t_line_segment;
+    use cgmath::assert_ulps_eq;
+    use glam::vec3;
+
+    #[test]
+    fn test_ray_closest_t_line_segment() {
+        let ray_x = macaw::Ray3::from_origin_dir(glam::Vec3::ZERO, glam::Vec3::X);
+
+        // through origin.
+        assert_ulps_eq!(
+            ray_closest_t_line_segment(&ray_x, [vec3(0.0, 0.0, 1.0), vec3(0.0, 0.0, -1.0)]),
+            0.0
+        );
+
+        // behind origin, orthogonal to ray.
+        assert_ulps_eq!(
+            ray_closest_t_line_segment(&ray_x, [vec3(-1.0, 0.0, 1.0), vec3(-1.0, 0.0, -1.0)]),
+            -1.0
+        );
+
+        // in front of origin, orthogonal to ray.
+        assert_ulps_eq!(
+            ray_closest_t_line_segment(&ray_x, [vec3(1.0, 0.0, 1.0), vec3(1.0, 0.0, -1.0)]),
+            1.0
+        );
+
+        // parallel to ray, half way in front
+        assert_ulps_eq!(
+            ray_closest_t_line_segment(&ray_x, [vec3(0.5, 1.0, 0.0), vec3(1.5, 1.0, 0.0)]),
+            0.5
+        );
+
+        // parallel to ray, half way behind
+        assert_ulps_eq!(
+            ray_closest_t_line_segment(&ray_x, [vec3(-0.5, 1.0, 0.0), vec3(-1.5, 1.0, 0.0)]),
+            -0.5
+        );
+
+        // Degenerated line segment at origin.
+        assert_ulps_eq!(
+            ray_closest_t_line_segment(&ray_x, [vec3(0.0, 0.0, 0.0), vec3(0.0, 0.0, 0.0)]),
+            0.0
+        );
+    }
+}

crates/re_viewer/src/misc/viewer_context.rs

@@ -1,5 +1,3 @@
-use macaw::Ray3;
-
 use re_data_store::{log_db::LogDb, InstanceId, ObjTypePath};
 use re_log_types::{DataPath, MsgId, ObjPath, TimeInt, Timeline};
 
@@ -238,7 +236,7 @@ pub enum HoveredSpace {
         space_3d: ObjPath,
 
         /// 2D spaces and pixel coordinates (with Z=depth)
-        target_spaces: Vec<(ObjPath, Option<Ray3>, Option<glam::Vec3>)>,
+        target_spaces: Vec<(ObjPath, Option<glam::Vec3>)>,
     },
 }
 

crates/re_viewer/src/ui/data_ui/image.rs

@@ -71,79 +71,89 @@ fn show_zoomed_image_region_tooltip(
         .on_hover_cursor(egui::CursorIcon::ZoomIn)
         .on_hover_ui_at_pointer(|ui| {
             ui.horizontal(|ui| {
-                show_zoomed_image_region(
-                    parent_ui,
-                    ui,
-                    tensor_view,
-                    image_rect,
-                    pointer_pos,
-                    meter,
-                );
+                let Some(dynamic_img) = tensor_view.dynamic_img else { return };
+                let w = dynamic_img.width() as _;
+                let h = dynamic_img.height() as _;
+
+                use egui::NumExt;
+
+                let center = [
+                    (egui::remap(pointer_pos.x, image_rect.x_range(), 0.0..=w as f32) as isize)
+                        .at_most(w),
+                    (egui::remap(pointer_pos.y, image_rect.y_range(), 0.0..=h as f32) as isize)
+                        .at_most(h),
+                ];
+                show_zoomed_image_region_area_outline(parent_ui, tensor_view, center, image_rect);
+                show_zoomed_image_region(ui, tensor_view, center, meter);
             });
         })
 }
 
-/// meter: iff this is a depth map, how long is one meter?
-pub(crate) fn show_zoomed_image_region(
-    parent_ui: &mut egui::Ui,
-    tooltip_ui: &mut egui::Ui,
+// Show the surrounding pixels:
+const ZOOMED_IMAGE_TEXEL_RADIUS: isize = 12;
+
+pub fn show_zoomed_image_region_area_outline(
+    ui: &mut egui::Ui,
     tensor_view: &TensorImageView<'_, '_>,
+    [center_x, center_y]: [isize; 2],
     image_rect: egui::Rect,
-    pointer_pos: egui::Pos2,
-    meter: Option<f32>,
 ) {
     let Some(dynamic_img) = tensor_view.dynamic_img else { return };
 
-    use egui::{color_picker, pos2, remap, Color32, Mesh, NumExt, Rect, Vec2};
+    use egui::{pos2, remap, Color32, Rect};
 
-    // Show the surrounding pixels:
-    let texel_radius = 12;
-    let size = Vec2::splat(128.0);
-
-    let (_id, zoom_rect) = tooltip_ui.allocate_space(size);
     let w = dynamic_img.width() as _;
     let h = dynamic_img.height() as _;
-    let center_x =
-        (remap(pointer_pos.x, image_rect.x_range(), 0.0..=(w as f32)).floor() as isize).at_most(w);
-    let center_y =
-        (remap(pointer_pos.y, image_rect.y_range(), 0.0..=(h as f32)).floor() as isize).at_most(h);
 
-    {
-        // Show where on the original image the zoomed-in region is at:
-        let left = (center_x - texel_radius) as f32;
-        let right = (center_x + texel_radius) as f32;
-        let top = (center_y - texel_radius) as f32;
-        let bottom = (center_y + texel_radius) as f32;
-
-        let left = remap(left, 0.0..=w as f32, image_rect.x_range());
-        let right = remap(right, 0.0..=w as f32, image_rect.x_range());
-        let top = remap(top, 0.0..=h as f32, image_rect.y_range());
-        let bottom = remap(bottom, 0.0..=h as f32, image_rect.y_range());
-
-        let rect = Rect::from_min_max(pos2(left, top), pos2(right, bottom));
-        // TODO(emilk): use `parent_ui.painter()` and put it in a high Z layer, when https://github.com/emilk/egui/issues/1516 is done
-        let painter = parent_ui.ctx().debug_painter();
-        painter.rect_stroke(rect, 0.0, (2.0, Color32::BLACK));
-        painter.rect_stroke(rect, 0.0, (1.0, Color32::WHITE));
-    }
+    // Show where on the original image the zoomed-in region is at:
+    let left = (center_x - ZOOMED_IMAGE_TEXEL_RADIUS) as f32;
+    let right = (center_x + ZOOMED_IMAGE_TEXEL_RADIUS) as f32;
+    let top = (center_y - ZOOMED_IMAGE_TEXEL_RADIUS) as f32;
+    let bottom = (center_y + ZOOMED_IMAGE_TEXEL_RADIUS) as f32;
+
+    let left = remap(left, 0.0..=w, image_rect.x_range());
+    let right = remap(right, 0.0..=w, image_rect.x_range());
+    let top = remap(top, 0.0..=h, image_rect.y_range());
+    let bottom = remap(bottom, 0.0..=h, image_rect.y_range());
+
+    let rect = Rect::from_min_max(pos2(left, top), pos2(right, bottom));
+    // TODO(emilk): use `parent_ui.painter()` and put it in a high Z layer, when https://github.com/emilk/egui/issues/1516 is done
+    let painter = ui.ctx().debug_painter();
+    painter.rect_stroke(rect, 0.0, (2.0, Color32::BLACK));
+    painter.rect_stroke(rect, 0.0, (1.0, Color32::WHITE));
+}
 
+/// `meter`: iff this is a depth map, how long is one meter?
+pub fn show_zoomed_image_region(
+    tooltip_ui: &mut egui::Ui,
+    tensor_view: &TensorImageView<'_, '_>,
+    image_position: [isize; 2],
+    meter: Option<f32>,
+) {
+    let Some(dynamic_img) = tensor_view.dynamic_img else { return };
+
+    use egui::{color_picker, pos2, remap, Color32, Mesh, Rect, Vec2};
+
+    let size = Vec2::splat(128.0);
+
+    let (_id, zoom_rect) = tooltip_ui.allocate_space(size);
     let painter = tooltip_ui.painter();
 
     painter.rect_filled(zoom_rect, 0.0, tooltip_ui.visuals().extreme_bg_color);
 
     let mut mesh = Mesh::default();
     let mut center_texel_rect = None;
-    for dx in -texel_radius..=texel_radius {
-        for dy in -texel_radius..=texel_radius {
-            let x = center_x + dx;
-            let y = center_y + dy;
+    for dx in -ZOOMED_IMAGE_TEXEL_RADIUS..=ZOOMED_IMAGE_TEXEL_RADIUS {
+        for dy in -ZOOMED_IMAGE_TEXEL_RADIUS..=ZOOMED_IMAGE_TEXEL_RADIUS {
+            let x = image_position[0] + dx;
+            let y = image_position[1] + dy;
             let color = get_pixel(dynamic_img, [x, y]);
             if let Some(color) = color {
                 let image::Rgba([r, g, b, a]) = color;
                 let color = egui::Color32::from_rgba_unmultiplied(r, g, b, a);
 
                 if color != Color32::TRANSPARENT {
-                    let tr = texel_radius as f32;
+                    let tr = ZOOMED_IMAGE_TEXEL_RADIUS as f32;
                     let left = remap(dx as f32, -tr..=(tr + 1.0), zoom_rect.x_range());
                     let right = remap((dx + 1) as f32, -tr..=(tr + 1.0), zoom_rect.x_range());
                     let top = remap(dy as f32, -tr..=(tr + 1.0), zoom_rect.y_range());
@@ -169,9 +179,9 @@ pub(crate) fn show_zoomed_image_region(
         painter.rect_stroke(center_texel_rect, 0.0, (1.0, Color32::WHITE));
     }
 
-    if let Some(color) = get_pixel(dynamic_img, [center_x, center_y]) {
+    if let Some(color) = get_pixel(dynamic_img, image_position) {
         tooltip_ui.separator();
-        let (x, y) = (center_x as _, center_y as _);
+        let (x, y) = (image_position[0] as _, image_position[1] as _);
 
         tooltip_ui.vertical(|ui| {
             if tensor_view.tensor.num_dim() == 2 {

crates/re_viewer/src/ui/view_spatial/eye.rs

@@ -12,7 +12,9 @@ use super::SpaceCamera3D;
 #[derive(Clone, Copy, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
 pub struct Eye {
     pub world_from_view: IsoTransform,
-    pub fov_y: f32,
+
+    /// If no angle is present, this is an orthographic camera.
+    pub fov_y: Option<f32>,
 }
 
 impl Eye {
@@ -26,25 +28,79 @@ impl Eye {
 
         Some(Self {
             world_from_view: space_cameras.world_from_rub_view()?,
-            fov_y,
+            fov_y: Some(fov_y),
         })
     }
 
-    #[allow(clippy::unused_self)]
     pub fn near(&self) -> f32 {
-        0.01 // TODO(emilk)
+        if self.is_perspective() {
+            0.01 // TODO(emilk)
+        } else {
+            -1000.0 // TODO(andreas)
+        }
+    }
+
+    pub fn far(&self) -> f32 {
+        if self.is_perspective() {
+            f32::INFINITY
+        } else {
+            1000.0
+        }
     }
 
     pub fn ui_from_world(&self, rect: &Rect) -> Mat4 {
         let aspect_ratio = rect.width() / rect.height();
+
+        let projection = if let Some(fov_y) = self.fov_y {
+            Mat4::perspective_infinite_rh(fov_y, aspect_ratio, self.near())
+        } else {
+            Mat4::orthographic_rh(
+                rect.left(),
+                rect.right(),
+                rect.bottom(),
+                rect.top(),
+                self.near(),
+                self.far(),
+            )
+        };
+
         Mat4::from_translation(vec3(rect.center().x, rect.center().y, 0.0))
             * Mat4::from_scale(0.5 * vec3(rect.width(), -rect.height(), 1.0))
-            * Mat4::perspective_infinite_rh(self.fov_y, aspect_ratio, self.near())
+            * projection
             * self.world_from_view.inverse()
     }
 
-    pub fn world_from_ui(&self, rect: &Rect) -> Mat4 {
-        self.ui_from_world(rect).inverse()
+    pub fn is_perspective(&self) -> bool {
+        self.fov_y.is_some()
+    }
+
+    // pub fn is_orthographic(&self) -> bool {
+    //     self.fov_y.is_none()
+    // }
+
+    /// Picking ray for a given pointer in the parent space
+    /// (i.e. prior to camera transform, "world" space)
+    pub fn picking_ray(&self, screen_rect: &Rect, pointer: glam::Vec2) -> macaw::Ray3 {
+        if let Some(fov_y) = self.fov_y {
+            let (w, h) = (screen_rect.width(), screen_rect.height());
+            let aspect_ratio = w / h;
+            let f = (fov_y * 0.5).tan();
+            let px = (2.0 * (pointer.x - screen_rect.left()) / w - 1.0) * f * aspect_ratio;
+            let py = (1.0 - 2.0 * (pointer.y - screen_rect.top()) / h) * f;
+            let ray_dir = self
+                .world_from_view
+                .transform_vector3(glam::vec3(px, py, -1.0));
+            macaw::Ray3::from_origin_dir(self.pos_in_world(), ray_dir.normalize())
+        } else {
+            // The ray originates on the camera plane, not from the camera position
+            let ray_dir = self.world_from_view.rotation().mul_vec3(glam::Vec3::Z);
+            let origin = self.world_from_view.translation()
+                + self.world_from_view.rotation().mul_vec3(glam::Vec3::X) * pointer.x
+                + self.world_from_view.rotation().mul_vec3(glam::Vec3::Y) * pointer.y
+                + ray_dir * self.near();
+
+            macaw::Ray3::from_origin_dir(origin, ray_dir)
+        }
     }
 
     pub fn pos_in_world(&self) -> glam::Vec3 {
@@ -64,7 +120,21 @@ impl Eye {
             .world_from_view
             .rotation()
             .slerp(other.world_from_view.rotation(), t);
-        let fov_y = egui::lerp(self.fov_y..=other.fov_y, t);
+
+        let fov_y = if t < 0.02 {
+            self.fov_y
+        } else if t > 0.98 {
+            other.fov_y
+        } else if self.fov_y.is_none() && other.fov_y.is_none() {
+            None
+        } else {
+            // TODO(andreas): Interpolating between perspective and ortho is untested and likely more involved than this.
+            Some(egui::lerp(
+                self.fov_y.unwrap_or(0.01)..=other.fov_y.unwrap_or(0.01),
+                t,
+            ))
+        };
+
         Eye {
             world_from_view: IsoTransform::from_rotation_translation(rotation, translation),
             fov_y,
@@ -102,7 +172,7 @@ impl OrbitEye {
                 self.world_from_view_rot,
                 self.position(),
             ),
-            fov_y: self.fov_y,
+            fov_y: Some(self.fov_y),
         }
     }
 
@@ -115,7 +185,7 @@ impl OrbitEye {
         self.orbit_radius = distance.at_least(self.orbit_radius / 5.0);
         self.orbit_center = eye.pos_in_world() + self.orbit_radius * eye.forward_in_world();
         self.world_from_view_rot = eye.world_from_view.rotation();
-        self.fov_y = eye.fov_y;
+        self.fov_y = eye.fov_y.unwrap_or(Eye::DEFAULT_FOV_Y);
         self.velocity = Vec3::ZERO;
     }