From 441e1b7a1d96a452367d4554a7eb6f8b5ee7e57b Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 29 Jun 2023 15:49:04 +0200 Subject: [PATCH 1/3] Fix pinhole visualization not working with camera extrinsics & intrinsics on the same path Our recent transform reform (as landed in 0.7) allows us to log pinhole and transform on the same entity. By convention we first apply transform3d then pinhole as-if transform3d was higher in the hierarchy. This was already the case, but the --- .../src/scene/parts/cameras.rs | 21 ++++++++++++++----- examples/python/objectron/main.py | 8 +++---- examples/rust/objectron/src/main.rs | 13 +++++------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/crates/re_space_view_spatial/src/scene/parts/cameras.rs b/crates/re_space_view_spatial/src/scene/parts/cameras.rs index 2223bf0c7368..0aecb0486df5 100644 --- a/crates/re_space_view_spatial/src/scene/parts/cameras.rs +++ b/crates/re_space_view_spatial/src/scene/parts/cameras.rs @@ -1,6 +1,6 @@ use re_components::{ coordinates::{Handedness, SignedAxis3}, - Component, InstanceKey, Pinhole, ViewCoordinates, + Component, InstanceKey, Pinhole, Transform3D, ViewCoordinates, }; use re_data_store::{EntityPath, EntityProperties}; use re_renderer::renderer::LineStripFlags; @@ -56,12 +56,14 @@ pub struct CamerasPart { } impl CamerasPart { + #[allow(clippy::too_many_arguments)] fn visit_instance( &mut self, scene_context: &SpatialSceneContext, ent_path: &EntityPath, props: &EntityProperties, pinhole: Pinhole, + transform_at_entity: Option, view_coordinates: ViewCoordinates, entity_highlight: &SpaceViewOutlineMasks, ) { @@ -76,7 +78,7 @@ impl CamerasPart { let parent_path = ent_path .parent() .expect("root path can't be part of scene query"); - let Some(world_from_parent) = scene_context.transforms.reference_from_entity(&parent_path) else { + let Some(mut world_from_camera) = scene_context.transforms.reference_from_entity(&parent_path) else { return; }; @@ -94,16 +96,24 @@ impl CamerasPart { return; } + // There's one wrinkle with using the parent transform though: + // The entity itself may have a 3D transform which (by convention!) we apply *before* the pinhole camera. + // Let's add that if it exists. + if let Some(transform_at_entity) = transform_at_entity { + world_from_camera = + world_from_camera * transform_at_entity.to_parent_from_child_transform(); + } + // If this transform is not representable an iso transform transform we can't display it yet. // This would happen if the camera is under another camera or under a transform with non-uniform scale. - let Some(world_from_camera) = macaw::IsoTransform::from_mat4(&world_from_parent.into()) else { + let Some(world_from_camera_iso) = macaw::IsoTransform::from_mat4(&world_from_camera.into()) else { return; }; self.space_cameras.push(SpaceCamera3D { ent_path: ent_path.clone(), view_coordinates, - world_from_camera, + world_from_camera: world_from_camera_iso, pinhole: Some(pinhole), picture_plane_distance: frustum_length, }); @@ -160,7 +170,7 @@ impl CamerasPart { let mut line_builder = scene_context.shared_render_builders.lines(); let mut batch = line_builder .batch("camera frustum") - .world_from_obj(world_from_parent) + .world_from_obj(world_from_camera) .outline_mask_ids(entity_highlight.overall) .picking_object_id(instance_layer_id.object); let lines = batch @@ -212,6 +222,7 @@ impl ScenePart for CamerasPart { ent_path, &props, pinhole, + store.query_latest_component::(ent_path, &query), view_coordinates, entity_highlight, ); diff --git a/examples/python/objectron/main.py b/examples/python/objectron/main.py index 357d33d34bfe..1f01b3ab0052 100755 --- a/examples/python/objectron/main.py +++ b/examples/python/objectron/main.py @@ -116,7 +116,7 @@ def log_ar_frames(samples: Iterable[SampleARFrame], seq: Sequence) -> None: rr.set_time_seconds("time", sample.timestamp) frame_times.append(sample.timestamp) - rr.log_image_file("world/camera/video", img_path=sample.image_path, img_format=rr.ImageFormat.JPEG) + rr.log_image_file("world/camera", img_path=sample.image_path, img_format=rr.ImageFormat.JPEG) log_camera(sample.frame.camera) log_point_cloud(sample.frame.raw_feature_points) @@ -149,7 +149,7 @@ def log_camera(cam: ARCamera) -> None: ) rr.log_view_coordinates("world/camera", xyz="RDF") # X=Right, Y=Down, Z=Forward rr.log_pinhole( - "world/camera/video", + "world/camera", child_from_parent=intrinsics, width=w, height=h, @@ -203,11 +203,11 @@ def log_frame_annotations(frame_times: list[float], frame_annotations: list[Fram keypoint_pos2s *= IMAGE_RESOLUTION if len(keypoint_pos2s) == 9: - log_projected_bbox(f"world/camera/video/estimates/box-{obj_ann.object_id}", keypoint_pos2s) + log_projected_bbox(f"world/camera/estimates/box-{obj_ann.object_id}", keypoint_pos2s) else: for id, pos2 in zip(keypoint_ids, keypoint_pos2s): rr.log_point( - f"world/camera/video/estimates/box-{obj_ann.object_id}/{id}", + f"world/camera/estimates/box-{obj_ann.object_id}/{id}", pos2, color=[130, 160, 250, 255], ) diff --git a/examples/rust/objectron/src/main.rs b/examples/rust/objectron/src/main.rs index 495cb5a7705a..9abcad1ecdd9 100644 --- a/examples/rust/objectron/src/main.rs +++ b/examples/rust/objectron/src/main.rs @@ -153,7 +153,7 @@ fn log_video_frame(rec_stream: &RecordingStream, ar_frame: &ArFrame) -> anyhow:: let image_path = ar_frame.dir.join(format!("video/{}.jpg", ar_frame.index)); let tensor = rerun::components::Tensor::from_jpeg_file(&image_path)?; - MsgSender::new("world/camera/video") + MsgSender::new("world/camera") .with_timepoint(ar_frame.timepoint.clone()) .with_component(&[tensor])? .send(rec_stream)?; @@ -198,7 +198,7 @@ fn log_ar_camera( rot, ))])? .send(rec_stream)?; - MsgSender::new("world/camera/video") + MsgSender::new("world/camera") .with_timepoint(timepoint) .with_component(&[Pinhole { image_from_cam: intrinsics.into(), @@ -262,12 +262,9 @@ fn log_frame_annotations( }) .unzip(); - let mut msg = MsgSender::new(format!( - "world/camera/video/estimates/box-{}", - ann.object_id - )) - .with_timepoint(timepoint.clone()) - .with_splat(ColorRGBA::from_rgb(130, 160, 250))?; + let mut msg = MsgSender::new(format!("world/camera/estimates/box-{}", ann.object_id)) + .with_timepoint(timepoint.clone()) + .with_splat(ColorRGBA::from_rgb(130, 160, 250))?; if points.len() == 9 { // Build the preprojected bounding box out of 2D line segments. From f30d1c2328e2b97ec5d09c6ec0af742999b55810 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 29 Jun 2023 15:52:37 +0200 Subject: [PATCH 2/3] fix box size in rust objectron --- examples/rust/objectron/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/rust/objectron/src/main.rs b/examples/rust/objectron/src/main.rs index 9abcad1ecdd9..383a3956a42d 100644 --- a/examples/rust/objectron/src/main.rs +++ b/examples/rust/objectron/src/main.rs @@ -123,7 +123,7 @@ fn log_baseline_objects( return None; } - let box3: Box3D = glam::Vec3::from_slice(&object.scale).into(); + let box3: Box3D = (glam::Vec3::from_slice(&object.scale) * 0.5).into(); let transform = { let translation = glam::Vec3::from_slice(&object.translation); // NOTE: the dataset is all row-major, transpose those matrices! From 3284fdad5c8e1ee65445a1f55095fa052400b654 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Thu, 29 Jun 2023 17:10:03 +0200 Subject: [PATCH 3/3] remove outdated comment on the inability to log pinhole & transform on the same entity --- crates/re_space_view_spatial/src/scene/parts/cameras.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/re_space_view_spatial/src/scene/parts/cameras.rs b/crates/re_space_view_spatial/src/scene/parts/cameras.rs index 0aecb0486df5..280852fda03e 100644 --- a/crates/re_space_view_spatial/src/scene/parts/cameras.rs +++ b/crates/re_space_view_spatial/src/scene/parts/cameras.rs @@ -72,9 +72,6 @@ impl CamerasPart { // The transform *at* this entity path already has the pinhole transformation we got passed in! // This makes sense, since if there's an image logged here one would expect that the transform applies. // We're however first interested in the rigid transform that led here, so query the parent transform. - // - // Note that currently a transform on an object can't have both a pinhole AND a rigid transform, - // which makes this rather well defined here. let parent_path = ent_path .parent() .expect("root path can't be part of scene query");