From 840417995634fa80d36a7d593ab6e0932249fb6b Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:21:34 +0200 Subject: [PATCH 01/43] remove old data density graph ui --- .../re_time_panel/src/data_density_graph.rs | 185 +---------------- crates/viewer/re_time_panel/src/lib.rs | 188 ++++++------------ crates/viewer/re_ui/src/command.rs | 13 -- crates/viewer/re_viewer/src/app.rs | 5 - crates/viewer/re_viewer/src/ui/rerun_menu.rs | 6 - .../re_viewer_context/src/app_options.rs | 5 - 6 files changed, 59 insertions(+), 343 deletions(-) diff --git a/crates/viewer/re_time_panel/src/data_density_graph.rs b/crates/viewer/re_time_panel/src/data_density_graph.rs index c68b014c02ca..63b8b426a98e 100644 --- a/crates/viewer/re_time_panel/src/data_density_graph.rs +++ b/crates/viewer/re_time_panel/src/data_density_graph.rs @@ -12,11 +12,10 @@ use egui::{epaint::Vertex, lerp, pos2, remap, Color32, NumExt as _, Rect, Shape} use re_chunk_store::Chunk; use re_chunk_store::RangeQuery; use re_data_ui::item_ui; -use re_entity_db::TimeHistogram; use re_log_types::EntityPath; use re_log_types::TimeInt; use re_log_types::Timeline; -use re_log_types::{ComponentPath, ResolvedTimeRange, TimeReal}; +use re_log_types::{ComponentPath, ResolvedTimeRange}; use re_types::ComponentName; use re_viewer_context::{Item, TimeControl, UiLayout, ViewerContext}; @@ -374,7 +373,7 @@ fn smooth(density: &[f32]) -> Vec { // ---------------------------------------------------------------------------- #[allow(clippy::too_many_arguments)] -pub fn data_density_graph_ui2( +pub fn data_density_graph_ui( data_density_graph_painter: &mut DataDensityGraphPainter, ctx: &ViewerContext<'_>, time_ctrl: &TimeControl, @@ -736,148 +735,6 @@ fn visit_relevant_chunks( } } -#[allow(clippy::too_many_arguments)] -pub fn data_density_graph_ui( - data_density_graph_painter: &mut DataDensityGraphPainter, - ctx: &ViewerContext<'_>, - time_ctrl: &mut TimeControl, - db: &re_entity_db::EntityDb, - time_area_response: &egui::Response, - time_area_painter: &egui::Painter, - ui: &egui::Ui, - time_histogram: &TimeHistogram, - row_rect: Rect, - time_ranges_ui: &TimeRangesUi, - item: &TimePanelItem, -) { - re_tracing::profile_function!(); - - let pointer_pos = ui.input(|i| i.pointer.hover_pos()); - let interact_radius_sq = ui.style().interaction.resize_grab_radius_side.powi(2); - let center_y = row_rect.center().y; - - // Density over x-axis in UI points. - let mut density_graph = DensityGraph::new(row_rect.x_range()); - - let mut num_hovered_messages = 0; - let mut hovered_time_range = ResolvedTimeRange::EMPTY; - - { - let mut add_data_point = |time_range: ResolvedTimeRange, count: usize| { - if count == 0 { - return; - } - - if let (Some(min_x), Some(max_x)) = ( - time_ranges_ui.x_from_time_f32(time_range.min().into()), - time_ranges_ui.x_from_time_f32(time_range.max().into()), - ) { - density_graph.add_range((min_x, max_x), count as _); - - // Hover: - if let Some(pointer_pos) = pointer_pos { - let center_x = (min_x + max_x) / 2.0; - let distance_sq = pos2(center_x, center_y).distance_sq(pointer_pos); - let is_hovered = distance_sq < interact_radius_sq; - - if is_hovered { - hovered_time_range = hovered_time_range.union(time_range); - num_hovered_messages += count; - } - } - } else { - // We (correctly) assume the time range is narrow, and can be approximated with its center: - let time_real = TimeReal::from(time_range.center()); - if let Some(x) = time_ranges_ui.x_from_time_f32(time_real) { - density_graph.add_point(x, count as _); - - if let Some(pointer_pos) = pointer_pos { - let distance_sq = pos2(x, center_y).distance_sq(pointer_pos); - let is_hovered = distance_sq < interact_radius_sq; - - if is_hovered { - hovered_time_range = hovered_time_range.union(time_range); - num_hovered_messages += count; - } - } - } - } - }; - - let visible_time_range = time_ranges_ui - .time_range_from_x_range((row_rect.left() - MARGIN_X)..=(row_rect.right() + MARGIN_X)); - - // The more zoomed out we are, the bigger chunks of time_histogram we can process at a time. - // Larger chunks is faster. - let chunk_size_in_ui_points = 4.0; - let time_chunk_size = - (chunk_size_in_ui_points / time_ranges_ui.points_per_time).round() as _; - let ranges: Vec<_> = { - re_tracing::profile_scope!("time_histogram.range"); - time_histogram - .range( - visible_time_range.min().as_i64()..=visible_time_range.max().as_i64(), - time_chunk_size, - ) - .collect() - }; - - re_tracing::profile_scope!("add_data_point"); - for (time_range, num_messages_at_time) in ranges { - add_data_point( - ResolvedTimeRange::new(time_range.min, time_range.max), - num_messages_at_time as _, - ); - } - } - - let hovered_x_range = (time_ranges_ui - .x_from_time_f32(hovered_time_range.min().into()) - .unwrap_or(f32::MAX) - - MARGIN_X) - ..=(time_ranges_ui - .x_from_time_f32(hovered_time_range.max().into()) - .unwrap_or(f32::MIN) - + MARGIN_X); - - density_graph.buckets = smooth(&density_graph.buckets); - - density_graph.paint( - data_density_graph_painter, - row_rect.y_range(), - time_area_painter, - graph_color(ctx, &item.to_item(), ui), - hovered_x_range, - ); - - if 0 < num_hovered_messages { - ctx.selection_state().set_hovered(item.to_item()); - - if time_area_response.clicked_by(egui::PointerButton::Primary) { - ctx.selection_state().set_selection(item.to_item()); - time_ctrl.set_time(hovered_time_range.min()); - time_ctrl.pause(); - } else if ui.ctx().dragged_id().is_none() && 0 < num_hovered_messages { - egui::show_tooltip_at_pointer( - ui.ctx(), - ui.layer_id(), - egui::Id::new("data_tooltip"), - |ui| { - show_row_ids_tooltip( - ctx, - ui, - time_ctrl, - db, - item, - hovered_time_range, - num_hovered_messages, - ); - }, - ); - } - } -} - fn graph_color(ctx: &ViewerContext<'_>, item: &Item, ui: &egui::Ui) -> Color32 { let is_selected = ctx.selection().contains_item(item); if is_selected { @@ -896,41 +753,3 @@ fn make_brighter(color: Color32) -> Color32 { b.saturating_add(64), ) } - -fn show_row_ids_tooltip( - ctx: &ViewerContext<'_>, - ui: &mut egui::Ui, - time_ctrl: &TimeControl, - db: &re_entity_db::EntityDb, - item: &TimePanelItem, - time_range: ResolvedTimeRange, - num_events: usize, -) { - use re_data_ui::DataUi as _; - - if num_events == 1 { - ui.label(format!("{num_events} event")); - } else { - ui.label(format!("{num_events} events")); - } - - let ui_layout = UiLayout::Tooltip; - let query = re_chunk_store::LatestAtQuery::new(*time_ctrl.timeline(), time_range.center()); - - let TimePanelItem { - entity_path, - component_name, - } = item; - - if let Some(component_name) = component_name { - let component_path = ComponentPath::new(entity_path.clone(), *component_name); - item_ui::component_path_button(ctx, ui, &component_path, db); - ui.add_space(8.0); - component_path.data_ui(ctx, ui, ui_layout, &query, db); - } else { - let instance_path = re_entity_db::InstancePath::entity_all(entity_path.clone()); - item_ui::instance_path_button(ctx, &query, db, ui, None, &instance_path); - ui.add_space(8.0); - instance_path.data_ui(ctx, ui, ui_layout, &query, db); - } -} diff --git a/crates/viewer/re_time_panel/src/lib.rs b/crates/viewer/re_time_panel/src/lib.rs index 896395a15c81..2fa898363b44 100644 --- a/crates/viewer/re_time_panel/src/lib.rs +++ b/crates/viewer/re_time_panel/src/lib.rs @@ -488,7 +488,6 @@ impl TimePanel { // Put time-marker on top and last, so that you can always drag it time_marker_ui( - ctx, &self.time_ranges_ui, time_ctrl, ui, @@ -702,40 +701,17 @@ impl TimePanel { // show the density graph only if that item is closed if is_closed { - let empty = re_entity_db::TimeHistogram::default(); - let num_messages_at_time = tree - .subtree - .time_histogram - .get(time_ctrl.timeline()) - .unwrap_or(&empty); - - if ctx.app_options.experimental_chunk_based_data_density_graph { - data_density_graph::data_density_graph_ui2( - &mut self.data_density_graph_painter, - ctx, - time_ctrl, - db, - time_area_painter, - ui, - &self.time_ranges_ui, - row_rect, - &item, - ); - } else { - data_density_graph::data_density_graph_ui( - &mut self.data_density_graph_painter, - ctx, - time_ctrl, - db, - time_area_response, - time_area_painter, - ui, - num_messages_at_time, - row_rect, - &self.time_ranges_ui, - &item, - ); - } + data_density_graph::data_density_graph_ui( + &mut self.data_density_graph_painter, + ctx, + time_ctrl, + db, + time_area_painter, + ui, + &self.time_ranges_ui, + row_rect, + &item, + ); } } } @@ -886,33 +862,17 @@ impl TimePanel { TimePanelSource::Blueprint => ctx.store_context.blueprint, }; - if ctx.app_options.experimental_chunk_based_data_density_graph { - data_density_graph::data_density_graph_ui2( - &mut self.data_density_graph_painter, - ctx, - time_ctrl, - db, - time_area_painter, - ui, - &self.time_ranges_ui, - row_rect, - &item, - ); - } else { - data_density_graph::data_density_graph_ui( - &mut self.data_density_graph_painter, - ctx, - time_ctrl, - db, - time_area_response, - time_area_painter, - ui, - messages_over_time, - row_rect, - &self.time_ranges_ui, - &item, - ); - } + data_density_graph::data_density_graph_ui( + &mut self.data_density_graph_painter, + ctx, + time_ctrl, + db, + time_area_painter, + ui, + &self.time_ranges_ui, + row_rect, + &item, + ); } } } @@ -1050,7 +1010,6 @@ fn collapsed_time_marker_and_time( ui.visuals().widgets.noninteractive.fg_stroke, ); time_marker_ui( - ctx, &time_ranges_ui, time_ctrl, ui, @@ -1365,7 +1324,6 @@ fn interact_with_streams_rect( /// A vertical line that shows the current time. fn time_marker_ui( - ctx: &ViewerContext<'_>, time_ranges_ui: &TimeRangesUi, time_ctrl: &mut TimeControl, ui: &egui::Ui, @@ -1428,88 +1386,56 @@ fn time_marker_ui( } // "click here to view time here" - if ctx.app_options.experimental_chunk_based_data_density_graph { - if let Some(pointer_pos) = pointer_pos { - let is_pointer_in_time_area_rect = time_area_painter.clip_rect().contains(pointer_pos); - let is_pointer_in_timeline_rect = timeline_rect.contains(pointer_pos); - - // Show preview? - if !is_hovering_time_cursor - && !time_area_double_clicked - && is_pointer_in_time_area_rect - && !is_anything_being_dragged - && !is_hovering_the_loop_selection - { - time_area_painter.vline( - pointer_pos.x, - timeline_rect.top()..=ui.max_rect().bottom(), - ui.visuals().widgets.noninteractive.fg_stroke, - ); - ui.ctx().set_cursor_icon(timeline_cursor_icon); // preview! - } - - // Click to move time here: - let time_area_response = ui.interact( - time_area_painter.clip_rect(), - ui.id().with("time_area_painter_id"), - egui::Sense::click(), - ); - - if !is_hovering_the_loop_selection { - let mut set_time_to_pointer = || { - if let Some(time) = time_ranges_ui.time_from_x_f32(pointer_pos.x) { - let time = time_ranges_ui.clamp_time(time); - time_ctrl.set_time(time); - time_ctrl.pause(); - } - }; - - // click on timeline = set time + start drag - // click on time area = set time - // double click on time area = reset time - if !is_anything_being_dragged - && is_pointer_in_timeline_rect - && ui.input(|i| i.pointer.primary_down()) - { - set_time_to_pointer(); - ui.ctx().set_dragged_id(time_drag_id); - } else if is_pointer_in_time_area_rect { - if time_area_response.double_clicked() { - time_ctrl.reset_time_view(); - } else if time_area_response.clicked() && !is_anything_being_dragged { - set_time_to_pointer(); - } - } - } - } - } else if let Some(pointer_pos) = pointer_pos { + if let Some(pointer_pos) = pointer_pos { + let is_pointer_in_time_area_rect = time_area_painter.clip_rect().contains(pointer_pos); let is_pointer_in_timeline_rect = timeline_rect.contains(pointer_pos); // Show preview? if !is_hovering_time_cursor - && is_pointer_in_timeline_rect + && !time_area_double_clicked + && is_pointer_in_time_area_rect && !is_anything_being_dragged && !is_hovering_the_loop_selection { time_area_painter.vline( pointer_pos.x, timeline_rect.top()..=ui.max_rect().bottom(), - ui.visuals().widgets.noninteractive.bg_stroke, + ui.visuals().widgets.noninteractive.fg_stroke, ); ui.ctx().set_cursor_icon(timeline_cursor_icon); // preview! } // Click to move time here: - if ui.input(|i| i.pointer.primary_down()) - && is_pointer_in_timeline_rect - && !is_anything_being_dragged - && !is_hovering_the_loop_selection - { - if let Some(time) = time_ranges_ui.time_from_x_f32(pointer_pos.x) { - let time = time_ranges_ui.clamp_time(time); - time_ctrl.set_time(time); - time_ctrl.pause(); - ui.ctx().set_dragged_id(time_drag_id); // act as if the user grabbed the time marker cursor + let time_area_response = ui.interact( + time_area_painter.clip_rect(), + ui.id().with("time_area_painter_id"), + egui::Sense::click(), + ); + + if !is_hovering_the_loop_selection { + let mut set_time_to_pointer = || { + if let Some(time) = time_ranges_ui.time_from_x_f32(pointer_pos.x) { + let time = time_ranges_ui.clamp_time(time); + time_ctrl.set_time(time); + time_ctrl.pause(); + } + }; + + // click on timeline = set time + start drag + // click on time area = set time + // double click on time area = reset time + if !is_anything_being_dragged + && is_pointer_in_timeline_rect + && ui.input(|i| i.pointer.primary_down()) + { + set_time_to_pointer(); + ui.ctx().set_dragged_id(time_drag_id); + } else if is_pointer_in_time_area_rect { + if time_area_response.double_clicked() { + time_ctrl.reset_time_view(); + } else if time_area_response.clicked() && !is_anything_being_dragged { + set_time_to_pointer(); + } } } } diff --git a/crates/viewer/re_ui/src/command.rs b/crates/viewer/re_ui/src/command.rs index 11ffd822dcfb..cc67d7aa9314 100644 --- a/crates/viewer/re_ui/src/command.rs +++ b/crates/viewer/re_ui/src/command.rs @@ -85,8 +85,6 @@ pub enum UICommand { RestartWithWebGl, #[cfg(target_arch = "wasm32")] RestartWithWebGpu, - - ToggleChunkBasedDataDensityGraph, } impl UICommand { @@ -249,11 +247,6 @@ impl UICommand { "Restart with WebGPU", "Reloads the webpage and force WebGPU for rendering. All data will be lost." ), - - Self::ToggleChunkBasedDataDensityGraph => ( - "Toggle chunk-based data density graph", - "Toggle between the old and new data density graph", - ), } } @@ -275,10 +268,6 @@ impl UICommand { KeyboardShortcut::new(Modifiers::CTRL.plus(Modifiers::SHIFT), key) } - fn cmd_shift(key: Key) -> KeyboardShortcut { - KeyboardShortcut::new(Modifiers::COMMAND.plus(Modifiers::SHIFT), key) - } - match self { Self::SaveRecording => Some(cmd(Key::S)), Self::SaveRecordingSelection => Some(cmd_alt(Key::S)), @@ -357,8 +346,6 @@ impl UICommand { #[cfg(target_arch = "wasm32 ")] Self::ViewportMode(_) => None, - - Self::ToggleChunkBasedDataDensityGraph => Some(cmd_shift(Key::D)), } } diff --git a/crates/viewer/re_viewer/src/app.rs b/crates/viewer/re_viewer/src/app.rs index 336b8f98211a..dd105ebf22ea 100644 --- a/crates/viewer/re_viewer/src/app.rs +++ b/crates/viewer/re_viewer/src/app.rs @@ -770,11 +770,6 @@ impl App { re_log::error!("Failed to set URL parameter `renderer=webgpu` & refresh page."); } } - - UICommand::ToggleChunkBasedDataDensityGraph => { - self.app_options_mut() - .experimental_chunk_based_data_density_graph ^= true; - } } } diff --git a/crates/viewer/re_viewer/src/ui/rerun_menu.rs b/crates/viewer/re_viewer/src/ui/rerun_menu.rs index dd2257e9d976..fa6fea0a61bf 100644 --- a/crates/viewer/re_viewer/src/ui/rerun_menu.rs +++ b/crates/viewer/re_viewer/src/ui/rerun_menu.rs @@ -371,12 +371,6 @@ fn experimental_feature_ui( "Plots: query clamping", ) .on_hover_text("Toggle query clamping for the plot visualizers."); - - ui.re_checkbox( - &mut app_options.experimental_chunk_based_data_density_graph, - "Chunk-based data density graph", - ) - .on_hover_text("Toggle chunk-based data density graph"); } #[cfg(debug_assertions)] diff --git a/crates/viewer/re_viewer_context/src/app_options.rs b/crates/viewer/re_viewer_context/src/app_options.rs index 7730f7255fee..73f880f3e3d9 100644 --- a/crates/viewer/re_viewer_context/src/app_options.rs +++ b/crates/viewer/re_viewer_context/src/app_options.rs @@ -23,9 +23,6 @@ pub struct AppOptions { /// Toggle query clamping for the plot visualizers. pub experimental_plot_query_clamping: bool, - /// Toggle new data density graph. - pub experimental_chunk_based_data_density_graph: bool, - /// Displays an overlay for debugging picking. pub show_picking_debug_overlay: bool, @@ -57,8 +54,6 @@ impl Default for AppOptions { experimental_plot_query_clamping: false, - experimental_chunk_based_data_density_graph: false, - show_picking_debug_overlay: false, inspect_blueprint_timeline: false, From 0534aa019a3857f1820445c68ab95190bffbe1a9 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 01:09:43 +0200 Subject: [PATCH 02/43] wip --- crates/store/re_chunk_store/src/query.rs | 307 ++++++++ crates/store/re_entity_db/src/entity_db.rs | 30 +- crates/store/re_entity_db/src/entity_tree.rs | 195 +---- .../src/time_histogram_per_timeline.rs | 25 +- crates/store/re_entity_db/tests/clear.rs | 4 +- .../re_entity_db/tests/time_histograms.rs | 705 ------------------ .../src/actions/collapse_expand_all.rs | 4 +- crates/viewer/re_data_ui/src/component.rs | 60 +- .../viewer/re_data_ui/src/component_path.rs | 7 +- crates/viewer/re_data_ui/src/instance_path.rs | 4 +- crates/viewer/re_data_ui/src/item_ui.rs | 32 +- .../re_selection_panel/src/selection_panel.rs | 9 +- .../src/space_view_entity_picker.rs | 2 +- .../re_space_view_spatial/src/view_2d.rs | 8 +- .../re_space_view_spatial/src/view_3d.rs | 8 +- .../re_time_panel/src/data_density_graph.rs | 2 +- crates/viewer/re_time_panel/src/lib.rs | 41 +- .../re_viewer_context/src/time_control.rs | 18 +- .../re_viewport_blueprint/src/space_view.rs | 11 +- .../src/space_view_contents.rs | 12 +- 20 files changed, 478 insertions(+), 1006 deletions(-) delete mode 100644 crates/store/re_entity_db/tests/time_histograms.rs diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 4ebe623f1f31..8ff96c12b17b 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -5,7 +5,9 @@ use std::{ use itertools::Itertools; use re_chunk::{Chunk, LatestAtQuery, RangeQuery}; +use re_log_types::ResolvedTimeRange; use re_log_types::{EntityPath, TimeInt, Timeline}; +use re_types_core::SizeBytes as _; use re_types_core::{ComponentName, ComponentNameSet}; use crate::{store::ChunkIdSetPerTime, ChunkStore}; @@ -61,6 +63,47 @@ impl ChunkStore { } } + /// Retrieve all the [`ComponentName`]s that have been written to for a given [`EntityPath`]. + /// + /// Static components are always included in the results. + /// + /// Returns `None` if the entity has never had any data logged to it. + pub fn all_components_on_all_timelines( + &self, + entity_path: &EntityPath, + ) -> Option { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let static_components: Option = self + .static_chunk_ids_per_entity + .get(entity_path) + .map(|static_chunks_per_component| { + static_chunks_per_component.keys().copied().collect() + }); + + let temporal_components: Option = self + .temporal_chunk_ids_per_entity_per_component + .get(entity_path) + .map(|temporal_chunk_ids_per_timeline| { + temporal_chunk_ids_per_timeline + .iter() + .flat_map(|(_, temporal_chunk_ids_per_component)| { + temporal_chunk_ids_per_component.keys().copied() + }) + .collect() + }); + + match (static_components, temporal_components) { + (None, None) => None, + (None, comps @ Some(_)) | (comps @ Some(_), None) => comps, + (Some(static_comps), Some(temporal_comps)) => { + Some(static_comps.into_iter().chain(temporal_comps).collect()) + } + } + } + /// Check whether a given entity has a specific [`ComponentName`] either on the specified /// timeline, or in its static data. #[inline] @@ -75,6 +118,56 @@ impl ChunkStore { .map_or(false, |components| components.contains(component_name)) } + pub fn entity_has_component_on_any_timeline( + &self, + entity_path: &EntityPath, + component_name: &ComponentName, + ) -> bool { + re_tracing::profile_function!(); + + if self + .static_chunk_ids_per_entity + .get(entity_path) + .and_then(|static_chunks_per_component| static_chunks_per_component.get(component_name)) + .and_then(|id| self.chunks_per_chunk_id.get(id)) + .is_some() + { + return true; + } + + for temporal_chunk_ids_per_component in self + .temporal_chunk_ids_per_entity_per_component + .get(entity_path) + .iter() + .flat_map(|temporal_chunk_ids_per_timeline| temporal_chunk_ids_per_timeline.values()) + { + if temporal_chunk_ids_per_component + .get(component_name) + .is_some() + { + return true; + } + } + + false + } + + /// Returns true if the given entity has a specific component with static data. + #[inline] + pub fn entity_has_static_component( + &self, + entity_path: &EntityPath, + component_name: &ComponentName, + ) -> bool { + re_tracing::profile_function!(); + + let Some(static_components) = self.static_chunk_ids_per_entity.get(entity_path) else { + return false; + }; + + static_components.contains_key(component_name) + } + /// Find the earliest time at which something was logged for a given entity on the specified /// timeline. /// @@ -413,3 +506,217 @@ impl ChunkStore { .collect() } } + +// Queries returning `usize` and `bool` +impl ChunkStore { + /// Returns the number of events logged for a given component on the given entity path across all timelines. + pub fn num_events_on_timeline_for_component( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + component_name: ComponentName, + ) -> usize { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + self.num_events_for_static_component(entity_path, component_name) + + self.num_events_on_timeline_for_temporal_component( + timeline, + entity_path, + component_name, + ) + } + + pub fn time_range_for_entity( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> Option { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let temporal_chunk_ids_per_timeline = + self.temporal_chunk_ids_per_entity.get(entity_path)?; + let chunk_id_sets = temporal_chunk_ids_per_timeline.get(timeline)?; + + let start = chunk_id_sets.per_start_time.keys().min()?; + let end = chunk_id_sets.per_end_time.keys().max()?; + + Some(ResolvedTimeRange::new(*start, *end)) + } + + pub fn num_events_on_timeline_for_all_components( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> usize { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let mut total_events = 0; + + if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) + { + for chunk in static_chunks_per_component + .values() + .filter_map(|id| self.chunks_per_chunk_id.get(id)) + { + total_events += chunk.num_events_cumulative(); + } + } + + if let Some(chunk_ids) = self + .temporal_chunk_ids_per_entity + .get(entity_path) + .and_then(|temporal_chunks_events_per_timeline| { + temporal_chunks_events_per_timeline.get(timeline) + }) + { + for chunk in chunk_ids + .per_start_time + .values() + .flat_map(|ids| ids.iter().filter_map(|id| self.chunks_per_chunk_id.get(id))) + { + total_events += chunk.num_events_cumulative(); + } + } + + total_events + } + + pub fn entity_has_data_on_timeline( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> bool { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + if self.static_chunk_ids_per_entity.get(entity_path).is_some() { + // Static data exists on all timelines + return true; + } + + if let Some(temporal_chunk_ids_per_timeline) = + self.temporal_chunk_ids_per_entity.get(entity_path) + { + if temporal_chunk_ids_per_timeline.contains_key(timeline) { + return true; + } + } + + false + } + + pub fn num_events_for_static_component( + &self, + entity_path: &EntityPath, + component_name: ComponentName, + ) -> usize { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + if let Some(static_chunk) = self + .static_chunk_ids_per_entity + .get(entity_path) + .and_then(|static_chunks_per_component| { + static_chunks_per_component.get(&component_name) + }) + .and_then(|chunk_id| self.chunks_per_chunk_id.get(chunk_id)) + { + // Static data overrides temporal data + static_chunk + .num_events_for_component(component_name) + .unwrap_or(0) + } else { + 0 + } + } + + /// Returns the number of events logged for a given component on the given entity path across all timelines. + /// + /// This ignores static data. + pub fn num_events_on_timeline_for_temporal_component( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + component_name: ComponentName, + ) -> usize { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let Some(temporal_chunk_ids_per_timeline) = self + .temporal_chunk_ids_per_entity_per_component + .get(entity_path) + else { + return 0; // no events logged for the entity path + }; + + let Some(temporal_chunk_ids_per_component) = temporal_chunk_ids_per_timeline.get(timeline) + else { + return 0; // no events logged on this timeline + }; + + let Some(chunk_ids) = temporal_chunk_ids_per_component.get(&component_name) else { + return 0; // no events logged for the component on this timeline + }; + + let mut num_events = 0; + for chunk in chunk_ids + .per_start_time + .values() + .flat_map(|ids| ids.iter().filter_map(|id| self.chunks_per_chunk_id.get(id))) + { + num_events += chunk.num_events_for_component(component_name).unwrap_or(0); + } + + num_events + } + + pub fn size_of_entity_on_timeline( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> usize { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let mut total_size = 0; + + if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) + { + for chunk in static_chunks_per_component + .values() + .filter_map(|id| self.chunks_per_chunk_id.get(id)) + { + total_size += Chunk::total_size_bytes(chunk) as usize; + } + } + + if let Some(chunk_id_sets) = self + .temporal_chunk_ids_per_entity + .get(entity_path) + .and_then(|temporal_chunk_ids_per_timeline| { + temporal_chunk_ids_per_timeline.get(timeline) + }) + { + for chunk in chunk_id_sets + .per_start_time + .values() + .flat_map(|v| v.iter()) + .filter_map(|id| self.chunks_per_chunk_id.get(id)) + { + total_size += Chunk::total_size_bytes(chunk) as usize; + } + } + + total_size + } +} diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index 50231342cf77..0cb02c8b9453 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -9,8 +9,8 @@ use re_chunk_store::{ GarbageCollectionTarget, }; use re_log_types::{ - ApplicationId, ComponentPath, EntityPath, EntityPathHash, LogMsg, ResolvedTimeRange, - ResolvedTimeRangeF, SetStoreInfo, StoreId, StoreInfo, StoreKind, Timeline, + ApplicationId, EntityPath, EntityPathHash, LogMsg, ResolvedTimeRange, ResolvedTimeRangeF, + SetStoreInfo, StoreId, StoreInfo, StoreKind, Timeline, }; use crate::{Error, TimesPerTimeline}; @@ -56,6 +56,9 @@ pub struct EntityDb { /// A tree-view (split on path components) of the entities. tree: crate::EntityTree, + /// Holds a [`TimeHistogram`][`crate::TimeHistogram`] spanning the whole entity tree for each [`Timeline`]. + time_histogram_per_timeline: crate::TimeHistogramPerTimeline, + /// Stores all components for all entities for all timelines. data_store: ChunkStore, @@ -85,6 +88,7 @@ impl EntityDb { entity_path_from_hash: Default::default(), times_per_timeline: Default::default(), tree: crate::EntityTree::root(), + time_histogram_per_timeline: Default::default(), data_store, resolver: re_query::PromiseResolver::default(), query_caches, @@ -267,25 +271,7 @@ impl EntityDb { /// Histogram of all events on the timeeline, of all entities. pub fn time_histogram(&self, timeline: &Timeline) -> Option<&crate::TimeHistogram> { - self.tree().subtree.time_histogram.get(timeline) - } - - /// Total number of static messages for any entity. - pub fn num_static_messages(&self) -> u64 { - self.tree.num_static_messages_recursive() - } - - /// Returns whether a component is static. - pub fn is_component_static(&self, component_path: &ComponentPath) -> Option { - if let Some(entity_tree) = self.tree().subtree(component_path.entity_path()) { - entity_tree - .entity - .components - .get(&component_path.component_name) - .map(|component_histogram| component_histogram.is_static()) - } else { - None - } + self.time_histogram_per_timeline.get(timeline) } #[inline] @@ -451,6 +437,7 @@ impl EntityDb { entity_path_from_hash: _, times_per_timeline, tree, + time_histogram_per_timeline, data_store: _, resolver: _, query_caches, @@ -459,6 +446,7 @@ impl EntityDb { times_per_timeline.on_events(store_events); query_caches.on_events(store_events); + time_histogram_per_timeline.on_events(store_events); tree.on_store_deletions(store_events); } diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index b25ce4800f0f..d19ecb3158b5 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -6,15 +6,13 @@ use nohash_hasher::IntMap; use re_chunk::RowId; use re_chunk_store::{ChunkStoreDiff, ChunkStoreDiffKind, ChunkStoreEvent, ChunkStoreSubscriber}; -use re_log_types::{ComponentPath, EntityPath, EntityPathHash, EntityPathPart, TimeInt, Timeline}; +use re_log_types::{EntityPath, EntityPathHash, EntityPathPart, TimeInt, Timeline}; use re_types_core::ComponentName; // Used all over in docstrings. #[allow(unused_imports)] use re_chunk_store::ChunkStore; -use crate::TimeHistogramPerTimeline; - // ---------------------------------------------------------------------------- /// A recursive, manually updated [`ChunkStoreSubscriber`] that maintains the entity hierarchy. @@ -26,12 +24,6 @@ pub struct EntityTree { /// Direct descendants of this (sub)tree. pub children: BTreeMap, - - /// Information about this specific entity (excluding children). - pub entity: EntityInfo, - - /// Info about this subtree, including all children, recursively. - pub subtree: SubtreeInfo, } // NOTE: This is only to let people know that this is in fact a [`ChunkStoreSubscriber`], so they A) don't try @@ -57,86 +49,6 @@ impl ChunkStoreSubscriber for EntityTree { } } -/// Information about this specific entity (excluding children). -#[derive(Default)] -pub struct EntityInfo { - /// Flat time histograms for each component of this [`EntityTree`]. - /// - /// Keeps track of the _number of times a component is logged_ per time per timeline, only for - /// this specific [`EntityTree`]. - /// A component logged twice at the same timestamp is counted twice. - /// - /// ⚠ Auto-generated instance keys are _not_ accounted for. ⚠ - pub components: BTreeMap, -} - -/// Info about stuff at a given [`EntityPath`], including all of its children, recursively. -#[derive(Default)] -pub struct SubtreeInfo { - /// Recursive time histogram for this [`EntityTree`]. - /// - /// Keeps track of the _number of components logged_ per time per timeline, recursively across - /// all of the [`EntityTree`]'s children. - /// A component logged twice at the same timestamp is counted twice. - /// - /// ⚠ Auto-generated instance keys are _not_ accounted for. ⚠ - pub time_histogram: TimeHistogramPerTimeline, - - /// Number of bytes used by all arrow data - data_bytes: u64, -} - -impl SubtreeInfo { - /// Assumes the event has been filtered to be part of this subtree. - fn on_event(&mut self, event: &ChunkStoreEvent) { - use re_types_core::SizeBytes as _; - - match event.kind { - ChunkStoreDiffKind::Addition => { - let times = event - .chunk - .timelines() - .iter() - .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) - .collect_vec(); - self.time_histogram.add(×, event.num_components() as _); - - self.data_bytes += event.chunk.total_size_bytes(); - } - ChunkStoreDiffKind::Deletion => { - let times = event - .chunk - .timelines() - .iter() - .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) - .collect_vec(); - self.time_histogram - .remove(×, event.num_components() as _); - - let removed_bytes = event.chunk.total_size_bytes(); - self.data_bytes - .checked_sub(removed_bytes) - .unwrap_or_else(|| { - re_log::debug!( - store_id = %event.store_id, - entity_path = %event.chunk.entity_path(), - current = self.data_bytes, - removed = removed_bytes, - "book keeping underflowed" - ); - u64::MIN - }); - } - } - } - - /// Number of bytes used by all arrow data in this tree (including their schemas, but otherwise ignoring book-keeping overhead). - #[inline] - pub fn data_bytes(&self) -> u64 { - self.data_bytes - } -} - /// Maintains an optimized representation of a batch of [`ChunkStoreEvent`]s specifically designed to /// accelerate garbage collection of [`EntityTree`]s. /// @@ -206,8 +118,6 @@ impl EntityTree { Self { path, children: Default::default(), - entity: Default::default(), - subtree: Default::default(), } } @@ -216,24 +126,8 @@ impl EntityTree { self.children.is_empty() } - pub fn num_children_and_fields(&self) -> usize { - self.children.len() + self.entity.components.len() - } - - /// Number of timeless messages in this tree, or any child, recursively. - pub fn num_static_messages_recursive(&self) -> u64 { - self.subtree.time_histogram.num_static_messages() - } - - pub fn time_histogram_for_component( - &self, - timeline: &Timeline, - component_name: impl Into, - ) -> Option<&crate::TimeHistogram> { - self.entity - .components - .get(&component_name.into()) - .and_then(|per_timeline| per_timeline.get(timeline)) + pub fn num_children(&self) -> usize { + self.children.len() } /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s. @@ -256,14 +150,12 @@ impl EntityTree { // Book-keeping for each level in the hierarchy: let mut tree = self; - tree.subtree.on_event(event); for (i, part) in entity_path.iter().enumerate() { tree = tree .children .entry(part.clone()) .or_insert_with(|| Self::new(entity_path.as_slice()[..=i].into())); - tree.subtree.on_event(event); } // Finally book-keeping for the entity where data was actually added: @@ -272,25 +164,7 @@ impl EntityTree { /// Handles the addition of new data into the tree. fn on_added_data(&mut self, store_diff: &ChunkStoreDiff) { - for component_name in store_diff.chunk.component_names() { - let component_path = - ComponentPath::new(store_diff.chunk.entity_path().clone(), component_name); - - let per_component = self - .entity - .components - .entry(component_path.component_name) - .or_default(); - per_component.add( - &store_diff - .chunk - .timelines() - .iter() - .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) - .collect_vec(), - 1, - ); - } + let _ = (self, store_diff); } /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s. @@ -299,12 +173,7 @@ impl EntityTree { pub fn on_store_deletions(&mut self, store_events: &[ChunkStoreEvent]) { re_tracing::profile_function!(); - let Self { - path, - children, - entity, - subtree, - } = self; + let Self { path, children } = self; // Only keep events relevant to this branch of the tree. let subtree_events = store_events @@ -314,61 +183,29 @@ impl EntityTree { .cloned() .collect_vec(); - { - re_tracing::profile_scope!("entity"); - for event in subtree_events - .iter() - .filter(|e| e.chunk.entity_path() == path) - { - for component_name in event.chunk.component_names() { - if let Some(histo) = entity.components.get_mut(&component_name) { - histo.remove( - &event - .chunk - .timelines() - .iter() - .map(|(timeline, time_chunk)| (*timeline, time_chunk.times_raw())) - .collect_vec(), - 1, - ); - if histo.is_empty() { - entity.components.remove(&component_name); - } - } - } - } - } - - { - re_tracing::profile_scope!("subtree"); - for event in &subtree_events { - subtree.on_event(event); - } - } - children.retain(|_, child| { child.on_store_deletions(&subtree_events); - child.num_children_and_fields() > 0 + child.num_children() > 0 }); } pub fn subtree(&self, path: &EntityPath) -> Option<&Self> { - fn subtree_recursive<'tree>( - this: &'tree EntityTree, - path: &[EntityPathPart], - ) -> Option<&'tree EntityTree> { + let mut this = self; + let mut path = path.as_slice(); + loop { match path { - [] => Some(this), - [first, rest @ ..] => subtree_recursive(this.children.get(first)?, rest), + [] => return Some(this), + [first, rest @ ..] => { + this = this.children.get(first)?; + path = rest; + } } } - - subtree_recursive(self, path.as_slice()) } // Invokes visitor for `self` and all children recursively. - pub fn visit_children_recursively(&self, visitor: &mut impl FnMut(&EntityPath, &EntityInfo)) { - visitor(&self.path, &self.entity); + pub fn visit_children_recursively(&self, visitor: &mut impl FnMut(&EntityPath)) { + visitor(&self.path); for child in self.children.values() { child.visit_children_recursively(visitor); } diff --git a/crates/store/re_entity_db/src/time_histogram_per_timeline.rs b/crates/store/re_entity_db/src/time_histogram_per_timeline.rs index ebb06767a4c6..3fde33578e3f 100644 --- a/crates/store/re_entity_db/src/time_histogram_per_timeline.rs +++ b/crates/store/re_entity_db/src/time_histogram_per_timeline.rs @@ -1,5 +1,7 @@ use std::collections::BTreeMap; +use itertools::Itertools as _; +use re_chunk_store::ChunkStoreDiffKind; use re_chunk_store::{ChunkStoreEvent, ChunkStoreSubscriber}; use re_log_types::Timeline; @@ -113,8 +115,6 @@ impl TimeHistogramPerTimeline { } } -// NOTE: This is only to let people know that this is in fact a [`ChunkStoreSubscriber`], so they A) don't try -// to implement it on their own and B) don't try to register it. impl ChunkStoreSubscriber for TimeHistogramPerTimeline { #[inline] fn name(&self) -> String { @@ -132,9 +132,22 @@ impl ChunkStoreSubscriber for TimeHistogramPerTimeline { } #[allow(clippy::unimplemented)] - fn on_events(&mut self, _events: &[ChunkStoreEvent]) { - unimplemented!( - r"TimeHistogramPerTimeline view is maintained as a sub-view of `EntityTree`", - ); + fn on_events(&mut self, events: &[ChunkStoreEvent]) { + for event in events { + let times = event + .chunk + .timelines() + .iter() + .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) + .collect_vec(); + match event.kind { + ChunkStoreDiffKind::Addition => { + self.add(×, event.num_components() as _); + } + ChunkStoreDiffKind::Deletion => { + self.remove(×, event.num_components() as _); + } + } + } } } diff --git a/crates/store/re_entity_db/tests/clear.rs b/crates/store/re_entity_db/tests/clear.rs index 5f3e32f45745..3de66f176d17 100644 --- a/crates/store/re_entity_db/tests/clear.rs +++ b/crates/store/re_entity_db/tests/clear.rs @@ -414,7 +414,7 @@ fn clear_and_gc() -> anyhow::Result<()> { // Insert a component, then clear it, then GC. { // EntityTree is Empty when we start - assert_eq!(db.tree().num_children_and_fields(), 0); + assert_eq!(db.tree().num_children(), 0); let point = MyPoint::new(1.0, 2.0); @@ -450,7 +450,7 @@ fn clear_and_gc() -> anyhow::Result<()> { assert_eq!(stats.temporal_chunks.total_num_rows, 0); // EntityTree should be empty again when we end since everything was GC'd - assert_eq!(db.tree().num_children_and_fields(), 0); + assert_eq!(db.tree().num_children(), 0); } Ok(()) diff --git a/crates/store/re_entity_db/tests/time_histograms.rs b/crates/store/re_entity_db/tests/time_histograms.rs deleted file mode 100644 index a83fd10b8eec..000000000000 --- a/crates/store/re_entity_db/tests/time_histograms.rs +++ /dev/null @@ -1,705 +0,0 @@ -// https://github.com/rust-lang/rust-clippy/issues/10011 -#![cfg(test)] - -use std::{collections::BTreeSet, sync::Arc}; - -use re_chunk::{Chunk, ChunkId, RowId}; -use re_chunk_store::GarbageCollectionOptions; -use re_entity_db::EntityDb; -use re_int_histogram::RangeI64; -use re_log_types::{ - example_components::{MyColor, MyIndex, MyPoint}, - EntityPath, StoreId, TimeInt, TimePoint, Timeline, -}; -use re_types_core::{components::ClearIsRecursive, ComponentName, Loggable}; - -// --- - -// Complete test suite for our various time histograms which, among other things, power the -// timeline widget. -#[test] -fn time_histograms() -> anyhow::Result<()> { - let mut db = EntityDb::new(StoreId::random(re_log_types::StoreKind::Recording)); - - let timeline_frame = Timeline::new_sequence("frame"); - let timeline_other = Timeline::new_temporal("other"); - let timeline_yet_another = Timeline::new_sequence("yet_another"); - - let entity_parent: EntityPath = "parent".into(); - let entity_grandchild: EntityPath = "parent/child/grandchild".into(); - let entity_unrelated: EntityPath = "very/unrelated".into(); - - // Single top-level entity, explicitly logged `MyIndex`s. - { - let chunk = Chunk::builder(entity_parent.clone()) - .with_component_batches( - RowId::new(), - TimePoint::from_iter([ - (timeline_frame, 42), // - (timeline_other, 666), // - (timeline_yet_another, 1), // - ]), - [&MyIndex::from_iter(0..10) as _], - ) - .build()?; - - db.add_chunk(&Arc::new(chunk))?; - - // times per timeline - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42])), - (&timeline_other, Some(&[666])), - (&timeline_yet_another, Some(&[1])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline ('parent') - assert_histogram_for_component( - &db, - &entity_parent, - MyIndex::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 3], - ); - } - - // Grand-child, multiple components, auto-generated `MyIndex`s. - { - let chunk = { - let num_instances = 3; - let points: Vec<_> = (0..num_instances) - .map(|i| MyPoint::new(0.0, i as f32)) - .collect(); - let colors = vec![MyColor::from(0xFF0000FF)]; - Chunk::builder(entity_grandchild.clone()) - .with_component_batches( - RowId::new(), - TimePoint::from_iter([ - (timeline_frame, 42), // - (timeline_yet_another, 1), // - ]), - [&points as _, &colors as _], - ) - .build()? - }; - let chunk = Arc::new(chunk); - - db.add_chunk(&chunk)?; - - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42])), - (&timeline_other, Some(&[666])), - (&timeline_yet_another, Some(&[1])), - ], - ); - - // histograms per timeline - // NOTE: notice how autogenerated instance keys are ignored. - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 3)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 3)])), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline ('parent') - assert_histogram_for_component( - &db, - &entity_parent, - MyIndex::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 3], - ); - // NOTE: per-component histograms are not recursive! - assert_histogram_for_component( - &db, - &entity_parent, - MyPoint::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_parent, - MyColor::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - - // histograms per component per timeline ('parent/child/grandchild') - // NOTE: notice how autogenerated instance keys are ignored! - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyPoint::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyColor::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 2], - ); - - db.add_chunk(&Arc::new(chunk.clone_as(ChunkId::new(), RowId::new())))?; // same chunk a second time! - - // times per timeline - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42])), - (&timeline_other, Some(&[666])), - (&timeline_yet_another, Some(&[1])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 5)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 5)])), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline ('parent/child/grandchild') - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyPoint::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyColor::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - } - - // Grand-child, timeless additions. - { - let chunk = { - let num_instances = 6; - let colors = vec![MyColor::from(0x00DD00FF); num_instances]; - Chunk::builder("entity".into()) - .with_component_batches( - RowId::new(), - TimePoint::default(), - [ - &MyIndex::from_iter(0..num_instances as _) as _, - &colors as _, - ], - ) - .build()? - }; - - db.add_chunk(&Arc::new(chunk))?; - - // times per timeline - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42])), - (&timeline_other, Some(&[666])), - (&timeline_yet_another, Some(&[1])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 5)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 5)])), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_parent, - MyPoint::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_parent, - MyColor::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyPoint::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyColor::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - } - - // Completely unrelated entity. - { - let chunk = { - let num_instances = 3; - let points: Vec<_> = (0..num_instances) - .map(|i| MyPoint::new(0.0, i as f32)) - .collect(); - let colors = vec![MyColor::from(0xFF0000FF)]; - Chunk::builder(entity_unrelated.clone()) - .with_component_batches( - RowId::new(), - TimePoint::from_iter([ - (timeline_frame, 1234), // - (timeline_other, 1235), // - (timeline_yet_another, 1236), // - ]), - [ - &MyIndex::from_iter(0..num_instances) as _, - &points as _, - &colors as _, - ], - ) - .build()? - }; - - db.add_chunk(&Arc::new(chunk))?; - - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42, 1234])), - (&timeline_other, Some(&[666, 1235])), - (&timeline_yet_another, Some(&[1, 1236])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - ( - &timeline_frame, - Some(&[(RangeI64::new(42, 42), 5), (RangeI64::new(1234, 1234), 3)]), - ), - ( - &timeline_other, - Some(&[(RangeI64::new(666, 666), 1), (RangeI64::new(1235, 1235), 3)]), - ), - ( - &timeline_yet_another, - Some(&[(RangeI64::new(1, 1), 5), (RangeI64::new(1236, 1236), 3)]), - ), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_parent, - MyIndex::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 3], - ); - // NOTE: per-component histograms are not recursive! - assert_histogram_for_component( - &db, - &entity_parent, - MyPoint::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_parent, - MyColor::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - - // histograms per component per timeline ('very/unrelated') - assert_histogram_for_component( - &db, - &entity_unrelated, - MyIndex::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(1234, 1234), 1)])), - (&timeline_other, Some(&[(RangeI64::new(1235, 1235), 1)])), - ( - &timeline_yet_another, - Some(&[(RangeI64::new(1236, 1236), 1)]), - ), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_unrelated, - MyPoint::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(1234, 1234), 1)])), - (&timeline_other, Some(&[(RangeI64::new(1235, 1235), 1)])), - ( - &timeline_yet_another, - Some(&[(RangeI64::new(1236, 1236), 1)]), - ), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_unrelated, - MyColor::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(1234, 1234), 1)])), - (&timeline_other, Some(&[(RangeI64::new(1235, 1235), 1)])), - ( - &timeline_yet_another, - Some(&[(RangeI64::new(1236, 1236), 1)]), - ), - ] as [(_, Option<&[_]>); 3], - ); - } - - // Immediate clear. - { - let chunk = { - Chunk::builder(entity_parent.clone()) - .with_component_batches( - RowId::new(), - TimePoint::from_iter([ - (timeline_frame, 1000), // - ]), - [&[ClearIsRecursive(true.into())] as _], - ) - .build()? - }; - - db.add_chunk(&Arc::new(chunk))?; - - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42, 1000, 1234])), - (&timeline_other, Some(&[666, 1235])), - (&timeline_yet_another, Some(&[1, 1236])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - ( - &timeline_frame, - Some(&[ - (RangeI64::new(42, 42), 5), - (RangeI64::new(1000, 1000), 1), - (RangeI64::new(1234, 1234), 3), - ]), - ), - ( - &timeline_other, - Some(&[(RangeI64::new(666, 666), 1), (RangeI64::new(1235, 1235), 3)]), - ), - ( - &timeline_yet_another, - Some(&[(RangeI64::new(1, 1), 5), (RangeI64::new(1236, 1236), 3)]), - ), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_parent, - MyIndex::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 3], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyPoint::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyColor::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - } - - // Full GC - { - db.gc(&GarbageCollectionOptions::gc_everything()); - - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[])), - (&timeline_other, Some(&[])), - (&timeline_yet_another, Some(&[])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - (&timeline_frame, Some(&[])), - (&timeline_other, Some(&[])), - (&timeline_yet_another, Some(&[])), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_parent, - MyIndex::name(), - [ - (&timeline_frame, Some(&[])), - (&timeline_other, Some(&[])), - (&timeline_yet_another, Some(&[])), - ] as [(_, Option<&[_]>); 3], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyPoint::name(), - [ - (&timeline_frame, Some(&[])), - (&timeline_yet_another, Some(&[])), - ] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyColor::name(), - [ - (&timeline_frame, Some(&[])), - (&timeline_yet_another, Some(&[])), - ] as [(_, Option<&[_]>); 2], - ); - } - - Ok(()) -} - -/// Checks the state of the global time tracker (at the `EntityDb` level). -fn assert_times_per_timeline<'a>( - db: &EntityDb, - expected: impl IntoIterator)>, -) { - for (timeline, expected_times) in expected { - let times = db.times_per_timeline().get(timeline); - - if let Some(expected) = expected_times { - let times: BTreeSet<_> = times.unwrap().keys().copied().collect(); - let expected: BTreeSet<_> = expected - .iter() - .map(|&t| TimeInt::try_from(t).unwrap()) - .collect(); - similar_asserts::assert_eq!(expected, times); - } else { - assert!(times.is_none()); - } - } -} - -/// Checks the state of the per-EntityTree recursive time tracker. -fn assert_recursive_histogram<'a>( - db: &EntityDb, - expected: impl IntoIterator)>, -) { - for (timeline, expected_times) in expected { - let histo = db.time_histogram(timeline); - - if let Some(expected) = expected_times { - if expected.is_empty() { - assert!(histo.is_none()); - continue; - } - let histo = histo.unwrap(); - let ranges = histo.range(i64::MIN.., 0).collect::>(); - let expected: Vec<_> = expected.to_vec(); - similar_asserts::assert_eq!(expected, ranges); - } else { - assert!(histo.is_none()); - } - } -} - -/// Checks the state of the per-`EntityTree` per-`ComponentName` flat time tracker. -fn assert_histogram_for_component<'a>( - db: &EntityDb, - entity_path: &EntityPath, - component_name: ComponentName, - expected: impl IntoIterator)>, -) { - for (timeline, expected_times) in expected { - let histo = db - .tree() - .subtree(entity_path) - .and_then(|tree| tree.time_histogram_for_component(timeline, component_name)); - - if let Some(expected) = expected_times { - let expected: Vec<_> = expected.to_vec(); - if expected.is_empty() { - assert!(histo.is_none()); - continue; - } - let histo = histo.unwrap(); - let ranges = histo.range(i64::MIN.., 0).collect::>(); - similar_asserts::assert_eq!(expected, ranges); - } else { - assert!(histo.is_none()); - } - } -} diff --git a/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs b/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs index 62f6f16c1629..6f5ce1e1edef 100644 --- a/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs +++ b/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs @@ -91,7 +91,7 @@ impl ContextMenuAction for CollapseExpandAllAction { return; }; - subtree.visit_children_recursively(&mut |entity_path, _| { + subtree.visit_children_recursively(&mut |entity_path| { CollapseScope::BlueprintTree .data_result(*space_view_id, entity_path.clone()) .set_open(&ctx.egui_context, self.open()); @@ -108,7 +108,7 @@ impl ContextMenuAction for CollapseExpandAllAction { return; }; - subtree.visit_children_recursively(&mut |entity_path, _| { + subtree.visit_children_recursively(&mut |entity_path| { CollapseScope::StreamsTree .entity(entity_path.clone()) .set_open(&ctx.egui_context, self.open()); diff --git a/crates/viewer/re_data_ui/src/component.rs b/crates/viewer/re_data_ui/src/component.rs index 3b7f44ad5e46..a848428c3a09 100644 --- a/crates/viewer/re_data_ui/src/component.rs +++ b/crates/viewer/re_data_ui/src/component.rs @@ -72,36 +72,38 @@ impl<'a> DataUi for EntityLatestAtResults<'a> { // if the component is static, we display extra diagnostic information if self.results.is_static() { - if let Some(histogram) = db - .tree() - .subtree(&self.entity_path) - .and_then(|tree| tree.entity.components.get(&component_name)) - { - if histogram.num_static_messages() > 1 { - ui.label(ui.ctx().warning_text(format!( - "Static component value was overridden {} times", - histogram.num_static_messages().saturating_sub(1), - ))) - .on_hover_text( - "When a static component is logged multiple times, only the last value \ - is stored. Previously logged values are overwritten and not \ - recoverable.", - ); - } + let num_static_messages = db + .store() + .num_events_for_static_component(&self.entity_path, component_name); + if num_static_messages > 1 { + ui.label(ui.ctx().warning_text(format!( + "Static component value was overridden {} times", + num_static_messages.saturating_sub(1), + ))) + .on_hover_text( + "When a static component is logged multiple times, only the last value \ + is stored. Previously logged values are overwritten and not \ + recoverable.", + ); + } - let timeline_message_count = histogram.num_temporal_messages(); - if timeline_message_count > 0 { - ui.label(ui.ctx().error_text(format!( - "Static component has {} event{} logged on timelines", - timeline_message_count, - if timeline_message_count > 1 { "s" } else { "" } - ))) - .on_hover_text( - "Components should be logged either as static or on timelines, but \ - never both. Values for static components logged to timelines cannot be \ - displayed.", - ); - } + let timeline_message_count = + db.store().num_events_on_timeline_for_temporal_component( + &query.timeline(), + &self.entity_path, + component_name, + ); + if timeline_message_count > 0 { + ui.label(ui.ctx().error_text(format!( + "Static component has {} event{} logged on timelines", + timeline_message_count, + if timeline_message_count > 1 { "s" } else { "" } + ))) + .on_hover_text( + "Components should be logged either as static or on timelines, but \ + never both. Values for static components logged to timelines cannot be \ + displayed.", + ); } } } diff --git a/crates/viewer/re_data_ui/src/component_path.rs b/crates/viewer/re_data_ui/src/component_path.rs index 9161e27a09ab..4597468955a1 100644 --- a/crates/viewer/re_data_ui/src/component_path.rs +++ b/crates/viewer/re_data_ui/src/component_path.rs @@ -32,8 +32,11 @@ impl DataUi for ComponentPath { results: results.as_ref(), } .data_ui(ctx, ui, ui_layout, query, db); - } else if let Some(entity_tree) = ctx.recording().tree().subtree(entity_path) { - if entity_tree.entity.components.contains_key(component_name) { + } else if ctx.recording().tree().subtree(entity_path).is_some() { + if db + .store() + .entity_has_component(&query.timeline(), entity_path, component_name) + { ui.label(""); } else { ui.label(format!( diff --git a/crates/viewer/re_data_ui/src/instance_path.rs b/crates/viewer/re_data_ui/src/instance_path.rs index 509556267470..3f6975c0f9bc 100644 --- a/crates/viewer/re_data_ui/src/instance_path.rs +++ b/crates/viewer/re_data_ui/src/instance_path.rs @@ -81,7 +81,9 @@ impl DataUi for InstancePath { } let component_path = ComponentPath::new(entity_path.clone(), component_name); - let is_static = db.is_component_static(&component_path).unwrap_or_default(); + let is_static = db + .store() + .entity_has_static_component(entity_path, &component_name); let icon = if is_static { &re_ui::icons::COMPONENT_STATIC } else { diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index 1447130f798b..e7e0c9071362 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -291,11 +291,16 @@ pub fn instance_path_parts_buttons( .response } -fn entity_tree_stats_ui(ui: &mut egui::Ui, timeline: &Timeline, tree: &EntityTree) { +fn entity_tree_stats_ui( + ui: &mut egui::Ui, + timeline: &Timeline, + db: &re_entity_db::EntityDb, + tree: &EntityTree, +) { use re_format::format_bytes; // Show total bytes used in whole subtree - let total_bytes = tree.subtree.data_bytes(); + let total_bytes = db.store().size_of_entity_on_timeline(timeline, &tree.path); let subtree_caveat = if tree.children.is_empty() { "" @@ -310,14 +315,16 @@ fn entity_tree_stats_ui(ui: &mut egui::Ui, timeline: &Timeline, tree: &EntityTre let mut data_rate = None; // Try to estimate data-rate - if let Some(time_histogram) = tree.subtree.time_histogram.get(timeline) { + if db.store().entity_has_data_on_timeline(timeline, &tree.path) { // `num_events` is approximate - we could be logging a Tensor image and a transform // at _almost_ approximately the same time, but it should only count as one fence-post. - let num_events = time_histogram.total_count(); // TODO(emilk): we should ask the histogram to count the number of non-zero keys instead. + let num_events = db + .store() + .num_events_on_timeline_for_all_components(timeline, &tree.path); - if let (Some(min_time), Some(max_time)) = - (time_histogram.min_key(), time_histogram.max_key()) - { + if let Some(time_range) = db.store().time_range_for_entity(timeline, &tree.path) { + let min_time = time_range.min(); + let max_time = time_range.max(); if min_time < max_time && 1 < num_events { // Let's do our best to avoid fencepost errors. // If we log 1 MiB once every second, then after three @@ -328,9 +335,9 @@ fn entity_tree_stats_ui(ui: &mut egui::Ui, timeline: &Timeline, tree: &EntityTre // t: 0s 1s 2s // data: 1MiB 1MiB 1MiB - let duration = max_time - min_time; + let duration = max_time.as_f64() - min_time.as_f64(); - let mut bytes_per_time = total_bytes as f64 / duration as f64; + let mut bytes_per_time = total_bytes as f64 / duration; // Fencepost adjustment: bytes_per_time *= (num_events - 1) as f64 / num_events as f64; @@ -393,7 +400,10 @@ pub fn component_path_button_to( db: &re_entity_db::EntityDb, ) -> egui::Response { let item = Item::ComponentPath(component_path.clone()); - let is_static = db.is_component_static(component_path).unwrap_or_default(); + let is_static = db.store().entity_has_static_component( + component_path.entity_path(), + component_path.component_name(), + ); let icon = if is_static { &icons::COMPONENT_STATIC } else { @@ -548,7 +558,7 @@ pub fn instance_hover_card_ui( if instance_path.instance.is_all() { if let Some(subtree) = ctx.recording().tree().subtree(&instance_path.entity_path) { - entity_tree_stats_ui(ui, &query.timeline(), subtree); + entity_tree_stats_ui(ui, &query.timeline(), db, subtree); } } else { // TODO(emilk): per-component stats diff --git a/crates/viewer/re_selection_panel/src/selection_panel.rs b/crates/viewer/re_selection_panel/src/selection_panel.rs index c24f1306db7d..b2e3d701a7a8 100644 --- a/crates/viewer/re_selection_panel/src/selection_panel.rs +++ b/crates/viewer/re_selection_panel/src/selection_panel.rs @@ -155,9 +155,12 @@ impl SelectionPanel { match item { Item::ComponentPath(component_path) => { let entity_path = &component_path.entity_path; + let component_name = &component_path.component_name; let (query, db) = guess_query_and_db_for_selected_entity(ctx, entity_path); - let is_static = db.is_component_static(component_path).unwrap_or_default(); + let is_static = db + .store() + .entity_has_static_component(entity_path, component_name); ui.list_item_flat_noninteractive( PropertyContent::new("Component type").value_text(if is_static { @@ -718,7 +721,9 @@ fn item_tile( let component_name = &component_path.component_name; let (_query, db) = guess_query_and_db_for_selected_entity(ctx, entity_path); - let is_static = db.is_component_static(component_path).unwrap_or_default(); + let is_static = db + .store() + .entity_has_static_component(entity_path, component_name); Some( ItemTitle::new(component_name.short_name()) diff --git a/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs b/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs index ffe6d4ecab9d..b5b83cf06eda 100644 --- a/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs +++ b/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs @@ -326,7 +326,7 @@ fn create_entity_add_info( &space_view.space_origin, ); - tree.visit_children_recursively(&mut |entity_path, _| { + tree.visit_children_recursively(&mut |entity_path| { let can_add: CanAddToSpaceView = if visualizable_entities.iter().any(|(_, entities)| entities.contains(entity_path)) { CanAddToSpaceView::Compatible { diff --git a/crates/viewer/re_space_view_spatial/src/view_2d.rs b/crates/viewer/re_space_view_spatial/src/view_2d.rs index 3750b57e9eba..97a66e75f253 100644 --- a/crates/viewer/re_space_view_spatial/src/view_2d.rs +++ b/crates/viewer/re_space_view_spatial/src/view_2d.rs @@ -274,12 +274,15 @@ impl SpaceViewClass for SpatialSpaceView2D { fn count_non_nested_images_with_component( image_dimensions: &IntMap, entity_bucket: &IntSet, + entity_db: &re_entity_db::EntityDb, subtree: &EntityTree, component_name: &ComponentName, ) -> usize { if image_dimensions.contains_key(&subtree.path) { // bool true -> 1 - subtree.entity.components.contains_key(component_name) as usize + entity_db + .store() + .entity_has_component_on_any_timeline(&subtree.path, component_name) as usize } else if !entity_bucket .iter() .any(|e| e.is_descendant_of(&subtree.path)) @@ -293,6 +296,7 @@ fn count_non_nested_images_with_component( count_non_nested_images_with_component( image_dimensions, entity_bucket, + entity_db, child, component_name, ) @@ -361,6 +365,7 @@ fn recommended_space_views_with_image_splits( let image_count = count_non_nested_images_with_component( image_dimensions, entities, + ctx.recording(), subtree, &Image::indicator().name(), ); @@ -368,6 +373,7 @@ fn recommended_space_views_with_image_splits( let depth_count = count_non_nested_images_with_component( image_dimensions, entities, + ctx.recording(), subtree, &DepthImage::indicator().name(), ); diff --git a/crates/viewer/re_space_view_spatial/src/view_3d.rs b/crates/viewer/re_space_view_spatial/src/view_3d.rs index f0abd2087ebe..18cc5117660e 100644 --- a/crates/viewer/re_space_view_spatial/src/view_3d.rs +++ b/crates/viewer/re_space_view_spatial/src/view_3d.rs @@ -279,8 +279,12 @@ impl SpaceViewClass for SpatialSpaceView3D { // There's also a strong argument to be made that ViewCoordinates implies a 3D space, thus changing the SpacialTopology accordingly! ctx.recording() .tree() - .visit_children_recursively(&mut |path, info| { - if info.components.contains_key(&ViewCoordinates::name()) { + .visit_children_recursively(&mut |path| { + if ctx + .recording() + .store() + .entity_has_component_on_any_timeline(path, &ViewCoordinates::name()) + { indicated_entities.insert(path.clone()); } }); diff --git a/crates/viewer/re_time_panel/src/data_density_graph.rs b/crates/viewer/re_time_panel/src/data_density_graph.rs index 63b8b426a98e..5be1bc097a02 100644 --- a/crates/viewer/re_time_panel/src/data_density_graph.rs +++ b/crates/viewer/re_time_panel/src/data_density_graph.rs @@ -716,7 +716,7 @@ fn visit_relevant_chunks( visitor(Arc::clone(&chunk), chunk_timeline.time_range(), num_events); } } else if let Some(subtree) = db.tree().subtree(entity_path) { - subtree.visit_children_recursively(&mut |entity_path, _| { + subtree.visit_children_recursively(&mut |entity_path| { for chunk in db .store() .range_relevant_chunks_for_all_components(&query, entity_path) diff --git a/crates/viewer/re_time_panel/src/lib.rs b/crates/viewer/re_time_panel/src/lib.rs index 2fa898363b44..5a2321ddd2ab 100644 --- a/crates/viewer/re_time_panel/src/lib.rs +++ b/crates/viewer/re_time_panel/src/lib.rs @@ -19,7 +19,7 @@ use egui::{pos2, Color32, CursorIcon, NumExt, Painter, PointerButton, Rect, Shap use re_context_menu::{context_menu_ui_for_item, SelectionUpdateBehavior}; use re_data_ui::DataUi as _; use re_data_ui::{item_ui::guess_instance_path_icon, sorted_component_list_for_ui}; -use re_entity_db::{EntityTree, InstancePath, TimeHistogram}; +use re_entity_db::{EntityTree, InstancePath}; use re_log_types::{ external::re_types_core::ComponentName, ComponentPath, EntityPath, EntityPathPart, ResolvedTimeRange, TimeInt, TimeReal, @@ -574,8 +574,6 @@ impl TimePanel { ui: &mut egui::Ui, show_root_as: &str, ) { - let tree_has_data_in_current_timeline = time_ctrl.tree_has_data_in_current_timeline(tree); - let db = match self.source { TimePanelSource::Recording => ctx.recording(), TimePanelSource::Blueprint => ctx.store_context.blueprint, @@ -693,6 +691,9 @@ impl TimePanel { // ---------------------------------------------- // show the data in the time area: + let tree_has_data_in_current_timeline = entity_db + .store() + .entity_has_data_on_timeline(time_ctrl.timeline(), &tree.path); if is_visible && tree_has_data_in_current_timeline { let row_rect = Rect::from_x_y_ranges(time_area_response.rect.x_range(), response_rect.y_range()); @@ -744,17 +745,29 @@ impl TimePanel { } // If this is an entity: - if !tree.entity.components.is_empty() { - for component_name in sorted_component_list_for_ui(tree.entity.components.keys()) { - let data = &tree.entity.components[&component_name]; - - let is_static = data.is_static(); - let component_has_data_in_current_timeline = - time_ctrl.component_has_data_in_current_timeline(data); + if let Some(components) = entity_db + .store() + .all_components_on_all_timelines(&tree.path) + { + for component_name in sorted_component_list_for_ui(components.iter()) { + let is_static = entity_db + .store() + .entity_has_static_component(&tree.path, &component_name); let component_path = ComponentPath::new(tree.path.clone(), component_name); let short_component_name = component_path.component_name.short_name(); let item = TimePanelItem::component_path(component_path.clone()); + let timeline = time_ctrl.timeline(); + + let component_has_data_in_current_timeline = entity_db + .store() + .entity_has_component(time_ctrl.timeline(), &tree.path, &component_name); + + let total_num_messages = entity_db.store().num_events_on_timeline_for_component( + time_ctrl.timeline(), + &tree.path, + component_name, + ); let response = ui .list_item() @@ -786,14 +799,6 @@ impl TimePanel { let response_rect = response.rect; - let timeline = time_ctrl.timeline(); - - let empty_messages_over_time = TimeHistogram::default(); - let messages_over_time = data.get(timeline).unwrap_or(&empty_messages_over_time); - - // `data.times` does not contain static. Need to add those manually: - let total_num_messages = - messages_over_time.total_count() + data.num_static_messages(); response.on_hover_ui(|ui| { if total_num_messages == 0 { ui.label(ui.ctx().warning_text(format!( diff --git a/crates/viewer/re_viewer_context/src/time_control.rs b/crates/viewer/re_viewer_context/src/time_control.rs index cb003750917c..bd1651944c0e 100644 --- a/crates/viewer/re_viewer_context/src/time_control.rs +++ b/crates/viewer/re_viewer_context/src/time_control.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use re_entity_db::{EntityTree, TimeCounts, TimeHistogramPerTimeline, TimesPerTimeline}; +use re_entity_db::{TimeCounts, TimesPerTimeline}; use re_log_types::{ Duration, ResolvedTimeRange, ResolvedTimeRangeF, TimeInt, TimeReal, TimeType, Timeline, }; @@ -571,22 +571,6 @@ impl TimeControl { state.view = None; } } - - /// Returns whether the given tree has any data logged in the current timeline, - /// or has any timeless messages. - pub fn tree_has_data_in_current_timeline(&self, tree: &EntityTree) -> bool { - let top_time_histogram = &tree.subtree.time_histogram; - top_time_histogram.has_timeline(self.timeline()) - || top_time_histogram.num_static_messages() > 0 - } - - /// Returns whether the given component has any data logged in the current timeline. - pub fn component_has_data_in_current_timeline( - &self, - component_stat: &TimeHistogramPerTimeline, - ) -> bool { - component_stat.has_timeline(self.timeline()) - } } fn min(values: &TimeCounts) -> TimeInt { diff --git a/crates/viewer/re_viewport_blueprint/src/space_view.rs b/crates/viewer/re_viewport_blueprint/src/space_view.rs index c3df969c0ca5..8e03cb3800fa 100644 --- a/crates/viewer/re_viewport_blueprint/src/space_view.rs +++ b/crates/viewer/re_viewport_blueprint/src/space_view.rs @@ -251,7 +251,7 @@ impl SpaceViewBlueprint { // Create pending write operations to duplicate the entire subtree // TODO(jleibs): This should be a helper somewhere. if let Some(tree) = blueprint.tree().subtree(¤t_path) { - tree.visit_children_recursively(&mut |path, info| { + tree.visit_children_recursively(&mut |path| { let sub_path: EntityPath = new_path .iter() .chain(&path[current_path.len()..]) @@ -262,8 +262,11 @@ impl SpaceViewBlueprint { .with_row( RowId::new(), store_context.blueprint_timepoint_for_writes(), - info.components - .keys() + blueprint + .store() + .all_components(&query.timeline(), path) + .into_iter() + .flat_map(|v| v.into_iter()) // It's important that we don't include the SpaceViewBlueprint's components // since those will be updated separately and may contain different data. .filter(|component| { @@ -271,7 +274,7 @@ impl SpaceViewBlueprint { || !blueprint_archetypes::SpaceViewBlueprint::all_components() .contains(component) }) - .filter_map(|&component_name| { + .filter_map(|component_name| { let results = blueprint.query_caches().latest_at( blueprint.store(), query, diff --git a/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs b/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs index b13c988a7f71..283d78186621 100644 --- a/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs +++ b/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs @@ -428,7 +428,11 @@ impl DataQueryPropertyResolver<'_> { if let Some(recursive_override_subtree) = blueprint.tree().subtree(&recursive_override_path) { - for &component_name in recursive_override_subtree.entity.components.keys() { + for component_name in blueprint + .store() + .all_components_on_all_timelines(&recursive_override_subtree.path) + .unwrap_or_default() + { let results = blueprint.query_caches().latest_at( blueprint.store(), blueprint_query, @@ -459,7 +463,11 @@ impl DataQueryPropertyResolver<'_> { if let Some(individual_override_subtree) = blueprint.tree().subtree(&individual_override_path) { - for &component_name in individual_override_subtree.entity.components.keys() { + for component_name in blueprint + .store() + .all_components_on_all_timelines(&individual_override_subtree.path) + .unwrap_or_default() + { let results = blueprint.query_caches().latest_at( blueprint.store(), blueprint_query, From e3a252b1ec1c1b9943ea4db2a4adfcae7c22042c Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 10:28:06 +0200 Subject: [PATCH 03/43] Revert "wip" This reverts commit 0534aa019a3857f1820445c68ab95190bffbe1a9. --- crates/store/re_chunk_store/src/query.rs | 307 -------- crates/store/re_entity_db/src/entity_db.rs | 30 +- crates/store/re_entity_db/src/entity_tree.rs | 195 ++++- .../src/time_histogram_per_timeline.rs | 25 +- crates/store/re_entity_db/tests/clear.rs | 4 +- .../re_entity_db/tests/time_histograms.rs | 705 ++++++++++++++++++ .../src/actions/collapse_expand_all.rs | 4 +- crates/viewer/re_data_ui/src/component.rs | 60 +- .../viewer/re_data_ui/src/component_path.rs | 7 +- crates/viewer/re_data_ui/src/instance_path.rs | 4 +- crates/viewer/re_data_ui/src/item_ui.rs | 32 +- .../re_selection_panel/src/selection_panel.rs | 9 +- .../src/space_view_entity_picker.rs | 2 +- .../re_space_view_spatial/src/view_2d.rs | 8 +- .../re_space_view_spatial/src/view_3d.rs | 8 +- .../re_time_panel/src/data_density_graph.rs | 2 +- crates/viewer/re_time_panel/src/lib.rs | 41 +- .../re_viewer_context/src/time_control.rs | 18 +- .../re_viewport_blueprint/src/space_view.rs | 11 +- .../src/space_view_contents.rs | 12 +- 20 files changed, 1006 insertions(+), 478 deletions(-) create mode 100644 crates/store/re_entity_db/tests/time_histograms.rs diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 8ff96c12b17b..4ebe623f1f31 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -5,9 +5,7 @@ use std::{ use itertools::Itertools; use re_chunk::{Chunk, LatestAtQuery, RangeQuery}; -use re_log_types::ResolvedTimeRange; use re_log_types::{EntityPath, TimeInt, Timeline}; -use re_types_core::SizeBytes as _; use re_types_core::{ComponentName, ComponentNameSet}; use crate::{store::ChunkIdSetPerTime, ChunkStore}; @@ -63,47 +61,6 @@ impl ChunkStore { } } - /// Retrieve all the [`ComponentName`]s that have been written to for a given [`EntityPath`]. - /// - /// Static components are always included in the results. - /// - /// Returns `None` if the entity has never had any data logged to it. - pub fn all_components_on_all_timelines( - &self, - entity_path: &EntityPath, - ) -> Option { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - let static_components: Option = self - .static_chunk_ids_per_entity - .get(entity_path) - .map(|static_chunks_per_component| { - static_chunks_per_component.keys().copied().collect() - }); - - let temporal_components: Option = self - .temporal_chunk_ids_per_entity_per_component - .get(entity_path) - .map(|temporal_chunk_ids_per_timeline| { - temporal_chunk_ids_per_timeline - .iter() - .flat_map(|(_, temporal_chunk_ids_per_component)| { - temporal_chunk_ids_per_component.keys().copied() - }) - .collect() - }); - - match (static_components, temporal_components) { - (None, None) => None, - (None, comps @ Some(_)) | (comps @ Some(_), None) => comps, - (Some(static_comps), Some(temporal_comps)) => { - Some(static_comps.into_iter().chain(temporal_comps).collect()) - } - } - } - /// Check whether a given entity has a specific [`ComponentName`] either on the specified /// timeline, or in its static data. #[inline] @@ -118,56 +75,6 @@ impl ChunkStore { .map_or(false, |components| components.contains(component_name)) } - pub fn entity_has_component_on_any_timeline( - &self, - entity_path: &EntityPath, - component_name: &ComponentName, - ) -> bool { - re_tracing::profile_function!(); - - if self - .static_chunk_ids_per_entity - .get(entity_path) - .and_then(|static_chunks_per_component| static_chunks_per_component.get(component_name)) - .and_then(|id| self.chunks_per_chunk_id.get(id)) - .is_some() - { - return true; - } - - for temporal_chunk_ids_per_component in self - .temporal_chunk_ids_per_entity_per_component - .get(entity_path) - .iter() - .flat_map(|temporal_chunk_ids_per_timeline| temporal_chunk_ids_per_timeline.values()) - { - if temporal_chunk_ids_per_component - .get(component_name) - .is_some() - { - return true; - } - } - - false - } - - /// Returns true if the given entity has a specific component with static data. - #[inline] - pub fn entity_has_static_component( - &self, - entity_path: &EntityPath, - component_name: &ComponentName, - ) -> bool { - re_tracing::profile_function!(); - - let Some(static_components) = self.static_chunk_ids_per_entity.get(entity_path) else { - return false; - }; - - static_components.contains_key(component_name) - } - /// Find the earliest time at which something was logged for a given entity on the specified /// timeline. /// @@ -506,217 +413,3 @@ impl ChunkStore { .collect() } } - -// Queries returning `usize` and `bool` -impl ChunkStore { - /// Returns the number of events logged for a given component on the given entity path across all timelines. - pub fn num_events_on_timeline_for_component( - &self, - timeline: &Timeline, - entity_path: &EntityPath, - component_name: ComponentName, - ) -> usize { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - self.num_events_for_static_component(entity_path, component_name) - + self.num_events_on_timeline_for_temporal_component( - timeline, - entity_path, - component_name, - ) - } - - pub fn time_range_for_entity( - &self, - timeline: &Timeline, - entity_path: &EntityPath, - ) -> Option { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - let temporal_chunk_ids_per_timeline = - self.temporal_chunk_ids_per_entity.get(entity_path)?; - let chunk_id_sets = temporal_chunk_ids_per_timeline.get(timeline)?; - - let start = chunk_id_sets.per_start_time.keys().min()?; - let end = chunk_id_sets.per_end_time.keys().max()?; - - Some(ResolvedTimeRange::new(*start, *end)) - } - - pub fn num_events_on_timeline_for_all_components( - &self, - timeline: &Timeline, - entity_path: &EntityPath, - ) -> usize { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - let mut total_events = 0; - - if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) - { - for chunk in static_chunks_per_component - .values() - .filter_map(|id| self.chunks_per_chunk_id.get(id)) - { - total_events += chunk.num_events_cumulative(); - } - } - - if let Some(chunk_ids) = self - .temporal_chunk_ids_per_entity - .get(entity_path) - .and_then(|temporal_chunks_events_per_timeline| { - temporal_chunks_events_per_timeline.get(timeline) - }) - { - for chunk in chunk_ids - .per_start_time - .values() - .flat_map(|ids| ids.iter().filter_map(|id| self.chunks_per_chunk_id.get(id))) - { - total_events += chunk.num_events_cumulative(); - } - } - - total_events - } - - pub fn entity_has_data_on_timeline( - &self, - timeline: &Timeline, - entity_path: &EntityPath, - ) -> bool { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - if self.static_chunk_ids_per_entity.get(entity_path).is_some() { - // Static data exists on all timelines - return true; - } - - if let Some(temporal_chunk_ids_per_timeline) = - self.temporal_chunk_ids_per_entity.get(entity_path) - { - if temporal_chunk_ids_per_timeline.contains_key(timeline) { - return true; - } - } - - false - } - - pub fn num_events_for_static_component( - &self, - entity_path: &EntityPath, - component_name: ComponentName, - ) -> usize { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - if let Some(static_chunk) = self - .static_chunk_ids_per_entity - .get(entity_path) - .and_then(|static_chunks_per_component| { - static_chunks_per_component.get(&component_name) - }) - .and_then(|chunk_id| self.chunks_per_chunk_id.get(chunk_id)) - { - // Static data overrides temporal data - static_chunk - .num_events_for_component(component_name) - .unwrap_or(0) - } else { - 0 - } - } - - /// Returns the number of events logged for a given component on the given entity path across all timelines. - /// - /// This ignores static data. - pub fn num_events_on_timeline_for_temporal_component( - &self, - timeline: &Timeline, - entity_path: &EntityPath, - component_name: ComponentName, - ) -> usize { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - let Some(temporal_chunk_ids_per_timeline) = self - .temporal_chunk_ids_per_entity_per_component - .get(entity_path) - else { - return 0; // no events logged for the entity path - }; - - let Some(temporal_chunk_ids_per_component) = temporal_chunk_ids_per_timeline.get(timeline) - else { - return 0; // no events logged on this timeline - }; - - let Some(chunk_ids) = temporal_chunk_ids_per_component.get(&component_name) else { - return 0; // no events logged for the component on this timeline - }; - - let mut num_events = 0; - for chunk in chunk_ids - .per_start_time - .values() - .flat_map(|ids| ids.iter().filter_map(|id| self.chunks_per_chunk_id.get(id))) - { - num_events += chunk.num_events_for_component(component_name).unwrap_or(0); - } - - num_events - } - - pub fn size_of_entity_on_timeline( - &self, - timeline: &Timeline, - entity_path: &EntityPath, - ) -> usize { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - let mut total_size = 0; - - if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) - { - for chunk in static_chunks_per_component - .values() - .filter_map(|id| self.chunks_per_chunk_id.get(id)) - { - total_size += Chunk::total_size_bytes(chunk) as usize; - } - } - - if let Some(chunk_id_sets) = self - .temporal_chunk_ids_per_entity - .get(entity_path) - .and_then(|temporal_chunk_ids_per_timeline| { - temporal_chunk_ids_per_timeline.get(timeline) - }) - { - for chunk in chunk_id_sets - .per_start_time - .values() - .flat_map(|v| v.iter()) - .filter_map(|id| self.chunks_per_chunk_id.get(id)) - { - total_size += Chunk::total_size_bytes(chunk) as usize; - } - } - - total_size - } -} diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index 0cb02c8b9453..50231342cf77 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -9,8 +9,8 @@ use re_chunk_store::{ GarbageCollectionTarget, }; use re_log_types::{ - ApplicationId, EntityPath, EntityPathHash, LogMsg, ResolvedTimeRange, ResolvedTimeRangeF, - SetStoreInfo, StoreId, StoreInfo, StoreKind, Timeline, + ApplicationId, ComponentPath, EntityPath, EntityPathHash, LogMsg, ResolvedTimeRange, + ResolvedTimeRangeF, SetStoreInfo, StoreId, StoreInfo, StoreKind, Timeline, }; use crate::{Error, TimesPerTimeline}; @@ -56,9 +56,6 @@ pub struct EntityDb { /// A tree-view (split on path components) of the entities. tree: crate::EntityTree, - /// Holds a [`TimeHistogram`][`crate::TimeHistogram`] spanning the whole entity tree for each [`Timeline`]. - time_histogram_per_timeline: crate::TimeHistogramPerTimeline, - /// Stores all components for all entities for all timelines. data_store: ChunkStore, @@ -88,7 +85,6 @@ impl EntityDb { entity_path_from_hash: Default::default(), times_per_timeline: Default::default(), tree: crate::EntityTree::root(), - time_histogram_per_timeline: Default::default(), data_store, resolver: re_query::PromiseResolver::default(), query_caches, @@ -271,7 +267,25 @@ impl EntityDb { /// Histogram of all events on the timeeline, of all entities. pub fn time_histogram(&self, timeline: &Timeline) -> Option<&crate::TimeHistogram> { - self.time_histogram_per_timeline.get(timeline) + self.tree().subtree.time_histogram.get(timeline) + } + + /// Total number of static messages for any entity. + pub fn num_static_messages(&self) -> u64 { + self.tree.num_static_messages_recursive() + } + + /// Returns whether a component is static. + pub fn is_component_static(&self, component_path: &ComponentPath) -> Option { + if let Some(entity_tree) = self.tree().subtree(component_path.entity_path()) { + entity_tree + .entity + .components + .get(&component_path.component_name) + .map(|component_histogram| component_histogram.is_static()) + } else { + None + } } #[inline] @@ -437,7 +451,6 @@ impl EntityDb { entity_path_from_hash: _, times_per_timeline, tree, - time_histogram_per_timeline, data_store: _, resolver: _, query_caches, @@ -446,7 +459,6 @@ impl EntityDb { times_per_timeline.on_events(store_events); query_caches.on_events(store_events); - time_histogram_per_timeline.on_events(store_events); tree.on_store_deletions(store_events); } diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index d19ecb3158b5..b25ce4800f0f 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -6,13 +6,15 @@ use nohash_hasher::IntMap; use re_chunk::RowId; use re_chunk_store::{ChunkStoreDiff, ChunkStoreDiffKind, ChunkStoreEvent, ChunkStoreSubscriber}; -use re_log_types::{EntityPath, EntityPathHash, EntityPathPart, TimeInt, Timeline}; +use re_log_types::{ComponentPath, EntityPath, EntityPathHash, EntityPathPart, TimeInt, Timeline}; use re_types_core::ComponentName; // Used all over in docstrings. #[allow(unused_imports)] use re_chunk_store::ChunkStore; +use crate::TimeHistogramPerTimeline; + // ---------------------------------------------------------------------------- /// A recursive, manually updated [`ChunkStoreSubscriber`] that maintains the entity hierarchy. @@ -24,6 +26,12 @@ pub struct EntityTree { /// Direct descendants of this (sub)tree. pub children: BTreeMap, + + /// Information about this specific entity (excluding children). + pub entity: EntityInfo, + + /// Info about this subtree, including all children, recursively. + pub subtree: SubtreeInfo, } // NOTE: This is only to let people know that this is in fact a [`ChunkStoreSubscriber`], so they A) don't try @@ -49,6 +57,86 @@ impl ChunkStoreSubscriber for EntityTree { } } +/// Information about this specific entity (excluding children). +#[derive(Default)] +pub struct EntityInfo { + /// Flat time histograms for each component of this [`EntityTree`]. + /// + /// Keeps track of the _number of times a component is logged_ per time per timeline, only for + /// this specific [`EntityTree`]. + /// A component logged twice at the same timestamp is counted twice. + /// + /// ⚠ Auto-generated instance keys are _not_ accounted for. ⚠ + pub components: BTreeMap, +} + +/// Info about stuff at a given [`EntityPath`], including all of its children, recursively. +#[derive(Default)] +pub struct SubtreeInfo { + /// Recursive time histogram for this [`EntityTree`]. + /// + /// Keeps track of the _number of components logged_ per time per timeline, recursively across + /// all of the [`EntityTree`]'s children. + /// A component logged twice at the same timestamp is counted twice. + /// + /// ⚠ Auto-generated instance keys are _not_ accounted for. ⚠ + pub time_histogram: TimeHistogramPerTimeline, + + /// Number of bytes used by all arrow data + data_bytes: u64, +} + +impl SubtreeInfo { + /// Assumes the event has been filtered to be part of this subtree. + fn on_event(&mut self, event: &ChunkStoreEvent) { + use re_types_core::SizeBytes as _; + + match event.kind { + ChunkStoreDiffKind::Addition => { + let times = event + .chunk + .timelines() + .iter() + .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) + .collect_vec(); + self.time_histogram.add(×, event.num_components() as _); + + self.data_bytes += event.chunk.total_size_bytes(); + } + ChunkStoreDiffKind::Deletion => { + let times = event + .chunk + .timelines() + .iter() + .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) + .collect_vec(); + self.time_histogram + .remove(×, event.num_components() as _); + + let removed_bytes = event.chunk.total_size_bytes(); + self.data_bytes + .checked_sub(removed_bytes) + .unwrap_or_else(|| { + re_log::debug!( + store_id = %event.store_id, + entity_path = %event.chunk.entity_path(), + current = self.data_bytes, + removed = removed_bytes, + "book keeping underflowed" + ); + u64::MIN + }); + } + } + } + + /// Number of bytes used by all arrow data in this tree (including their schemas, but otherwise ignoring book-keeping overhead). + #[inline] + pub fn data_bytes(&self) -> u64 { + self.data_bytes + } +} + /// Maintains an optimized representation of a batch of [`ChunkStoreEvent`]s specifically designed to /// accelerate garbage collection of [`EntityTree`]s. /// @@ -118,6 +206,8 @@ impl EntityTree { Self { path, children: Default::default(), + entity: Default::default(), + subtree: Default::default(), } } @@ -126,8 +216,24 @@ impl EntityTree { self.children.is_empty() } - pub fn num_children(&self) -> usize { - self.children.len() + pub fn num_children_and_fields(&self) -> usize { + self.children.len() + self.entity.components.len() + } + + /// Number of timeless messages in this tree, or any child, recursively. + pub fn num_static_messages_recursive(&self) -> u64 { + self.subtree.time_histogram.num_static_messages() + } + + pub fn time_histogram_for_component( + &self, + timeline: &Timeline, + component_name: impl Into, + ) -> Option<&crate::TimeHistogram> { + self.entity + .components + .get(&component_name.into()) + .and_then(|per_timeline| per_timeline.get(timeline)) } /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s. @@ -150,12 +256,14 @@ impl EntityTree { // Book-keeping for each level in the hierarchy: let mut tree = self; + tree.subtree.on_event(event); for (i, part) in entity_path.iter().enumerate() { tree = tree .children .entry(part.clone()) .or_insert_with(|| Self::new(entity_path.as_slice()[..=i].into())); + tree.subtree.on_event(event); } // Finally book-keeping for the entity where data was actually added: @@ -164,7 +272,25 @@ impl EntityTree { /// Handles the addition of new data into the tree. fn on_added_data(&mut self, store_diff: &ChunkStoreDiff) { - let _ = (self, store_diff); + for component_name in store_diff.chunk.component_names() { + let component_path = + ComponentPath::new(store_diff.chunk.entity_path().clone(), component_name); + + let per_component = self + .entity + .components + .entry(component_path.component_name) + .or_default(); + per_component.add( + &store_diff + .chunk + .timelines() + .iter() + .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) + .collect_vec(), + 1, + ); + } } /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s. @@ -173,7 +299,12 @@ impl EntityTree { pub fn on_store_deletions(&mut self, store_events: &[ChunkStoreEvent]) { re_tracing::profile_function!(); - let Self { path, children } = self; + let Self { + path, + children, + entity, + subtree, + } = self; // Only keep events relevant to this branch of the tree. let subtree_events = store_events @@ -183,29 +314,61 @@ impl EntityTree { .cloned() .collect_vec(); + { + re_tracing::profile_scope!("entity"); + for event in subtree_events + .iter() + .filter(|e| e.chunk.entity_path() == path) + { + for component_name in event.chunk.component_names() { + if let Some(histo) = entity.components.get_mut(&component_name) { + histo.remove( + &event + .chunk + .timelines() + .iter() + .map(|(timeline, time_chunk)| (*timeline, time_chunk.times_raw())) + .collect_vec(), + 1, + ); + if histo.is_empty() { + entity.components.remove(&component_name); + } + } + } + } + } + + { + re_tracing::profile_scope!("subtree"); + for event in &subtree_events { + subtree.on_event(event); + } + } + children.retain(|_, child| { child.on_store_deletions(&subtree_events); - child.num_children() > 0 + child.num_children_and_fields() > 0 }); } pub fn subtree(&self, path: &EntityPath) -> Option<&Self> { - let mut this = self; - let mut path = path.as_slice(); - loop { + fn subtree_recursive<'tree>( + this: &'tree EntityTree, + path: &[EntityPathPart], + ) -> Option<&'tree EntityTree> { match path { - [] => return Some(this), - [first, rest @ ..] => { - this = this.children.get(first)?; - path = rest; - } + [] => Some(this), + [first, rest @ ..] => subtree_recursive(this.children.get(first)?, rest), } } + + subtree_recursive(self, path.as_slice()) } // Invokes visitor for `self` and all children recursively. - pub fn visit_children_recursively(&self, visitor: &mut impl FnMut(&EntityPath)) { - visitor(&self.path); + pub fn visit_children_recursively(&self, visitor: &mut impl FnMut(&EntityPath, &EntityInfo)) { + visitor(&self.path, &self.entity); for child in self.children.values() { child.visit_children_recursively(visitor); } diff --git a/crates/store/re_entity_db/src/time_histogram_per_timeline.rs b/crates/store/re_entity_db/src/time_histogram_per_timeline.rs index 3fde33578e3f..ebb06767a4c6 100644 --- a/crates/store/re_entity_db/src/time_histogram_per_timeline.rs +++ b/crates/store/re_entity_db/src/time_histogram_per_timeline.rs @@ -1,7 +1,5 @@ use std::collections::BTreeMap; -use itertools::Itertools as _; -use re_chunk_store::ChunkStoreDiffKind; use re_chunk_store::{ChunkStoreEvent, ChunkStoreSubscriber}; use re_log_types::Timeline; @@ -115,6 +113,8 @@ impl TimeHistogramPerTimeline { } } +// NOTE: This is only to let people know that this is in fact a [`ChunkStoreSubscriber`], so they A) don't try +// to implement it on their own and B) don't try to register it. impl ChunkStoreSubscriber for TimeHistogramPerTimeline { #[inline] fn name(&self) -> String { @@ -132,22 +132,9 @@ impl ChunkStoreSubscriber for TimeHistogramPerTimeline { } #[allow(clippy::unimplemented)] - fn on_events(&mut self, events: &[ChunkStoreEvent]) { - for event in events { - let times = event - .chunk - .timelines() - .iter() - .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) - .collect_vec(); - match event.kind { - ChunkStoreDiffKind::Addition => { - self.add(×, event.num_components() as _); - } - ChunkStoreDiffKind::Deletion => { - self.remove(×, event.num_components() as _); - } - } - } + fn on_events(&mut self, _events: &[ChunkStoreEvent]) { + unimplemented!( + r"TimeHistogramPerTimeline view is maintained as a sub-view of `EntityTree`", + ); } } diff --git a/crates/store/re_entity_db/tests/clear.rs b/crates/store/re_entity_db/tests/clear.rs index 3de66f176d17..5f3e32f45745 100644 --- a/crates/store/re_entity_db/tests/clear.rs +++ b/crates/store/re_entity_db/tests/clear.rs @@ -414,7 +414,7 @@ fn clear_and_gc() -> anyhow::Result<()> { // Insert a component, then clear it, then GC. { // EntityTree is Empty when we start - assert_eq!(db.tree().num_children(), 0); + assert_eq!(db.tree().num_children_and_fields(), 0); let point = MyPoint::new(1.0, 2.0); @@ -450,7 +450,7 @@ fn clear_and_gc() -> anyhow::Result<()> { assert_eq!(stats.temporal_chunks.total_num_rows, 0); // EntityTree should be empty again when we end since everything was GC'd - assert_eq!(db.tree().num_children(), 0); + assert_eq!(db.tree().num_children_and_fields(), 0); } Ok(()) diff --git a/crates/store/re_entity_db/tests/time_histograms.rs b/crates/store/re_entity_db/tests/time_histograms.rs new file mode 100644 index 000000000000..a83fd10b8eec --- /dev/null +++ b/crates/store/re_entity_db/tests/time_histograms.rs @@ -0,0 +1,705 @@ +// https://github.com/rust-lang/rust-clippy/issues/10011 +#![cfg(test)] + +use std::{collections::BTreeSet, sync::Arc}; + +use re_chunk::{Chunk, ChunkId, RowId}; +use re_chunk_store::GarbageCollectionOptions; +use re_entity_db::EntityDb; +use re_int_histogram::RangeI64; +use re_log_types::{ + example_components::{MyColor, MyIndex, MyPoint}, + EntityPath, StoreId, TimeInt, TimePoint, Timeline, +}; +use re_types_core::{components::ClearIsRecursive, ComponentName, Loggable}; + +// --- + +// Complete test suite for our various time histograms which, among other things, power the +// timeline widget. +#[test] +fn time_histograms() -> anyhow::Result<()> { + let mut db = EntityDb::new(StoreId::random(re_log_types::StoreKind::Recording)); + + let timeline_frame = Timeline::new_sequence("frame"); + let timeline_other = Timeline::new_temporal("other"); + let timeline_yet_another = Timeline::new_sequence("yet_another"); + + let entity_parent: EntityPath = "parent".into(); + let entity_grandchild: EntityPath = "parent/child/grandchild".into(); + let entity_unrelated: EntityPath = "very/unrelated".into(); + + // Single top-level entity, explicitly logged `MyIndex`s. + { + let chunk = Chunk::builder(entity_parent.clone()) + .with_component_batches( + RowId::new(), + TimePoint::from_iter([ + (timeline_frame, 42), // + (timeline_other, 666), // + (timeline_yet_another, 1), // + ]), + [&MyIndex::from_iter(0..10) as _], + ) + .build()?; + + db.add_chunk(&Arc::new(chunk))?; + + // times per timeline + assert_times_per_timeline( + &db, + [ + (&Timeline::log_time(), Some(&[] as &[i64])), + (&timeline_frame, Some(&[42])), + (&timeline_other, Some(&[666])), + (&timeline_yet_another, Some(&[1])), + ], + ); + + // histograms per timeline + assert_recursive_histogram( + &db, + [ + (&Timeline::log_time(), None), + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), + (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), + ] as [(_, Option<&[_]>); 4], + ); + + // histograms per component per timeline ('parent') + assert_histogram_for_component( + &db, + &entity_parent, + MyIndex::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), + (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), + ] as [(_, Option<&[_]>); 3], + ); + } + + // Grand-child, multiple components, auto-generated `MyIndex`s. + { + let chunk = { + let num_instances = 3; + let points: Vec<_> = (0..num_instances) + .map(|i| MyPoint::new(0.0, i as f32)) + .collect(); + let colors = vec![MyColor::from(0xFF0000FF)]; + Chunk::builder(entity_grandchild.clone()) + .with_component_batches( + RowId::new(), + TimePoint::from_iter([ + (timeline_frame, 42), // + (timeline_yet_another, 1), // + ]), + [&points as _, &colors as _], + ) + .build()? + }; + let chunk = Arc::new(chunk); + + db.add_chunk(&chunk)?; + + assert_times_per_timeline( + &db, + [ + (&Timeline::log_time(), Some(&[] as &[i64])), + (&timeline_frame, Some(&[42])), + (&timeline_other, Some(&[666])), + (&timeline_yet_another, Some(&[1])), + ], + ); + + // histograms per timeline + // NOTE: notice how autogenerated instance keys are ignored. + assert_recursive_histogram( + &db, + [ + (&Timeline::log_time(), None), + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 3)])), + (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 3)])), + ] as [(_, Option<&[_]>); 4], + ); + + // histograms per component per timeline ('parent') + assert_histogram_for_component( + &db, + &entity_parent, + MyIndex::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), + (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), + ] as [(_, Option<&[_]>); 3], + ); + // NOTE: per-component histograms are not recursive! + assert_histogram_for_component( + &db, + &entity_parent, + MyPoint::name(), + [ + (&timeline_frame, None), + (&timeline_other, None), + (&timeline_yet_another, None), + ] as [(_, Option<&[_]>); 3], + ); + assert_histogram_for_component( + &db, + &entity_parent, + MyColor::name(), + [ + (&timeline_frame, None), + (&timeline_other, None), + (&timeline_yet_another, None), + ] as [(_, Option<&[_]>); 3], + ); + + // histograms per component per timeline ('parent/child/grandchild') + // NOTE: notice how autogenerated instance keys are ignored! + assert_histogram_for_component( + &db, + &entity_grandchild, + MyIndex::name(), + [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyPoint::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), + ] as [(_, Option<&[_]>); 2], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyColor::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), + ] as [(_, Option<&[_]>); 2], + ); + + db.add_chunk(&Arc::new(chunk.clone_as(ChunkId::new(), RowId::new())))?; // same chunk a second time! + + // times per timeline + assert_times_per_timeline( + &db, + [ + (&Timeline::log_time(), Some(&[] as &[i64])), + (&timeline_frame, Some(&[42])), + (&timeline_other, Some(&[666])), + (&timeline_yet_another, Some(&[1])), + ], + ); + + // histograms per timeline + assert_recursive_histogram( + &db, + [ + (&Timeline::log_time(), None), + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 5)])), + (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 5)])), + ] as [(_, Option<&[_]>); 4], + ); + + // histograms per component per timeline ('parent/child/grandchild') + assert_histogram_for_component( + &db, + &entity_grandchild, + MyIndex::name(), + [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyPoint::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), + ] as [(_, Option<&[_]>); 2], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyColor::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), + ] as [(_, Option<&[_]>); 2], + ); + } + + // Grand-child, timeless additions. + { + let chunk = { + let num_instances = 6; + let colors = vec![MyColor::from(0x00DD00FF); num_instances]; + Chunk::builder("entity".into()) + .with_component_batches( + RowId::new(), + TimePoint::default(), + [ + &MyIndex::from_iter(0..num_instances as _) as _, + &colors as _, + ], + ) + .build()? + }; + + db.add_chunk(&Arc::new(chunk))?; + + // times per timeline + assert_times_per_timeline( + &db, + [ + (&Timeline::log_time(), Some(&[] as &[i64])), + (&timeline_frame, Some(&[42])), + (&timeline_other, Some(&[666])), + (&timeline_yet_another, Some(&[1])), + ], + ); + + // histograms per timeline + assert_recursive_histogram( + &db, + [ + (&Timeline::log_time(), None), + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 5)])), + (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 5)])), + ] as [(_, Option<&[_]>); 4], + ); + + // histograms per component per timeline + assert_histogram_for_component( + &db, + &entity_parent, + MyPoint::name(), + [ + (&timeline_frame, None), + (&timeline_other, None), + (&timeline_yet_another, None), + ] as [(_, Option<&[_]>); 3], + ); + assert_histogram_for_component( + &db, + &entity_parent, + MyColor::name(), + [ + (&timeline_frame, None), + (&timeline_other, None), + (&timeline_yet_another, None), + ] as [(_, Option<&[_]>); 3], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyIndex::name(), + [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyPoint::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), + ] as [(_, Option<&[_]>); 2], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyColor::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), + ] as [(_, Option<&[_]>); 2], + ); + } + + // Completely unrelated entity. + { + let chunk = { + let num_instances = 3; + let points: Vec<_> = (0..num_instances) + .map(|i| MyPoint::new(0.0, i as f32)) + .collect(); + let colors = vec![MyColor::from(0xFF0000FF)]; + Chunk::builder(entity_unrelated.clone()) + .with_component_batches( + RowId::new(), + TimePoint::from_iter([ + (timeline_frame, 1234), // + (timeline_other, 1235), // + (timeline_yet_another, 1236), // + ]), + [ + &MyIndex::from_iter(0..num_instances) as _, + &points as _, + &colors as _, + ], + ) + .build()? + }; + + db.add_chunk(&Arc::new(chunk))?; + + assert_times_per_timeline( + &db, + [ + (&Timeline::log_time(), Some(&[] as &[i64])), + (&timeline_frame, Some(&[42, 1234])), + (&timeline_other, Some(&[666, 1235])), + (&timeline_yet_another, Some(&[1, 1236])), + ], + ); + + // histograms per timeline + assert_recursive_histogram( + &db, + [ + (&Timeline::log_time(), None), + ( + &timeline_frame, + Some(&[(RangeI64::new(42, 42), 5), (RangeI64::new(1234, 1234), 3)]), + ), + ( + &timeline_other, + Some(&[(RangeI64::new(666, 666), 1), (RangeI64::new(1235, 1235), 3)]), + ), + ( + &timeline_yet_another, + Some(&[(RangeI64::new(1, 1), 5), (RangeI64::new(1236, 1236), 3)]), + ), + ] as [(_, Option<&[_]>); 4], + ); + + // histograms per component per timeline + assert_histogram_for_component( + &db, + &entity_parent, + MyIndex::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), + (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), + ] as [(_, Option<&[_]>); 3], + ); + // NOTE: per-component histograms are not recursive! + assert_histogram_for_component( + &db, + &entity_parent, + MyPoint::name(), + [ + (&timeline_frame, None), + (&timeline_other, None), + (&timeline_yet_another, None), + ] as [(_, Option<&[_]>); 3], + ); + assert_histogram_for_component( + &db, + &entity_parent, + MyColor::name(), + [ + (&timeline_frame, None), + (&timeline_other, None), + (&timeline_yet_another, None), + ] as [(_, Option<&[_]>); 3], + ); + + // histograms per component per timeline ('very/unrelated') + assert_histogram_for_component( + &db, + &entity_unrelated, + MyIndex::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(1234, 1234), 1)])), + (&timeline_other, Some(&[(RangeI64::new(1235, 1235), 1)])), + ( + &timeline_yet_another, + Some(&[(RangeI64::new(1236, 1236), 1)]), + ), + ] as [(_, Option<&[_]>); 3], + ); + assert_histogram_for_component( + &db, + &entity_unrelated, + MyPoint::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(1234, 1234), 1)])), + (&timeline_other, Some(&[(RangeI64::new(1235, 1235), 1)])), + ( + &timeline_yet_another, + Some(&[(RangeI64::new(1236, 1236), 1)]), + ), + ] as [(_, Option<&[_]>); 3], + ); + assert_histogram_for_component( + &db, + &entity_unrelated, + MyColor::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(1234, 1234), 1)])), + (&timeline_other, Some(&[(RangeI64::new(1235, 1235), 1)])), + ( + &timeline_yet_another, + Some(&[(RangeI64::new(1236, 1236), 1)]), + ), + ] as [(_, Option<&[_]>); 3], + ); + } + + // Immediate clear. + { + let chunk = { + Chunk::builder(entity_parent.clone()) + .with_component_batches( + RowId::new(), + TimePoint::from_iter([ + (timeline_frame, 1000), // + ]), + [&[ClearIsRecursive(true.into())] as _], + ) + .build()? + }; + + db.add_chunk(&Arc::new(chunk))?; + + assert_times_per_timeline( + &db, + [ + (&Timeline::log_time(), Some(&[] as &[i64])), + (&timeline_frame, Some(&[42, 1000, 1234])), + (&timeline_other, Some(&[666, 1235])), + (&timeline_yet_another, Some(&[1, 1236])), + ], + ); + + // histograms per timeline + assert_recursive_histogram( + &db, + [ + (&Timeline::log_time(), None), + ( + &timeline_frame, + Some(&[ + (RangeI64::new(42, 42), 5), + (RangeI64::new(1000, 1000), 1), + (RangeI64::new(1234, 1234), 3), + ]), + ), + ( + &timeline_other, + Some(&[(RangeI64::new(666, 666), 1), (RangeI64::new(1235, 1235), 3)]), + ), + ( + &timeline_yet_another, + Some(&[(RangeI64::new(1, 1), 5), (RangeI64::new(1236, 1236), 3)]), + ), + ] as [(_, Option<&[_]>); 4], + ); + + // histograms per component per timeline + assert_histogram_for_component( + &db, + &entity_parent, + MyIndex::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), + (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), + ] as [(_, Option<&[_]>); 3], + ); + + // histograms per component per timeline + assert_histogram_for_component( + &db, + &entity_grandchild, + MyIndex::name(), + [ + (&timeline_frame, None), + (&timeline_other, None), + (&timeline_yet_another, None), + ] as [(_, Option<&[_]>); 3], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyIndex::name(), + [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyPoint::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), + ] as [(_, Option<&[_]>); 2], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyColor::name(), + [ + (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), + (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), + ] as [(_, Option<&[_]>); 2], + ); + } + + // Full GC + { + db.gc(&GarbageCollectionOptions::gc_everything()); + + assert_times_per_timeline( + &db, + [ + (&Timeline::log_time(), Some(&[] as &[i64])), + (&timeline_frame, Some(&[])), + (&timeline_other, Some(&[])), + (&timeline_yet_another, Some(&[])), + ], + ); + + // histograms per timeline + assert_recursive_histogram( + &db, + [ + (&Timeline::log_time(), None), + (&timeline_frame, Some(&[])), + (&timeline_other, Some(&[])), + (&timeline_yet_another, Some(&[])), + ] as [(_, Option<&[_]>); 4], + ); + + // histograms per component per timeline + assert_histogram_for_component( + &db, + &entity_parent, + MyIndex::name(), + [ + (&timeline_frame, Some(&[])), + (&timeline_other, Some(&[])), + (&timeline_yet_another, Some(&[])), + ] as [(_, Option<&[_]>); 3], + ); + + // histograms per component per timeline + assert_histogram_for_component( + &db, + &entity_grandchild, + MyIndex::name(), + [ + (&timeline_frame, None), + (&timeline_other, None), + (&timeline_yet_another, None), + ] as [(_, Option<&[_]>); 3], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyIndex::name(), + [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyPoint::name(), + [ + (&timeline_frame, Some(&[])), + (&timeline_yet_another, Some(&[])), + ] as [(_, Option<&[_]>); 2], + ); + assert_histogram_for_component( + &db, + &entity_grandchild, + MyColor::name(), + [ + (&timeline_frame, Some(&[])), + (&timeline_yet_another, Some(&[])), + ] as [(_, Option<&[_]>); 2], + ); + } + + Ok(()) +} + +/// Checks the state of the global time tracker (at the `EntityDb` level). +fn assert_times_per_timeline<'a>( + db: &EntityDb, + expected: impl IntoIterator)>, +) { + for (timeline, expected_times) in expected { + let times = db.times_per_timeline().get(timeline); + + if let Some(expected) = expected_times { + let times: BTreeSet<_> = times.unwrap().keys().copied().collect(); + let expected: BTreeSet<_> = expected + .iter() + .map(|&t| TimeInt::try_from(t).unwrap()) + .collect(); + similar_asserts::assert_eq!(expected, times); + } else { + assert!(times.is_none()); + } + } +} + +/// Checks the state of the per-EntityTree recursive time tracker. +fn assert_recursive_histogram<'a>( + db: &EntityDb, + expected: impl IntoIterator)>, +) { + for (timeline, expected_times) in expected { + let histo = db.time_histogram(timeline); + + if let Some(expected) = expected_times { + if expected.is_empty() { + assert!(histo.is_none()); + continue; + } + let histo = histo.unwrap(); + let ranges = histo.range(i64::MIN.., 0).collect::>(); + let expected: Vec<_> = expected.to_vec(); + similar_asserts::assert_eq!(expected, ranges); + } else { + assert!(histo.is_none()); + } + } +} + +/// Checks the state of the per-`EntityTree` per-`ComponentName` flat time tracker. +fn assert_histogram_for_component<'a>( + db: &EntityDb, + entity_path: &EntityPath, + component_name: ComponentName, + expected: impl IntoIterator)>, +) { + for (timeline, expected_times) in expected { + let histo = db + .tree() + .subtree(entity_path) + .and_then(|tree| tree.time_histogram_for_component(timeline, component_name)); + + if let Some(expected) = expected_times { + let expected: Vec<_> = expected.to_vec(); + if expected.is_empty() { + assert!(histo.is_none()); + continue; + } + let histo = histo.unwrap(); + let ranges = histo.range(i64::MIN.., 0).collect::>(); + similar_asserts::assert_eq!(expected, ranges); + } else { + assert!(histo.is_none()); + } + } +} diff --git a/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs b/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs index 6f5ce1e1edef..62f6f16c1629 100644 --- a/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs +++ b/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs @@ -91,7 +91,7 @@ impl ContextMenuAction for CollapseExpandAllAction { return; }; - subtree.visit_children_recursively(&mut |entity_path| { + subtree.visit_children_recursively(&mut |entity_path, _| { CollapseScope::BlueprintTree .data_result(*space_view_id, entity_path.clone()) .set_open(&ctx.egui_context, self.open()); @@ -108,7 +108,7 @@ impl ContextMenuAction for CollapseExpandAllAction { return; }; - subtree.visit_children_recursively(&mut |entity_path| { + subtree.visit_children_recursively(&mut |entity_path, _| { CollapseScope::StreamsTree .entity(entity_path.clone()) .set_open(&ctx.egui_context, self.open()); diff --git a/crates/viewer/re_data_ui/src/component.rs b/crates/viewer/re_data_ui/src/component.rs index a848428c3a09..3b7f44ad5e46 100644 --- a/crates/viewer/re_data_ui/src/component.rs +++ b/crates/viewer/re_data_ui/src/component.rs @@ -72,38 +72,36 @@ impl<'a> DataUi for EntityLatestAtResults<'a> { // if the component is static, we display extra diagnostic information if self.results.is_static() { - let num_static_messages = db - .store() - .num_events_for_static_component(&self.entity_path, component_name); - if num_static_messages > 1 { - ui.label(ui.ctx().warning_text(format!( - "Static component value was overridden {} times", - num_static_messages.saturating_sub(1), - ))) - .on_hover_text( - "When a static component is logged multiple times, only the last value \ - is stored. Previously logged values are overwritten and not \ - recoverable.", - ); - } + if let Some(histogram) = db + .tree() + .subtree(&self.entity_path) + .and_then(|tree| tree.entity.components.get(&component_name)) + { + if histogram.num_static_messages() > 1 { + ui.label(ui.ctx().warning_text(format!( + "Static component value was overridden {} times", + histogram.num_static_messages().saturating_sub(1), + ))) + .on_hover_text( + "When a static component is logged multiple times, only the last value \ + is stored. Previously logged values are overwritten and not \ + recoverable.", + ); + } - let timeline_message_count = - db.store().num_events_on_timeline_for_temporal_component( - &query.timeline(), - &self.entity_path, - component_name, - ); - if timeline_message_count > 0 { - ui.label(ui.ctx().error_text(format!( - "Static component has {} event{} logged on timelines", - timeline_message_count, - if timeline_message_count > 1 { "s" } else { "" } - ))) - .on_hover_text( - "Components should be logged either as static or on timelines, but \ - never both. Values for static components logged to timelines cannot be \ - displayed.", - ); + let timeline_message_count = histogram.num_temporal_messages(); + if timeline_message_count > 0 { + ui.label(ui.ctx().error_text(format!( + "Static component has {} event{} logged on timelines", + timeline_message_count, + if timeline_message_count > 1 { "s" } else { "" } + ))) + .on_hover_text( + "Components should be logged either as static or on timelines, but \ + never both. Values for static components logged to timelines cannot be \ + displayed.", + ); + } } } } diff --git a/crates/viewer/re_data_ui/src/component_path.rs b/crates/viewer/re_data_ui/src/component_path.rs index 4597468955a1..9161e27a09ab 100644 --- a/crates/viewer/re_data_ui/src/component_path.rs +++ b/crates/viewer/re_data_ui/src/component_path.rs @@ -32,11 +32,8 @@ impl DataUi for ComponentPath { results: results.as_ref(), } .data_ui(ctx, ui, ui_layout, query, db); - } else if ctx.recording().tree().subtree(entity_path).is_some() { - if db - .store() - .entity_has_component(&query.timeline(), entity_path, component_name) - { + } else if let Some(entity_tree) = ctx.recording().tree().subtree(entity_path) { + if entity_tree.entity.components.contains_key(component_name) { ui.label(""); } else { ui.label(format!( diff --git a/crates/viewer/re_data_ui/src/instance_path.rs b/crates/viewer/re_data_ui/src/instance_path.rs index 3f6975c0f9bc..509556267470 100644 --- a/crates/viewer/re_data_ui/src/instance_path.rs +++ b/crates/viewer/re_data_ui/src/instance_path.rs @@ -81,9 +81,7 @@ impl DataUi for InstancePath { } let component_path = ComponentPath::new(entity_path.clone(), component_name); - let is_static = db - .store() - .entity_has_static_component(entity_path, &component_name); + let is_static = db.is_component_static(&component_path).unwrap_or_default(); let icon = if is_static { &re_ui::icons::COMPONENT_STATIC } else { diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index e7e0c9071362..1447130f798b 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -291,16 +291,11 @@ pub fn instance_path_parts_buttons( .response } -fn entity_tree_stats_ui( - ui: &mut egui::Ui, - timeline: &Timeline, - db: &re_entity_db::EntityDb, - tree: &EntityTree, -) { +fn entity_tree_stats_ui(ui: &mut egui::Ui, timeline: &Timeline, tree: &EntityTree) { use re_format::format_bytes; // Show total bytes used in whole subtree - let total_bytes = db.store().size_of_entity_on_timeline(timeline, &tree.path); + let total_bytes = tree.subtree.data_bytes(); let subtree_caveat = if tree.children.is_empty() { "" @@ -315,16 +310,14 @@ fn entity_tree_stats_ui( let mut data_rate = None; // Try to estimate data-rate - if db.store().entity_has_data_on_timeline(timeline, &tree.path) { + if let Some(time_histogram) = tree.subtree.time_histogram.get(timeline) { // `num_events` is approximate - we could be logging a Tensor image and a transform // at _almost_ approximately the same time, but it should only count as one fence-post. - let num_events = db - .store() - .num_events_on_timeline_for_all_components(timeline, &tree.path); + let num_events = time_histogram.total_count(); // TODO(emilk): we should ask the histogram to count the number of non-zero keys instead. - if let Some(time_range) = db.store().time_range_for_entity(timeline, &tree.path) { - let min_time = time_range.min(); - let max_time = time_range.max(); + if let (Some(min_time), Some(max_time)) = + (time_histogram.min_key(), time_histogram.max_key()) + { if min_time < max_time && 1 < num_events { // Let's do our best to avoid fencepost errors. // If we log 1 MiB once every second, then after three @@ -335,9 +328,9 @@ fn entity_tree_stats_ui( // t: 0s 1s 2s // data: 1MiB 1MiB 1MiB - let duration = max_time.as_f64() - min_time.as_f64(); + let duration = max_time - min_time; - let mut bytes_per_time = total_bytes as f64 / duration; + let mut bytes_per_time = total_bytes as f64 / duration as f64; // Fencepost adjustment: bytes_per_time *= (num_events - 1) as f64 / num_events as f64; @@ -400,10 +393,7 @@ pub fn component_path_button_to( db: &re_entity_db::EntityDb, ) -> egui::Response { let item = Item::ComponentPath(component_path.clone()); - let is_static = db.store().entity_has_static_component( - component_path.entity_path(), - component_path.component_name(), - ); + let is_static = db.is_component_static(component_path).unwrap_or_default(); let icon = if is_static { &icons::COMPONENT_STATIC } else { @@ -558,7 +548,7 @@ pub fn instance_hover_card_ui( if instance_path.instance.is_all() { if let Some(subtree) = ctx.recording().tree().subtree(&instance_path.entity_path) { - entity_tree_stats_ui(ui, &query.timeline(), db, subtree); + entity_tree_stats_ui(ui, &query.timeline(), subtree); } } else { // TODO(emilk): per-component stats diff --git a/crates/viewer/re_selection_panel/src/selection_panel.rs b/crates/viewer/re_selection_panel/src/selection_panel.rs index b2e3d701a7a8..c24f1306db7d 100644 --- a/crates/viewer/re_selection_panel/src/selection_panel.rs +++ b/crates/viewer/re_selection_panel/src/selection_panel.rs @@ -155,12 +155,9 @@ impl SelectionPanel { match item { Item::ComponentPath(component_path) => { let entity_path = &component_path.entity_path; - let component_name = &component_path.component_name; let (query, db) = guess_query_and_db_for_selected_entity(ctx, entity_path); - let is_static = db - .store() - .entity_has_static_component(entity_path, component_name); + let is_static = db.is_component_static(component_path).unwrap_or_default(); ui.list_item_flat_noninteractive( PropertyContent::new("Component type").value_text(if is_static { @@ -721,9 +718,7 @@ fn item_tile( let component_name = &component_path.component_name; let (_query, db) = guess_query_and_db_for_selected_entity(ctx, entity_path); - let is_static = db - .store() - .entity_has_static_component(entity_path, component_name); + let is_static = db.is_component_static(component_path).unwrap_or_default(); Some( ItemTitle::new(component_name.short_name()) diff --git a/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs b/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs index b5b83cf06eda..ffe6d4ecab9d 100644 --- a/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs +++ b/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs @@ -326,7 +326,7 @@ fn create_entity_add_info( &space_view.space_origin, ); - tree.visit_children_recursively(&mut |entity_path| { + tree.visit_children_recursively(&mut |entity_path, _| { let can_add: CanAddToSpaceView = if visualizable_entities.iter().any(|(_, entities)| entities.contains(entity_path)) { CanAddToSpaceView::Compatible { diff --git a/crates/viewer/re_space_view_spatial/src/view_2d.rs b/crates/viewer/re_space_view_spatial/src/view_2d.rs index 97a66e75f253..3750b57e9eba 100644 --- a/crates/viewer/re_space_view_spatial/src/view_2d.rs +++ b/crates/viewer/re_space_view_spatial/src/view_2d.rs @@ -274,15 +274,12 @@ impl SpaceViewClass for SpatialSpaceView2D { fn count_non_nested_images_with_component( image_dimensions: &IntMap, entity_bucket: &IntSet, - entity_db: &re_entity_db::EntityDb, subtree: &EntityTree, component_name: &ComponentName, ) -> usize { if image_dimensions.contains_key(&subtree.path) { // bool true -> 1 - entity_db - .store() - .entity_has_component_on_any_timeline(&subtree.path, component_name) as usize + subtree.entity.components.contains_key(component_name) as usize } else if !entity_bucket .iter() .any(|e| e.is_descendant_of(&subtree.path)) @@ -296,7 +293,6 @@ fn count_non_nested_images_with_component( count_non_nested_images_with_component( image_dimensions, entity_bucket, - entity_db, child, component_name, ) @@ -365,7 +361,6 @@ fn recommended_space_views_with_image_splits( let image_count = count_non_nested_images_with_component( image_dimensions, entities, - ctx.recording(), subtree, &Image::indicator().name(), ); @@ -373,7 +368,6 @@ fn recommended_space_views_with_image_splits( let depth_count = count_non_nested_images_with_component( image_dimensions, entities, - ctx.recording(), subtree, &DepthImage::indicator().name(), ); diff --git a/crates/viewer/re_space_view_spatial/src/view_3d.rs b/crates/viewer/re_space_view_spatial/src/view_3d.rs index 18cc5117660e..f0abd2087ebe 100644 --- a/crates/viewer/re_space_view_spatial/src/view_3d.rs +++ b/crates/viewer/re_space_view_spatial/src/view_3d.rs @@ -279,12 +279,8 @@ impl SpaceViewClass for SpatialSpaceView3D { // There's also a strong argument to be made that ViewCoordinates implies a 3D space, thus changing the SpacialTopology accordingly! ctx.recording() .tree() - .visit_children_recursively(&mut |path| { - if ctx - .recording() - .store() - .entity_has_component_on_any_timeline(path, &ViewCoordinates::name()) - { + .visit_children_recursively(&mut |path, info| { + if info.components.contains_key(&ViewCoordinates::name()) { indicated_entities.insert(path.clone()); } }); diff --git a/crates/viewer/re_time_panel/src/data_density_graph.rs b/crates/viewer/re_time_panel/src/data_density_graph.rs index 5be1bc097a02..63b8b426a98e 100644 --- a/crates/viewer/re_time_panel/src/data_density_graph.rs +++ b/crates/viewer/re_time_panel/src/data_density_graph.rs @@ -716,7 +716,7 @@ fn visit_relevant_chunks( visitor(Arc::clone(&chunk), chunk_timeline.time_range(), num_events); } } else if let Some(subtree) = db.tree().subtree(entity_path) { - subtree.visit_children_recursively(&mut |entity_path| { + subtree.visit_children_recursively(&mut |entity_path, _| { for chunk in db .store() .range_relevant_chunks_for_all_components(&query, entity_path) diff --git a/crates/viewer/re_time_panel/src/lib.rs b/crates/viewer/re_time_panel/src/lib.rs index 5a2321ddd2ab..2fa898363b44 100644 --- a/crates/viewer/re_time_panel/src/lib.rs +++ b/crates/viewer/re_time_panel/src/lib.rs @@ -19,7 +19,7 @@ use egui::{pos2, Color32, CursorIcon, NumExt, Painter, PointerButton, Rect, Shap use re_context_menu::{context_menu_ui_for_item, SelectionUpdateBehavior}; use re_data_ui::DataUi as _; use re_data_ui::{item_ui::guess_instance_path_icon, sorted_component_list_for_ui}; -use re_entity_db::{EntityTree, InstancePath}; +use re_entity_db::{EntityTree, InstancePath, TimeHistogram}; use re_log_types::{ external::re_types_core::ComponentName, ComponentPath, EntityPath, EntityPathPart, ResolvedTimeRange, TimeInt, TimeReal, @@ -574,6 +574,8 @@ impl TimePanel { ui: &mut egui::Ui, show_root_as: &str, ) { + let tree_has_data_in_current_timeline = time_ctrl.tree_has_data_in_current_timeline(tree); + let db = match self.source { TimePanelSource::Recording => ctx.recording(), TimePanelSource::Blueprint => ctx.store_context.blueprint, @@ -691,9 +693,6 @@ impl TimePanel { // ---------------------------------------------- // show the data in the time area: - let tree_has_data_in_current_timeline = entity_db - .store() - .entity_has_data_on_timeline(time_ctrl.timeline(), &tree.path); if is_visible && tree_has_data_in_current_timeline { let row_rect = Rect::from_x_y_ranges(time_area_response.rect.x_range(), response_rect.y_range()); @@ -745,29 +744,17 @@ impl TimePanel { } // If this is an entity: - if let Some(components) = entity_db - .store() - .all_components_on_all_timelines(&tree.path) - { - for component_name in sorted_component_list_for_ui(components.iter()) { - let is_static = entity_db - .store() - .entity_has_static_component(&tree.path, &component_name); + if !tree.entity.components.is_empty() { + for component_name in sorted_component_list_for_ui(tree.entity.components.keys()) { + let data = &tree.entity.components[&component_name]; + + let is_static = data.is_static(); + let component_has_data_in_current_timeline = + time_ctrl.component_has_data_in_current_timeline(data); let component_path = ComponentPath::new(tree.path.clone(), component_name); let short_component_name = component_path.component_name.short_name(); let item = TimePanelItem::component_path(component_path.clone()); - let timeline = time_ctrl.timeline(); - - let component_has_data_in_current_timeline = entity_db - .store() - .entity_has_component(time_ctrl.timeline(), &tree.path, &component_name); - - let total_num_messages = entity_db.store().num_events_on_timeline_for_component( - time_ctrl.timeline(), - &tree.path, - component_name, - ); let response = ui .list_item() @@ -799,6 +786,14 @@ impl TimePanel { let response_rect = response.rect; + let timeline = time_ctrl.timeline(); + + let empty_messages_over_time = TimeHistogram::default(); + let messages_over_time = data.get(timeline).unwrap_or(&empty_messages_over_time); + + // `data.times` does not contain static. Need to add those manually: + let total_num_messages = + messages_over_time.total_count() + data.num_static_messages(); response.on_hover_ui(|ui| { if total_num_messages == 0 { ui.label(ui.ctx().warning_text(format!( diff --git a/crates/viewer/re_viewer_context/src/time_control.rs b/crates/viewer/re_viewer_context/src/time_control.rs index bd1651944c0e..cb003750917c 100644 --- a/crates/viewer/re_viewer_context/src/time_control.rs +++ b/crates/viewer/re_viewer_context/src/time_control.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use re_entity_db::{TimeCounts, TimesPerTimeline}; +use re_entity_db::{EntityTree, TimeCounts, TimeHistogramPerTimeline, TimesPerTimeline}; use re_log_types::{ Duration, ResolvedTimeRange, ResolvedTimeRangeF, TimeInt, TimeReal, TimeType, Timeline, }; @@ -571,6 +571,22 @@ impl TimeControl { state.view = None; } } + + /// Returns whether the given tree has any data logged in the current timeline, + /// or has any timeless messages. + pub fn tree_has_data_in_current_timeline(&self, tree: &EntityTree) -> bool { + let top_time_histogram = &tree.subtree.time_histogram; + top_time_histogram.has_timeline(self.timeline()) + || top_time_histogram.num_static_messages() > 0 + } + + /// Returns whether the given component has any data logged in the current timeline. + pub fn component_has_data_in_current_timeline( + &self, + component_stat: &TimeHistogramPerTimeline, + ) -> bool { + component_stat.has_timeline(self.timeline()) + } } fn min(values: &TimeCounts) -> TimeInt { diff --git a/crates/viewer/re_viewport_blueprint/src/space_view.rs b/crates/viewer/re_viewport_blueprint/src/space_view.rs index 8e03cb3800fa..c3df969c0ca5 100644 --- a/crates/viewer/re_viewport_blueprint/src/space_view.rs +++ b/crates/viewer/re_viewport_blueprint/src/space_view.rs @@ -251,7 +251,7 @@ impl SpaceViewBlueprint { // Create pending write operations to duplicate the entire subtree // TODO(jleibs): This should be a helper somewhere. if let Some(tree) = blueprint.tree().subtree(¤t_path) { - tree.visit_children_recursively(&mut |path| { + tree.visit_children_recursively(&mut |path, info| { let sub_path: EntityPath = new_path .iter() .chain(&path[current_path.len()..]) @@ -262,11 +262,8 @@ impl SpaceViewBlueprint { .with_row( RowId::new(), store_context.blueprint_timepoint_for_writes(), - blueprint - .store() - .all_components(&query.timeline(), path) - .into_iter() - .flat_map(|v| v.into_iter()) + info.components + .keys() // It's important that we don't include the SpaceViewBlueprint's components // since those will be updated separately and may contain different data. .filter(|component| { @@ -274,7 +271,7 @@ impl SpaceViewBlueprint { || !blueprint_archetypes::SpaceViewBlueprint::all_components() .contains(component) }) - .filter_map(|component_name| { + .filter_map(|&component_name| { let results = blueprint.query_caches().latest_at( blueprint.store(), query, diff --git a/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs b/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs index 283d78186621..b13c988a7f71 100644 --- a/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs +++ b/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs @@ -428,11 +428,7 @@ impl DataQueryPropertyResolver<'_> { if let Some(recursive_override_subtree) = blueprint.tree().subtree(&recursive_override_path) { - for component_name in blueprint - .store() - .all_components_on_all_timelines(&recursive_override_subtree.path) - .unwrap_or_default() - { + for &component_name in recursive_override_subtree.entity.components.keys() { let results = blueprint.query_caches().latest_at( blueprint.store(), blueprint_query, @@ -463,11 +459,7 @@ impl DataQueryPropertyResolver<'_> { if let Some(individual_override_subtree) = blueprint.tree().subtree(&individual_override_path) { - for component_name in blueprint - .store() - .all_components_on_all_timelines(&individual_override_subtree.path) - .unwrap_or_default() - { + for &component_name in individual_override_subtree.entity.components.keys() { let results = blueprint.query_caches().latest_at( blueprint.store(), blueprint_query, From e823464bbe64882430dc30a75c428ad9ea82ea48 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:11:14 +0200 Subject: [PATCH 04/43] add new APIs --- crates/store/re_chunk_store/src/query.rs | 313 ++++++++++++++++++ crates/store/re_entity_db/src/entity_db.rs | 22 ++ crates/store/re_entity_db/src/entity_tree.rs | 12 +- .../src/actions/collapse_expand_all.rs | 4 +- .../src/space_view_entity_picker.rs | 2 +- .../re_space_view_spatial/src/view_3d.rs | 2 +- .../re_time_panel/src/data_density_graph.rs | 2 +- 7 files changed, 348 insertions(+), 9 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 4ebe623f1f31..782141b1cb11 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -5,7 +5,9 @@ use std::{ use itertools::Itertools; use re_chunk::{Chunk, LatestAtQuery, RangeQuery}; +use re_log_types::ResolvedTimeRange; use re_log_types::{EntityPath, TimeInt, Timeline}; +use re_types_core::SizeBytes as _; use re_types_core::{ComponentName, ComponentNameSet}; use crate::{store::ChunkIdSetPerTime, ChunkStore}; @@ -61,6 +63,47 @@ impl ChunkStore { } } + /// Retrieve all the [`ComponentName`]s that have been written to for a given [`EntityPath`]. + /// + /// Static components are always included in the results. + /// + /// Returns `None` if the entity has never had any data logged to it. + pub fn all_components_on_all_timelines( + &self, + entity_path: &EntityPath, + ) -> Option { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let static_components: Option = self + .static_chunk_ids_per_entity + .get(entity_path) + .map(|static_chunks_per_component| { + static_chunks_per_component.keys().copied().collect() + }); + + let temporal_components: Option = self + .temporal_chunk_ids_per_entity_per_component + .get(entity_path) + .map(|temporal_chunk_ids_per_timeline| { + temporal_chunk_ids_per_timeline + .iter() + .flat_map(|(_, temporal_chunk_ids_per_component)| { + temporal_chunk_ids_per_component.keys().copied() + }) + .collect() + }); + + match (static_components, temporal_components) { + (None, None) => None, + (None, comps @ Some(_)) | (comps @ Some(_), None) => comps, + (Some(static_comps), Some(temporal_comps)) => { + Some(static_comps.into_iter().chain(temporal_comps).collect()) + } + } + } + /// Check whether a given entity has a specific [`ComponentName`] either on the specified /// timeline, or in its static data. #[inline] @@ -75,6 +118,56 @@ impl ChunkStore { .map_or(false, |components| components.contains(component_name)) } + pub fn entity_has_component_on_any_timeline( + &self, + entity_path: &EntityPath, + component_name: &ComponentName, + ) -> bool { + re_tracing::profile_function!(); + + if self + .static_chunk_ids_per_entity + .get(entity_path) + .and_then(|static_chunks_per_component| static_chunks_per_component.get(component_name)) + .and_then(|id| self.chunks_per_chunk_id.get(id)) + .is_some() + { + return true; + } + + for temporal_chunk_ids_per_component in self + .temporal_chunk_ids_per_entity_per_component + .get(entity_path) + .iter() + .flat_map(|temporal_chunk_ids_per_timeline| temporal_chunk_ids_per_timeline.values()) + { + if temporal_chunk_ids_per_component + .get(component_name) + .is_some() + { + return true; + } + } + + false + } + + /// Returns true if the given entity has a specific component with static data. + #[inline] + pub fn entity_has_static_component( + &self, + entity_path: &EntityPath, + component_name: &ComponentName, + ) -> bool { + re_tracing::profile_function!(); + + let Some(static_components) = self.static_chunk_ids_per_entity.get(entity_path) else { + return false; + }; + + static_components.contains_key(component_name) + } + /// Find the earliest time at which something was logged for a given entity on the specified /// timeline. /// @@ -413,3 +506,223 @@ impl ChunkStore { .collect() } } + +// Queries returning `usize` and `bool` +impl ChunkStore { + /// Returns the number of events logged for a given component on the given entity path across all timelines. + pub fn num_events_on_timeline_for_component( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + component_name: ComponentName, + ) -> usize { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + self.num_events_for_static_component(entity_path, component_name) + + self.num_events_on_timeline_for_temporal_component( + timeline, + entity_path, + component_name, + ) + } + + pub fn time_range_for_entity( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> Option { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let temporal_chunk_ids_per_timeline = + self.temporal_chunk_ids_per_entity.get(entity_path)?; + let chunk_id_sets = temporal_chunk_ids_per_timeline.get(timeline)?; + + let start = chunk_id_sets.per_start_time.keys().min()?; + let end = chunk_id_sets.per_end_time.keys().max()?; + + Some(ResolvedTimeRange::new(*start, *end)) + } + + pub fn num_events_on_timeline_for_all_components( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> usize { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let mut total_events = 0; + + if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) + { + for chunk in static_chunks_per_component + .values() + .filter_map(|id| self.chunks_per_chunk_id.get(id)) + { + total_events += chunk.num_events_cumulative(); + } + } + + if let Some(chunk_ids) = self + .temporal_chunk_ids_per_entity + .get(entity_path) + .and_then(|temporal_chunks_events_per_timeline| { + temporal_chunks_events_per_timeline.get(timeline) + }) + { + for chunk in chunk_ids + .per_start_time + .values() + .flat_map(|ids| ids.iter().filter_map(|id| self.chunks_per_chunk_id.get(id))) + { + total_events += chunk.num_events_cumulative(); + } + } + + total_events + } + + pub fn entity_has_data_on_timeline( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> bool { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + if self.static_chunk_ids_per_entity.get(entity_path).is_some() { + // Static data exists on all timelines + return true; + } + + if let Some(temporal_chunk_ids_per_timeline) = + self.temporal_chunk_ids_per_entity.get(entity_path) + { + if temporal_chunk_ids_per_timeline.contains_key(timeline) { + return true; + } + } + + false + } + + pub fn num_events_for_static_component( + &self, + entity_path: &EntityPath, + component_name: ComponentName, + ) -> usize { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + if let Some(static_chunk) = self + .static_chunk_ids_per_entity + .get(entity_path) + .and_then(|static_chunks_per_component| { + static_chunks_per_component.get(&component_name) + }) + .and_then(|chunk_id| self.chunks_per_chunk_id.get(chunk_id)) + { + // Static data overrides temporal data + static_chunk + .num_events_for_component(component_name) + .unwrap_or(0) + } else { + 0 + } + } + + /// Returns the number of events logged for a given component on the given entity path across all timelines. + /// + /// This ignores static data. + pub fn num_events_on_timeline_for_temporal_component( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + component_name: ComponentName, + ) -> usize { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let Some(temporal_chunk_ids_per_timeline) = self + .temporal_chunk_ids_per_entity_per_component + .get(entity_path) + else { + return 0; // no events logged for the entity path + }; + + let Some(temporal_chunk_ids_per_component) = temporal_chunk_ids_per_timeline.get(timeline) + else { + return 0; // no events logged on this timeline + }; + + let Some(chunk_ids) = temporal_chunk_ids_per_component.get(&component_name) else { + return 0; // no events logged for the component on this timeline + }; + + let mut num_events = 0; + for chunk in chunk_ids + .per_start_time + .values() + .flat_map(|ids| ids.iter().filter_map(|id| self.chunks_per_chunk_id.get(id))) + { + num_events += chunk.num_events_for_component(component_name).unwrap_or(0); + } + + num_events + } + + /// Returns the size of the entity on a specific timeline. + /// + /// This includes static data. + /// + /// ⚠ This does not return the _total_ size of the entity and all its children! + /// For that, use `entity_db.size_of_subtree_on_timeline`. + pub fn size_of_entity_on_timeline( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> usize { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let mut total_size = 0; + + if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) + { + for chunk in static_chunks_per_component + .values() + .filter_map(|id| self.chunks_per_chunk_id.get(id)) + { + total_size += Chunk::total_size_bytes(chunk) as usize; + } + } + + if let Some(chunk_id_sets) = self + .temporal_chunk_ids_per_entity + .get(entity_path) + .and_then(|temporal_chunk_ids_per_timeline| { + temporal_chunk_ids_per_timeline.get(timeline) + }) + { + for chunk in chunk_id_sets + .per_start_time + .values() + .flat_map(|v| v.iter()) + .filter_map(|id| self.chunks_per_chunk_id.get(id)) + { + total_size += Chunk::total_size_bytes(chunk) as usize; + } + } + + total_size + } +} diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index 50231342cf77..df8a7a1120e6 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -571,6 +571,28 @@ impl EntityDb { Ok(new_db) } + + /// Returns the byte size of an entity and all its children on the given timeline, recursively. + /// + /// This includes static data. + pub fn size_of_subtree_on_timeline( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> usize { + re_tracing::profile_function!(); + + let Some(subtree) = self.tree.subtree(entity_path) else { + return 0; + }; + + let mut size = 0; + subtree.visit_children_recursively(|path, _| { + size += self.store().size_of_entity_on_timeline(timeline, path); + }); + + size + } } impl re_types_core::SizeBytes for EntityDb { diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index b25ce4800f0f..e80181c87f31 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -367,10 +367,14 @@ impl EntityTree { } // Invokes visitor for `self` and all children recursively. - pub fn visit_children_recursively(&self, visitor: &mut impl FnMut(&EntityPath, &EntityInfo)) { - visitor(&self.path, &self.entity); - for child in self.children.values() { - child.visit_children_recursively(visitor); + pub fn visit_children_recursively(&self, mut visitor: impl FnMut(&EntityPath, &EntityInfo)) { + fn visit(this: &EntityTree, visitor: &mut impl FnMut(&EntityPath, &EntityInfo)) { + visitor(&this.path, &this.entity); + for child in this.children.values() { + visit(child, visitor); + } } + + visit(self, &mut visitor); } } diff --git a/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs b/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs index 62f6f16c1629..db75b6b490ad 100644 --- a/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs +++ b/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs @@ -91,7 +91,7 @@ impl ContextMenuAction for CollapseExpandAllAction { return; }; - subtree.visit_children_recursively(&mut |entity_path, _| { + subtree.visit_children_recursively(|entity_path, _| { CollapseScope::BlueprintTree .data_result(*space_view_id, entity_path.clone()) .set_open(&ctx.egui_context, self.open()); @@ -108,7 +108,7 @@ impl ContextMenuAction for CollapseExpandAllAction { return; }; - subtree.visit_children_recursively(&mut |entity_path, _| { + subtree.visit_children_recursively(|entity_path, _| { CollapseScope::StreamsTree .entity(entity_path.clone()) .set_open(&ctx.egui_context, self.open()); diff --git a/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs b/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs index ffe6d4ecab9d..616911d67893 100644 --- a/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs +++ b/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs @@ -326,7 +326,7 @@ fn create_entity_add_info( &space_view.space_origin, ); - tree.visit_children_recursively(&mut |entity_path, _| { + tree.visit_children_recursively(|entity_path, _| { let can_add: CanAddToSpaceView = if visualizable_entities.iter().any(|(_, entities)| entities.contains(entity_path)) { CanAddToSpaceView::Compatible { diff --git a/crates/viewer/re_space_view_spatial/src/view_3d.rs b/crates/viewer/re_space_view_spatial/src/view_3d.rs index f0abd2087ebe..59cf11abd5cd 100644 --- a/crates/viewer/re_space_view_spatial/src/view_3d.rs +++ b/crates/viewer/re_space_view_spatial/src/view_3d.rs @@ -279,7 +279,7 @@ impl SpaceViewClass for SpatialSpaceView3D { // There's also a strong argument to be made that ViewCoordinates implies a 3D space, thus changing the SpacialTopology accordingly! ctx.recording() .tree() - .visit_children_recursively(&mut |path, info| { + .visit_children_recursively(|path, info| { if info.components.contains_key(&ViewCoordinates::name()) { indicated_entities.insert(path.clone()); } diff --git a/crates/viewer/re_time_panel/src/data_density_graph.rs b/crates/viewer/re_time_panel/src/data_density_graph.rs index 63b8b426a98e..671b0c7eaf53 100644 --- a/crates/viewer/re_time_panel/src/data_density_graph.rs +++ b/crates/viewer/re_time_panel/src/data_density_graph.rs @@ -716,7 +716,7 @@ fn visit_relevant_chunks( visitor(Arc::clone(&chunk), chunk_timeline.time_range(), num_events); } } else if let Some(subtree) = db.tree().subtree(entity_path) { - subtree.visit_children_recursively(&mut |entity_path, _| { + subtree.visit_children_recursively(|entity_path, _| { for chunk in db .store() .range_relevant_chunks_for_all_components(&query, entity_path) From 4bd129c9e9bf0f4386ab294152b4dc7df513aafb Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:11:34 +0200 Subject: [PATCH 05/43] update data UI to not use `EntityInfo` --- crates/viewer/re_data_ui/src/component_path.rs | 7 +++++-- crates/viewer/re_data_ui/src/instance_path.rs | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/viewer/re_data_ui/src/component_path.rs b/crates/viewer/re_data_ui/src/component_path.rs index 9161e27a09ab..4597468955a1 100644 --- a/crates/viewer/re_data_ui/src/component_path.rs +++ b/crates/viewer/re_data_ui/src/component_path.rs @@ -32,8 +32,11 @@ impl DataUi for ComponentPath { results: results.as_ref(), } .data_ui(ctx, ui, ui_layout, query, db); - } else if let Some(entity_tree) = ctx.recording().tree().subtree(entity_path) { - if entity_tree.entity.components.contains_key(component_name) { + } else if ctx.recording().tree().subtree(entity_path).is_some() { + if db + .store() + .entity_has_component(&query.timeline(), entity_path, component_name) + { ui.label(""); } else { ui.label(format!( diff --git a/crates/viewer/re_data_ui/src/instance_path.rs b/crates/viewer/re_data_ui/src/instance_path.rs index 509556267470..3f6975c0f9bc 100644 --- a/crates/viewer/re_data_ui/src/instance_path.rs +++ b/crates/viewer/re_data_ui/src/instance_path.rs @@ -81,7 +81,9 @@ impl DataUi for InstancePath { } let component_path = ComponentPath::new(entity_path.clone(), component_name); - let is_static = db.is_component_static(&component_path).unwrap_or_default(); + let is_static = db + .store() + .entity_has_static_component(entity_path, &component_name); let icon = if is_static { &re_ui::icons::COMPONENT_STATIC } else { From 81f33eb35ee565bde6a7c0e252751a733a3b27c2 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:11:46 +0200 Subject: [PATCH 06/43] update blueprint to not use `EntityInfo` --- .../viewer/re_viewport_blueprint/src/space_view.rs | 11 +++++++---- .../re_viewport_blueprint/src/space_view_contents.rs | 12 ++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/crates/viewer/re_viewport_blueprint/src/space_view.rs b/crates/viewer/re_viewport_blueprint/src/space_view.rs index c3df969c0ca5..9fb6f3dd81f4 100644 --- a/crates/viewer/re_viewport_blueprint/src/space_view.rs +++ b/crates/viewer/re_viewport_blueprint/src/space_view.rs @@ -251,7 +251,7 @@ impl SpaceViewBlueprint { // Create pending write operations to duplicate the entire subtree // TODO(jleibs): This should be a helper somewhere. if let Some(tree) = blueprint.tree().subtree(¤t_path) { - tree.visit_children_recursively(&mut |path, info| { + tree.visit_children_recursively(|path, _| { let sub_path: EntityPath = new_path .iter() .chain(&path[current_path.len()..]) @@ -262,8 +262,11 @@ impl SpaceViewBlueprint { .with_row( RowId::new(), store_context.blueprint_timepoint_for_writes(), - info.components - .keys() + blueprint + .store() + .all_components(&query.timeline(), path) + .into_iter() + .flat_map(|v| v.into_iter()) // It's important that we don't include the SpaceViewBlueprint's components // since those will be updated separately and may contain different data. .filter(|component| { @@ -271,7 +274,7 @@ impl SpaceViewBlueprint { || !blueprint_archetypes::SpaceViewBlueprint::all_components() .contains(component) }) - .filter_map(|&component_name| { + .filter_map(|component_name| { let results = blueprint.query_caches().latest_at( blueprint.store(), query, diff --git a/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs b/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs index b13c988a7f71..283d78186621 100644 --- a/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs +++ b/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs @@ -428,7 +428,11 @@ impl DataQueryPropertyResolver<'_> { if let Some(recursive_override_subtree) = blueprint.tree().subtree(&recursive_override_path) { - for &component_name in recursive_override_subtree.entity.components.keys() { + for component_name in blueprint + .store() + .all_components_on_all_timelines(&recursive_override_subtree.path) + .unwrap_or_default() + { let results = blueprint.query_caches().latest_at( blueprint.store(), blueprint_query, @@ -459,7 +463,11 @@ impl DataQueryPropertyResolver<'_> { if let Some(individual_override_subtree) = blueprint.tree().subtree(&individual_override_path) { - for &component_name in individual_override_subtree.entity.components.keys() { + for component_name in blueprint + .store() + .all_components_on_all_timelines(&individual_override_subtree.path) + .unwrap_or_default() + { let results = blueprint.query_caches().latest_at( blueprint.store(), blueprint_query, From dae84a8101cf4ee1c99d3db85011e8d72f59daaf Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:12:10 +0200 Subject: [PATCH 07/43] use entity subtree size in `entity_tree_stats_ui` --- crates/viewer/re_data_ui/src/item_ui.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index 1447130f798b..bc931c3edbeb 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -291,11 +291,16 @@ pub fn instance_path_parts_buttons( .response } -fn entity_tree_stats_ui(ui: &mut egui::Ui, timeline: &Timeline, tree: &EntityTree) { +fn entity_tree_stats_ui( + ui: &mut egui::Ui, + timeline: &Timeline, + db: &re_entity_db::EntityDb, + tree: &EntityTree, +) { use re_format::format_bytes; // Show total bytes used in whole subtree - let total_bytes = tree.subtree.data_bytes(); + let total_bytes = db.size_of_subtree_on_timeline(timeline, &tree.path); let subtree_caveat = if tree.children.is_empty() { "" @@ -548,7 +553,7 @@ pub fn instance_hover_card_ui( if instance_path.instance.is_all() { if let Some(subtree) = ctx.recording().tree().subtree(&instance_path.entity_path) { - entity_tree_stats_ui(ui, &query.timeline(), subtree); + entity_tree_stats_ui(ui, &query.timeline(), db, subtree); } } else { // TODO(emilk): per-component stats From fda44be06ff6b741c7cf2158fc087843efb77e8b Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:22:04 +0200 Subject: [PATCH 08/43] update time UI to not use `SubtreeInfo` --- crates/viewer/re_data_ui/src/item_ui.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index bc931c3edbeb..b87e77fdaa3b 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -315,14 +315,14 @@ fn entity_tree_stats_ui( let mut data_rate = None; // Try to estimate data-rate - if let Some(time_histogram) = tree.subtree.time_histogram.get(timeline) { - // `num_events` is approximate - we could be logging a Tensor image and a transform - // at _almost_ approximately the same time, but it should only count as one fence-post. - let num_events = time_histogram.total_count(); // TODO(emilk): we should ask the histogram to count the number of non-zero keys instead. + if db.store().entity_has_data_on_timeline(timeline, &tree.path) { + let num_events = db + .store() + .num_events_on_timeline_for_all_components(timeline, &tree.path); - if let (Some(min_time), Some(max_time)) = - (time_histogram.min_key(), time_histogram.max_key()) - { + if let Some(time_range) = db.store().time_range_for_entity(timeline, &tree.path) { + let min_time = time_range.min(); + let max_time = time_range.max(); if min_time < max_time && 1 < num_events { // Let's do our best to avoid fencepost errors. // If we log 1 MiB once every second, then after three @@ -333,9 +333,9 @@ fn entity_tree_stats_ui( // t: 0s 1s 2s // data: 1MiB 1MiB 1MiB - let duration = max_time - min_time; + let duration = max_time.as_f64() - min_time.as_f64(); - let mut bytes_per_time = total_bytes as f64 / duration as f64; + let mut bytes_per_time = total_bytes as f64 / duration; // Fencepost adjustment: bytes_per_time *= (num_events - 1) as f64 / num_events as f64; From 71bad344a28b49a7708e317beb12e51fded769a3 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:25:14 +0200 Subject: [PATCH 09/43] remove time histogram usage from component data ui --- crates/viewer/re_data_ui/src/item_ui.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index b87e77fdaa3b..870308aff1f6 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -398,7 +398,10 @@ pub fn component_path_button_to( db: &re_entity_db::EntityDb, ) -> egui::Response { let item = Item::ComponentPath(component_path.clone()); - let is_static = db.is_component_static(component_path).unwrap_or_default(); + let is_static = db.store().entity_has_static_component( + component_path.entity_path(), + component_path.component_name(), + ); let icon = if is_static { &icons::COMPONENT_STATIC } else { From 339c66fe68c8ca88a6790aa852f678b9250cda6a Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:29:27 +0200 Subject: [PATCH 10/43] remove `EntityInfo` usage from selection panel --- crates/viewer/re_selection_panel/src/selection_panel.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/viewer/re_selection_panel/src/selection_panel.rs b/crates/viewer/re_selection_panel/src/selection_panel.rs index c24f1306db7d..b2e3d701a7a8 100644 --- a/crates/viewer/re_selection_panel/src/selection_panel.rs +++ b/crates/viewer/re_selection_panel/src/selection_panel.rs @@ -155,9 +155,12 @@ impl SelectionPanel { match item { Item::ComponentPath(component_path) => { let entity_path = &component_path.entity_path; + let component_name = &component_path.component_name; let (query, db) = guess_query_and_db_for_selected_entity(ctx, entity_path); - let is_static = db.is_component_static(component_path).unwrap_or_default(); + let is_static = db + .store() + .entity_has_static_component(entity_path, component_name); ui.list_item_flat_noninteractive( PropertyContent::new("Component type").value_text(if is_static { @@ -718,7 +721,9 @@ fn item_tile( let component_name = &component_path.component_name; let (_query, db) = guess_query_and_db_for_selected_entity(ctx, entity_path); - let is_static = db.is_component_static(component_path).unwrap_or_default(); + let is_static = db + .store() + .entity_has_static_component(entity_path, component_name); Some( ItemTitle::new(component_name.short_name()) From bcc92d456248b8bdc29d5c40ccd8d7d2491ffe93 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:48:25 +0200 Subject: [PATCH 11/43] remove `EntityInfo` usage from 2d/3d space views --- crates/viewer/re_space_view_spatial/src/view_2d.rs | 8 +++++++- crates/viewer/re_space_view_spatial/src/view_3d.rs | 8 ++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/viewer/re_space_view_spatial/src/view_2d.rs b/crates/viewer/re_space_view_spatial/src/view_2d.rs index 3750b57e9eba..97a66e75f253 100644 --- a/crates/viewer/re_space_view_spatial/src/view_2d.rs +++ b/crates/viewer/re_space_view_spatial/src/view_2d.rs @@ -274,12 +274,15 @@ impl SpaceViewClass for SpatialSpaceView2D { fn count_non_nested_images_with_component( image_dimensions: &IntMap, entity_bucket: &IntSet, + entity_db: &re_entity_db::EntityDb, subtree: &EntityTree, component_name: &ComponentName, ) -> usize { if image_dimensions.contains_key(&subtree.path) { // bool true -> 1 - subtree.entity.components.contains_key(component_name) as usize + entity_db + .store() + .entity_has_component_on_any_timeline(&subtree.path, component_name) as usize } else if !entity_bucket .iter() .any(|e| e.is_descendant_of(&subtree.path)) @@ -293,6 +296,7 @@ fn count_non_nested_images_with_component( count_non_nested_images_with_component( image_dimensions, entity_bucket, + entity_db, child, component_name, ) @@ -361,6 +365,7 @@ fn recommended_space_views_with_image_splits( let image_count = count_non_nested_images_with_component( image_dimensions, entities, + ctx.recording(), subtree, &Image::indicator().name(), ); @@ -368,6 +373,7 @@ fn recommended_space_views_with_image_splits( let depth_count = count_non_nested_images_with_component( image_dimensions, entities, + ctx.recording(), subtree, &DepthImage::indicator().name(), ); diff --git a/crates/viewer/re_space_view_spatial/src/view_3d.rs b/crates/viewer/re_space_view_spatial/src/view_3d.rs index 59cf11abd5cd..17c0784f3ad7 100644 --- a/crates/viewer/re_space_view_spatial/src/view_3d.rs +++ b/crates/viewer/re_space_view_spatial/src/view_3d.rs @@ -279,8 +279,12 @@ impl SpaceViewClass for SpatialSpaceView3D { // There's also a strong argument to be made that ViewCoordinates implies a 3D space, thus changing the SpacialTopology accordingly! ctx.recording() .tree() - .visit_children_recursively(|path, info| { - if info.components.contains_key(&ViewCoordinates::name()) { + .visit_children_recursively(|path, _| { + if ctx + .recording() + .store() + .entity_has_component_on_any_timeline(path, &ViewCoordinates::name()) + { indicated_entities.insert(path.clone()); } }); From 28a130dd45be0a31140bfefebea9f7b36522ce48 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:21:05 +0200 Subject: [PATCH 12/43] actually chunkify time panel --- crates/store/re_entity_db/src/entity_db.rs | 18 +++++++++ crates/store/re_entity_db/src/entity_tree.rs | 26 +++++++++++++ crates/viewer/re_time_panel/src/lib.rs | 41 +++++++++++--------- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index df8a7a1120e6..1e231f9170f3 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -593,6 +593,24 @@ impl EntityDb { size } + + pub fn subtree_has_data_on_timeline( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> bool { + re_tracing::profile_function!(); + + let Some(subtree) = self.tree.subtree(entity_path) else { + return false; + }; + + subtree + .find_child_recursive(|path, _| { + self.store().entity_has_data_on_timeline(timeline, path) + }) + .is_some() + } } impl re_types_core::SizeBytes for EntityDb { diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index e80181c87f31..a92fc2bbc5b9 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -377,4 +377,30 @@ impl EntityTree { visit(self, &mut visitor); } + + pub fn find_child_recursive( + &self, + mut predicate: impl FnMut(&EntityPath, &EntityInfo) -> bool, + ) -> Option<&Self> { + use std::ops::ControlFlow; + + fn visit<'a>( + this: &'a EntityTree, + predicate: &mut impl FnMut(&EntityPath, &EntityInfo) -> bool, + ) -> ControlFlow<&'a EntityTree> { + if predicate(&this.path, &this.entity) { + return ControlFlow::Break(this); + }; + for child in this.children.values() { + visit(child, predicate)?; + } + ControlFlow::Continue(()) + } + + let result = visit(self, &mut predicate); + match result { + ControlFlow::Continue(()) => None, + ControlFlow::Break(v) => Some(v), + } + } } diff --git a/crates/viewer/re_time_panel/src/lib.rs b/crates/viewer/re_time_panel/src/lib.rs index 2fa898363b44..d7d3767dbcab 100644 --- a/crates/viewer/re_time_panel/src/lib.rs +++ b/crates/viewer/re_time_panel/src/lib.rs @@ -19,7 +19,7 @@ use egui::{pos2, Color32, CursorIcon, NumExt, Painter, PointerButton, Rect, Shap use re_context_menu::{context_menu_ui_for_item, SelectionUpdateBehavior}; use re_data_ui::DataUi as _; use re_data_ui::{item_ui::guess_instance_path_icon, sorted_component_list_for_ui}; -use re_entity_db::{EntityTree, InstancePath, TimeHistogram}; +use re_entity_db::{EntityTree, InstancePath}; use re_log_types::{ external::re_types_core::ComponentName, ComponentPath, EntityPath, EntityPathPart, ResolvedTimeRange, TimeInt, TimeReal, @@ -574,8 +574,6 @@ impl TimePanel { ui: &mut egui::Ui, show_root_as: &str, ) { - let tree_has_data_in_current_timeline = time_ctrl.tree_has_data_in_current_timeline(tree); - let db = match self.source { TimePanelSource::Recording => ctx.recording(), TimePanelSource::Blueprint => ctx.store_context.blueprint, @@ -693,6 +691,8 @@ impl TimePanel { // ---------------------------------------------- // show the data in the time area: + let tree_has_data_in_current_timeline = + entity_db.subtree_has_data_on_timeline(time_ctrl.timeline(), &tree.path); if is_visible && tree_has_data_in_current_timeline { let row_rect = Rect::from_x_y_ranges(time_area_response.rect.x_range(), response_rect.y_range()); @@ -744,17 +744,29 @@ impl TimePanel { } // If this is an entity: - if !tree.entity.components.is_empty() { - for component_name in sorted_component_list_for_ui(tree.entity.components.keys()) { - let data = &tree.entity.components[&component_name]; - - let is_static = data.is_static(); - let component_has_data_in_current_timeline = - time_ctrl.component_has_data_in_current_timeline(data); + if let Some(components) = entity_db + .store() + .all_components_on_all_timelines(&tree.path) + { + for component_name in sorted_component_list_for_ui(components.iter()) { + let is_static = entity_db + .store() + .entity_has_static_component(&tree.path, &component_name); let component_path = ComponentPath::new(tree.path.clone(), component_name); let short_component_name = component_path.component_name.short_name(); let item = TimePanelItem::component_path(component_path.clone()); + let timeline = time_ctrl.timeline(); + + let component_has_data_in_current_timeline = entity_db + .store() + .entity_has_component(time_ctrl.timeline(), &tree.path, &component_name); + + let total_num_messages = entity_db.store().num_events_on_timeline_for_component( + time_ctrl.timeline(), + &tree.path, + component_name, + ); let response = ui .list_item() @@ -786,14 +798,6 @@ impl TimePanel { let response_rect = response.rect; - let timeline = time_ctrl.timeline(); - - let empty_messages_over_time = TimeHistogram::default(); - let messages_over_time = data.get(timeline).unwrap_or(&empty_messages_over_time); - - // `data.times` does not contain static. Need to add those manually: - let total_num_messages = - messages_over_time.total_count() + data.num_static_messages(); response.on_hover_ui(|ui| { if total_num_messages == 0 { ui.label(ui.ctx().warning_text(format!( @@ -847,6 +851,7 @@ impl TimePanel { response_rect.left()..=ui.max_rect().right(), response_rect.y_range(), ); + let is_visible = ui.is_rect_visible(full_width_rect); if is_visible && component_has_data_in_current_timeline { // show the data in the time area: From a186a00990b44e14ae7a637c22b9fe614c38c39d Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:21:39 +0200 Subject: [PATCH 13/43] rm dead code --- .../re_viewer_context/src/time_control.rs | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/crates/viewer/re_viewer_context/src/time_control.rs b/crates/viewer/re_viewer_context/src/time_control.rs index cb003750917c..bd1651944c0e 100644 --- a/crates/viewer/re_viewer_context/src/time_control.rs +++ b/crates/viewer/re_viewer_context/src/time_control.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use re_entity_db::{EntityTree, TimeCounts, TimeHistogramPerTimeline, TimesPerTimeline}; +use re_entity_db::{TimeCounts, TimesPerTimeline}; use re_log_types::{ Duration, ResolvedTimeRange, ResolvedTimeRangeF, TimeInt, TimeReal, TimeType, Timeline, }; @@ -571,22 +571,6 @@ impl TimeControl { state.view = None; } } - - /// Returns whether the given tree has any data logged in the current timeline, - /// or has any timeless messages. - pub fn tree_has_data_in_current_timeline(&self, tree: &EntityTree) -> bool { - let top_time_histogram = &tree.subtree.time_histogram; - top_time_histogram.has_timeline(self.timeline()) - || top_time_histogram.num_static_messages() > 0 - } - - /// Returns whether the given component has any data logged in the current timeline. - pub fn component_has_data_in_current_timeline( - &self, - component_stat: &TimeHistogramPerTimeline, - ) -> bool { - component_stat.has_timeline(self.timeline()) - } } fn min(values: &TimeCounts) -> TimeInt { From 92935092b99db3c9df1678a9da0daf3ad6be97c7 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:33:48 +0200 Subject: [PATCH 14/43] remove time histogram usage from EntityLatestAtResults data ui --- crates/viewer/re_data_ui/src/component.rs | 60 ++++++++++++----------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/crates/viewer/re_data_ui/src/component.rs b/crates/viewer/re_data_ui/src/component.rs index 3b7f44ad5e46..e721b7f28c78 100644 --- a/crates/viewer/re_data_ui/src/component.rs +++ b/crates/viewer/re_data_ui/src/component.rs @@ -72,36 +72,38 @@ impl<'a> DataUi for EntityLatestAtResults<'a> { // if the component is static, we display extra diagnostic information if self.results.is_static() { - if let Some(histogram) = db - .tree() - .subtree(&self.entity_path) - .and_then(|tree| tree.entity.components.get(&component_name)) - { - if histogram.num_static_messages() > 1 { - ui.label(ui.ctx().warning_text(format!( - "Static component value was overridden {} times", - histogram.num_static_messages().saturating_sub(1), - ))) - .on_hover_text( - "When a static component is logged multiple times, only the last value \ - is stored. Previously logged values are overwritten and not \ - recoverable.", - ); - } + let static_message_count = db + .store() + .num_events_for_static_component(&self.entity_path, component_name); + if static_message_count > 1 { + ui.label(ui.ctx().warning_text(format!( + "Static component value was overridden {} times", + static_message_count.saturating_sub(1), + ))) + .on_hover_text( + "When a static component is logged multiple times, only the last value \ + is stored. Previously logged values are overwritten and not \ + recoverable.", + ); + } - let timeline_message_count = histogram.num_temporal_messages(); - if timeline_message_count > 0 { - ui.label(ui.ctx().error_text(format!( - "Static component has {} event{} logged on timelines", - timeline_message_count, - if timeline_message_count > 1 { "s" } else { "" } - ))) - .on_hover_text( - "Components should be logged either as static or on timelines, but \ - never both. Values for static components logged to timelines cannot be \ - displayed.", - ); - } + let temporal_message_count = + db.store().num_events_on_timeline_for_temporal_component( + &query.timeline(), + &self.entity_path, + component_name, + ); + if temporal_message_count > 0 { + ui.label(ui.ctx().error_text(format!( + "Static component has {} event{} logged on timelines", + temporal_message_count, + if temporal_message_count > 1 { "s" } else { "" } + ))) + .on_hover_text( + "Components should be logged either as static or on timelines, but \ + never both. Values for static components logged to timelines cannot be \ + displayed.", + ); } } } From 8e8560eea60102331e353f7f9108592b2b63af36 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:42:57 +0200 Subject: [PATCH 15/43] remove `EntityInfo` from entity tree visitors --- crates/store/re_entity_db/src/entity_db.rs | 6 ++---- crates/store/re_entity_db/src/entity_tree.rs | 12 +++++------ .../src/actions/collapse_expand_all.rs | 4 ++-- .../src/space_view_entity_picker.rs | 2 +- .../re_space_view_spatial/src/view_3d.rs | 20 +++++++++---------- .../re_time_panel/src/data_density_graph.rs | 2 +- .../re_viewport_blueprint/src/space_view.rs | 2 +- 7 files changed, 22 insertions(+), 26 deletions(-) diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index 1e231f9170f3..b74c3f3c8985 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -587,7 +587,7 @@ impl EntityDb { }; let mut size = 0; - subtree.visit_children_recursively(|path, _| { + subtree.visit_children_recursively(|path| { size += self.store().size_of_entity_on_timeline(timeline, path); }); @@ -606,9 +606,7 @@ impl EntityDb { }; subtree - .find_child_recursive(|path, _| { - self.store().entity_has_data_on_timeline(timeline, path) - }) + .find_child_recursive(|path| self.store().entity_has_data_on_timeline(timeline, path)) .is_some() } } diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index a92fc2bbc5b9..e63129a9f9a3 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -367,9 +367,9 @@ impl EntityTree { } // Invokes visitor for `self` and all children recursively. - pub fn visit_children_recursively(&self, mut visitor: impl FnMut(&EntityPath, &EntityInfo)) { - fn visit(this: &EntityTree, visitor: &mut impl FnMut(&EntityPath, &EntityInfo)) { - visitor(&this.path, &this.entity); + pub fn visit_children_recursively(&self, mut visitor: impl FnMut(&EntityPath)) { + fn visit(this: &EntityTree, visitor: &mut impl FnMut(&EntityPath)) { + visitor(&this.path); for child in this.children.values() { visit(child, visitor); } @@ -380,15 +380,15 @@ impl EntityTree { pub fn find_child_recursive( &self, - mut predicate: impl FnMut(&EntityPath, &EntityInfo) -> bool, + mut predicate: impl FnMut(&EntityPath) -> bool, ) -> Option<&Self> { use std::ops::ControlFlow; fn visit<'a>( this: &'a EntityTree, - predicate: &mut impl FnMut(&EntityPath, &EntityInfo) -> bool, + predicate: &mut impl FnMut(&EntityPath) -> bool, ) -> ControlFlow<&'a EntityTree> { - if predicate(&this.path, &this.entity) { + if predicate(&this.path) { return ControlFlow::Break(this); }; for child in this.children.values() { diff --git a/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs b/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs index db75b6b490ad..f65215413fff 100644 --- a/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs +++ b/crates/viewer/re_context_menu/src/actions/collapse_expand_all.rs @@ -91,7 +91,7 @@ impl ContextMenuAction for CollapseExpandAllAction { return; }; - subtree.visit_children_recursively(|entity_path, _| { + subtree.visit_children_recursively(|entity_path| { CollapseScope::BlueprintTree .data_result(*space_view_id, entity_path.clone()) .set_open(&ctx.egui_context, self.open()); @@ -108,7 +108,7 @@ impl ContextMenuAction for CollapseExpandAllAction { return; }; - subtree.visit_children_recursively(|entity_path, _| { + subtree.visit_children_recursively(|entity_path| { CollapseScope::StreamsTree .entity(entity_path.clone()) .set_open(&ctx.egui_context, self.open()); diff --git a/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs b/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs index 616911d67893..3018d5d71a46 100644 --- a/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs +++ b/crates/viewer/re_selection_panel/src/space_view_entity_picker.rs @@ -326,7 +326,7 @@ fn create_entity_add_info( &space_view.space_origin, ); - tree.visit_children_recursively(|entity_path, _| { + tree.visit_children_recursively(|entity_path| { let can_add: CanAddToSpaceView = if visualizable_entities.iter().any(|(_, entities)| entities.contains(entity_path)) { CanAddToSpaceView::Compatible { diff --git a/crates/viewer/re_space_view_spatial/src/view_3d.rs b/crates/viewer/re_space_view_spatial/src/view_3d.rs index 17c0784f3ad7..0b3a0a580441 100644 --- a/crates/viewer/re_space_view_spatial/src/view_3d.rs +++ b/crates/viewer/re_space_view_spatial/src/view_3d.rs @@ -277,17 +277,15 @@ impl SpaceViewClass for SpatialSpaceView3D { // It's tempting to add a visualizer for view coordinates so that it's already picked up via `entities_with_indicator_for_visualizer_kind`. // Is there a nicer way for this or do we want a visualizer for view coordinates anyways? // There's also a strong argument to be made that ViewCoordinates implies a 3D space, thus changing the SpacialTopology accordingly! - ctx.recording() - .tree() - .visit_children_recursively(|path, _| { - if ctx - .recording() - .store() - .entity_has_component_on_any_timeline(path, &ViewCoordinates::name()) - { - indicated_entities.insert(path.clone()); - } - }); + ctx.recording().tree().visit_children_recursively(|path| { + if ctx + .recording() + .store() + .entity_has_component_on_any_timeline(path, &ViewCoordinates::name()) + { + indicated_entities.insert(path.clone()); + } + }); // Spawn a space view at each subspace that has any potential 3D content. // Note that visualizability filtering is all about being in the right subspace, diff --git a/crates/viewer/re_time_panel/src/data_density_graph.rs b/crates/viewer/re_time_panel/src/data_density_graph.rs index 671b0c7eaf53..affa3adbf549 100644 --- a/crates/viewer/re_time_panel/src/data_density_graph.rs +++ b/crates/viewer/re_time_panel/src/data_density_graph.rs @@ -716,7 +716,7 @@ fn visit_relevant_chunks( visitor(Arc::clone(&chunk), chunk_timeline.time_range(), num_events); } } else if let Some(subtree) = db.tree().subtree(entity_path) { - subtree.visit_children_recursively(|entity_path, _| { + subtree.visit_children_recursively(|entity_path| { for chunk in db .store() .range_relevant_chunks_for_all_components(&query, entity_path) diff --git a/crates/viewer/re_viewport_blueprint/src/space_view.rs b/crates/viewer/re_viewport_blueprint/src/space_view.rs index 9fb6f3dd81f4..46856545e282 100644 --- a/crates/viewer/re_viewport_blueprint/src/space_view.rs +++ b/crates/viewer/re_viewport_blueprint/src/space_view.rs @@ -251,7 +251,7 @@ impl SpaceViewBlueprint { // Create pending write operations to duplicate the entire subtree // TODO(jleibs): This should be a helper somewhere. if let Some(tree) = blueprint.tree().subtree(¤t_path) { - tree.visit_children_recursively(|path, _| { + tree.visit_children_recursively(|path| { let sub_path: EntityPath = new_path .iter() .chain(&path[current_path.len()..]) From a19cd6b579839a60fd6120e26fcc57f56a335e34 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:43:02 +0200 Subject: [PATCH 16/43] remove some dead code --- crates/store/re_entity_db/src/entity_db.rs | 22 +- crates/store/re_entity_db/src/entity_tree.rs | 22 +- crates/store/re_entity_db/tests/clear.rs | 4 +- .../re_entity_db/tests/time_histograms.rs | 705 ------------------ 4 files changed, 7 insertions(+), 746 deletions(-) delete mode 100644 crates/store/re_entity_db/tests/time_histograms.rs diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index b74c3f3c8985..ac53c440c04b 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -9,8 +9,8 @@ use re_chunk_store::{ GarbageCollectionTarget, }; use re_log_types::{ - ApplicationId, ComponentPath, EntityPath, EntityPathHash, LogMsg, ResolvedTimeRange, - ResolvedTimeRangeF, SetStoreInfo, StoreId, StoreInfo, StoreKind, Timeline, + ApplicationId, EntityPath, EntityPathHash, LogMsg, ResolvedTimeRange, ResolvedTimeRangeF, + SetStoreInfo, StoreId, StoreInfo, StoreKind, Timeline, }; use crate::{Error, TimesPerTimeline}; @@ -270,24 +270,6 @@ impl EntityDb { self.tree().subtree.time_histogram.get(timeline) } - /// Total number of static messages for any entity. - pub fn num_static_messages(&self) -> u64 { - self.tree.num_static_messages_recursive() - } - - /// Returns whether a component is static. - pub fn is_component_static(&self, component_path: &ComponentPath) -> Option { - if let Some(entity_tree) = self.tree().subtree(component_path.entity_path()) { - entity_tree - .entity - .components - .get(&component_path.component_name) - .map(|component_histogram| component_histogram.is_static()) - } else { - None - } - } - #[inline] pub fn num_rows(&self) -> u64 { self.data_store.stats().total().total_num_rows diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index e63129a9f9a3..5e3e68240b7e 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -216,24 +216,8 @@ impl EntityTree { self.children.is_empty() } - pub fn num_children_and_fields(&self) -> usize { - self.children.len() + self.entity.components.len() - } - - /// Number of timeless messages in this tree, or any child, recursively. - pub fn num_static_messages_recursive(&self) -> u64 { - self.subtree.time_histogram.num_static_messages() - } - - pub fn time_histogram_for_component( - &self, - timeline: &Timeline, - component_name: impl Into, - ) -> Option<&crate::TimeHistogram> { - self.entity - .components - .get(&component_name.into()) - .and_then(|per_timeline| per_timeline.get(timeline)) + pub fn num_children(&self) -> usize { + self.children.len() } /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s. @@ -348,7 +332,7 @@ impl EntityTree { children.retain(|_, child| { child.on_store_deletions(&subtree_events); - child.num_children_and_fields() > 0 + child.num_children() > 0 }); } diff --git a/crates/store/re_entity_db/tests/clear.rs b/crates/store/re_entity_db/tests/clear.rs index 5f3e32f45745..3de66f176d17 100644 --- a/crates/store/re_entity_db/tests/clear.rs +++ b/crates/store/re_entity_db/tests/clear.rs @@ -414,7 +414,7 @@ fn clear_and_gc() -> anyhow::Result<()> { // Insert a component, then clear it, then GC. { // EntityTree is Empty when we start - assert_eq!(db.tree().num_children_and_fields(), 0); + assert_eq!(db.tree().num_children(), 0); let point = MyPoint::new(1.0, 2.0); @@ -450,7 +450,7 @@ fn clear_and_gc() -> anyhow::Result<()> { assert_eq!(stats.temporal_chunks.total_num_rows, 0); // EntityTree should be empty again when we end since everything was GC'd - assert_eq!(db.tree().num_children_and_fields(), 0); + assert_eq!(db.tree().num_children(), 0); } Ok(()) diff --git a/crates/store/re_entity_db/tests/time_histograms.rs b/crates/store/re_entity_db/tests/time_histograms.rs deleted file mode 100644 index a83fd10b8eec..000000000000 --- a/crates/store/re_entity_db/tests/time_histograms.rs +++ /dev/null @@ -1,705 +0,0 @@ -// https://github.com/rust-lang/rust-clippy/issues/10011 -#![cfg(test)] - -use std::{collections::BTreeSet, sync::Arc}; - -use re_chunk::{Chunk, ChunkId, RowId}; -use re_chunk_store::GarbageCollectionOptions; -use re_entity_db::EntityDb; -use re_int_histogram::RangeI64; -use re_log_types::{ - example_components::{MyColor, MyIndex, MyPoint}, - EntityPath, StoreId, TimeInt, TimePoint, Timeline, -}; -use re_types_core::{components::ClearIsRecursive, ComponentName, Loggable}; - -// --- - -// Complete test suite for our various time histograms which, among other things, power the -// timeline widget. -#[test] -fn time_histograms() -> anyhow::Result<()> { - let mut db = EntityDb::new(StoreId::random(re_log_types::StoreKind::Recording)); - - let timeline_frame = Timeline::new_sequence("frame"); - let timeline_other = Timeline::new_temporal("other"); - let timeline_yet_another = Timeline::new_sequence("yet_another"); - - let entity_parent: EntityPath = "parent".into(); - let entity_grandchild: EntityPath = "parent/child/grandchild".into(); - let entity_unrelated: EntityPath = "very/unrelated".into(); - - // Single top-level entity, explicitly logged `MyIndex`s. - { - let chunk = Chunk::builder(entity_parent.clone()) - .with_component_batches( - RowId::new(), - TimePoint::from_iter([ - (timeline_frame, 42), // - (timeline_other, 666), // - (timeline_yet_another, 1), // - ]), - [&MyIndex::from_iter(0..10) as _], - ) - .build()?; - - db.add_chunk(&Arc::new(chunk))?; - - // times per timeline - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42])), - (&timeline_other, Some(&[666])), - (&timeline_yet_another, Some(&[1])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline ('parent') - assert_histogram_for_component( - &db, - &entity_parent, - MyIndex::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 3], - ); - } - - // Grand-child, multiple components, auto-generated `MyIndex`s. - { - let chunk = { - let num_instances = 3; - let points: Vec<_> = (0..num_instances) - .map(|i| MyPoint::new(0.0, i as f32)) - .collect(); - let colors = vec![MyColor::from(0xFF0000FF)]; - Chunk::builder(entity_grandchild.clone()) - .with_component_batches( - RowId::new(), - TimePoint::from_iter([ - (timeline_frame, 42), // - (timeline_yet_another, 1), // - ]), - [&points as _, &colors as _], - ) - .build()? - }; - let chunk = Arc::new(chunk); - - db.add_chunk(&chunk)?; - - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42])), - (&timeline_other, Some(&[666])), - (&timeline_yet_another, Some(&[1])), - ], - ); - - // histograms per timeline - // NOTE: notice how autogenerated instance keys are ignored. - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 3)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 3)])), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline ('parent') - assert_histogram_for_component( - &db, - &entity_parent, - MyIndex::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 3], - ); - // NOTE: per-component histograms are not recursive! - assert_histogram_for_component( - &db, - &entity_parent, - MyPoint::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_parent, - MyColor::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - - // histograms per component per timeline ('parent/child/grandchild') - // NOTE: notice how autogenerated instance keys are ignored! - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyPoint::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyColor::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 2], - ); - - db.add_chunk(&Arc::new(chunk.clone_as(ChunkId::new(), RowId::new())))?; // same chunk a second time! - - // times per timeline - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42])), - (&timeline_other, Some(&[666])), - (&timeline_yet_another, Some(&[1])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 5)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 5)])), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline ('parent/child/grandchild') - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyPoint::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyColor::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - } - - // Grand-child, timeless additions. - { - let chunk = { - let num_instances = 6; - let colors = vec![MyColor::from(0x00DD00FF); num_instances]; - Chunk::builder("entity".into()) - .with_component_batches( - RowId::new(), - TimePoint::default(), - [ - &MyIndex::from_iter(0..num_instances as _) as _, - &colors as _, - ], - ) - .build()? - }; - - db.add_chunk(&Arc::new(chunk))?; - - // times per timeline - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42])), - (&timeline_other, Some(&[666])), - (&timeline_yet_another, Some(&[1])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 5)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 5)])), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_parent, - MyPoint::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_parent, - MyColor::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyPoint::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyColor::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - } - - // Completely unrelated entity. - { - let chunk = { - let num_instances = 3; - let points: Vec<_> = (0..num_instances) - .map(|i| MyPoint::new(0.0, i as f32)) - .collect(); - let colors = vec![MyColor::from(0xFF0000FF)]; - Chunk::builder(entity_unrelated.clone()) - .with_component_batches( - RowId::new(), - TimePoint::from_iter([ - (timeline_frame, 1234), // - (timeline_other, 1235), // - (timeline_yet_another, 1236), // - ]), - [ - &MyIndex::from_iter(0..num_instances) as _, - &points as _, - &colors as _, - ], - ) - .build()? - }; - - db.add_chunk(&Arc::new(chunk))?; - - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42, 1234])), - (&timeline_other, Some(&[666, 1235])), - (&timeline_yet_another, Some(&[1, 1236])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - ( - &timeline_frame, - Some(&[(RangeI64::new(42, 42), 5), (RangeI64::new(1234, 1234), 3)]), - ), - ( - &timeline_other, - Some(&[(RangeI64::new(666, 666), 1), (RangeI64::new(1235, 1235), 3)]), - ), - ( - &timeline_yet_another, - Some(&[(RangeI64::new(1, 1), 5), (RangeI64::new(1236, 1236), 3)]), - ), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_parent, - MyIndex::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 3], - ); - // NOTE: per-component histograms are not recursive! - assert_histogram_for_component( - &db, - &entity_parent, - MyPoint::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_parent, - MyColor::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - - // histograms per component per timeline ('very/unrelated') - assert_histogram_for_component( - &db, - &entity_unrelated, - MyIndex::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(1234, 1234), 1)])), - (&timeline_other, Some(&[(RangeI64::new(1235, 1235), 1)])), - ( - &timeline_yet_another, - Some(&[(RangeI64::new(1236, 1236), 1)]), - ), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_unrelated, - MyPoint::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(1234, 1234), 1)])), - (&timeline_other, Some(&[(RangeI64::new(1235, 1235), 1)])), - ( - &timeline_yet_another, - Some(&[(RangeI64::new(1236, 1236), 1)]), - ), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_unrelated, - MyColor::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(1234, 1234), 1)])), - (&timeline_other, Some(&[(RangeI64::new(1235, 1235), 1)])), - ( - &timeline_yet_another, - Some(&[(RangeI64::new(1236, 1236), 1)]), - ), - ] as [(_, Option<&[_]>); 3], - ); - } - - // Immediate clear. - { - let chunk = { - Chunk::builder(entity_parent.clone()) - .with_component_batches( - RowId::new(), - TimePoint::from_iter([ - (timeline_frame, 1000), // - ]), - [&[ClearIsRecursive(true.into())] as _], - ) - .build()? - }; - - db.add_chunk(&Arc::new(chunk))?; - - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[42, 1000, 1234])), - (&timeline_other, Some(&[666, 1235])), - (&timeline_yet_another, Some(&[1, 1236])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - ( - &timeline_frame, - Some(&[ - (RangeI64::new(42, 42), 5), - (RangeI64::new(1000, 1000), 1), - (RangeI64::new(1234, 1234), 3), - ]), - ), - ( - &timeline_other, - Some(&[(RangeI64::new(666, 666), 1), (RangeI64::new(1235, 1235), 3)]), - ), - ( - &timeline_yet_another, - Some(&[(RangeI64::new(1, 1), 5), (RangeI64::new(1236, 1236), 3)]), - ), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_parent, - MyIndex::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 1)])), - (&timeline_other, Some(&[(RangeI64::new(666, 666), 1)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 1)])), - ] as [(_, Option<&[_]>); 3], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyPoint::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyColor::name(), - [ - (&timeline_frame, Some(&[(RangeI64::new(42, 42), 2)])), - (&timeline_yet_another, Some(&[(RangeI64::new(1, 1), 2)])), - ] as [(_, Option<&[_]>); 2], - ); - } - - // Full GC - { - db.gc(&GarbageCollectionOptions::gc_everything()); - - assert_times_per_timeline( - &db, - [ - (&Timeline::log_time(), Some(&[] as &[i64])), - (&timeline_frame, Some(&[])), - (&timeline_other, Some(&[])), - (&timeline_yet_another, Some(&[])), - ], - ); - - // histograms per timeline - assert_recursive_histogram( - &db, - [ - (&Timeline::log_time(), None), - (&timeline_frame, Some(&[])), - (&timeline_other, Some(&[])), - (&timeline_yet_another, Some(&[])), - ] as [(_, Option<&[_]>); 4], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_parent, - MyIndex::name(), - [ - (&timeline_frame, Some(&[])), - (&timeline_other, Some(&[])), - (&timeline_yet_another, Some(&[])), - ] as [(_, Option<&[_]>); 3], - ); - - // histograms per component per timeline - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [ - (&timeline_frame, None), - (&timeline_other, None), - (&timeline_yet_another, None), - ] as [(_, Option<&[_]>); 3], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyIndex::name(), - [(&timeline_frame, None), (&timeline_yet_another, None)] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyPoint::name(), - [ - (&timeline_frame, Some(&[])), - (&timeline_yet_another, Some(&[])), - ] as [(_, Option<&[_]>); 2], - ); - assert_histogram_for_component( - &db, - &entity_grandchild, - MyColor::name(), - [ - (&timeline_frame, Some(&[])), - (&timeline_yet_another, Some(&[])), - ] as [(_, Option<&[_]>); 2], - ); - } - - Ok(()) -} - -/// Checks the state of the global time tracker (at the `EntityDb` level). -fn assert_times_per_timeline<'a>( - db: &EntityDb, - expected: impl IntoIterator)>, -) { - for (timeline, expected_times) in expected { - let times = db.times_per_timeline().get(timeline); - - if let Some(expected) = expected_times { - let times: BTreeSet<_> = times.unwrap().keys().copied().collect(); - let expected: BTreeSet<_> = expected - .iter() - .map(|&t| TimeInt::try_from(t).unwrap()) - .collect(); - similar_asserts::assert_eq!(expected, times); - } else { - assert!(times.is_none()); - } - } -} - -/// Checks the state of the per-EntityTree recursive time tracker. -fn assert_recursive_histogram<'a>( - db: &EntityDb, - expected: impl IntoIterator)>, -) { - for (timeline, expected_times) in expected { - let histo = db.time_histogram(timeline); - - if let Some(expected) = expected_times { - if expected.is_empty() { - assert!(histo.is_none()); - continue; - } - let histo = histo.unwrap(); - let ranges = histo.range(i64::MIN.., 0).collect::>(); - let expected: Vec<_> = expected.to_vec(); - similar_asserts::assert_eq!(expected, ranges); - } else { - assert!(histo.is_none()); - } - } -} - -/// Checks the state of the per-`EntityTree` per-`ComponentName` flat time tracker. -fn assert_histogram_for_component<'a>( - db: &EntityDb, - entity_path: &EntityPath, - component_name: ComponentName, - expected: impl IntoIterator)>, -) { - for (timeline, expected_times) in expected { - let histo = db - .tree() - .subtree(entity_path) - .and_then(|tree| tree.time_histogram_for_component(timeline, component_name)); - - if let Some(expected) = expected_times { - let expected: Vec<_> = expected.to_vec(); - if expected.is_empty() { - assert!(histo.is_none()); - continue; - } - let histo = histo.unwrap(); - let ranges = histo.range(i64::MIN.., 0).collect::>(); - similar_asserts::assert_eq!(expected, ranges); - } else { - assert!(histo.is_none()); - } - } -} From c27b0b2dc9730271c8393f82627a7f7e6ff96a3e Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:12:41 +0200 Subject: [PATCH 17/43] add query method --- crates/store/re_chunk_store/src/query.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 782141b1cb11..f1ac0b40ddf9 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -612,6 +612,28 @@ impl ChunkStore { false } + pub fn entity_has_data_on_any_timeline(&self, entity_path: &EntityPath) -> bool { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + if self.static_chunk_ids_per_entity.get(entity_path).is_some() { + return true; + } + + if let Some(temporal_chunk_ids_per_timeline) = + self.temporal_chunk_ids_per_entity.get(entity_path) + { + for chunk_id_set in temporal_chunk_ids_per_timeline.values() { + if !chunk_id_set.per_start_time.is_empty() { + return true; + } + } + } + + false + } + pub fn num_events_for_static_component( &self, entity_path: &EntityPath, From 57538343dd3dc77f0975a20f056c0f67d64e6558 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:31:27 +0200 Subject: [PATCH 18/43] remove `EntityTree.entity` --- crates/store/re_chunk_store/src/query.rs | 120 +++++++++++-------- crates/store/re_entity_db/src/entity_db.rs | 26 ++-- crates/store/re_entity_db/src/entity_tree.rs | 94 +++------------ crates/store/re_entity_db/tests/clear.rs | 4 +- 4 files changed, 101 insertions(+), 143 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index f1ac0b40ddf9..557bb109eec0 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -168,6 +168,79 @@ impl ChunkStore { static_components.contains_key(component_name) } + pub fn entity_has_data_on_timeline( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> bool { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + if self.static_chunk_ids_per_entity.get(entity_path).is_some() { + // Static data exists on all timelines + return true; + } + + if let Some(temporal_chunk_ids_per_timeline) = + self.temporal_chunk_ids_per_entity.get(entity_path) + { + if temporal_chunk_ids_per_timeline.contains_key(timeline) { + return true; + } + } + + false + } + + pub fn entity_has_data_on_any_timeline(&self, entity_path: &EntityPath) -> bool { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + if self.static_chunk_ids_per_entity.get(entity_path).is_some() { + return true; + } + + if let Some(temporal_chunk_ids_per_timeline) = + self.temporal_chunk_ids_per_entity.get(entity_path) + { + for chunk_id_set in temporal_chunk_ids_per_timeline.values() { + if !chunk_id_set.per_start_time.is_empty() { + return true; + } + } + } + + false + } + + pub fn entity_has_any_component_on_any_timeline(&self, entity_path: &EntityPath) -> bool { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) + { + if !static_chunks_per_component.is_empty() { + return true; + } + } + + if let Some(temporal_chunks_per_timeline) = self + .temporal_chunk_ids_per_entity_per_component + .get(entity_path) + { + for temporal_chunks_per_component in temporal_chunks_per_timeline.values() { + if !temporal_chunks_per_component.is_empty() { + return true; + } + } + } + + false + } + /// Find the earliest time at which something was logged for a given entity on the specified /// timeline. /// @@ -587,53 +660,6 @@ impl ChunkStore { total_events } - pub fn entity_has_data_on_timeline( - &self, - timeline: &Timeline, - entity_path: &EntityPath, - ) -> bool { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - if self.static_chunk_ids_per_entity.get(entity_path).is_some() { - // Static data exists on all timelines - return true; - } - - if let Some(temporal_chunk_ids_per_timeline) = - self.temporal_chunk_ids_per_entity.get(entity_path) - { - if temporal_chunk_ids_per_timeline.contains_key(timeline) { - return true; - } - } - - false - } - - pub fn entity_has_data_on_any_timeline(&self, entity_path: &EntityPath) -> bool { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - if self.static_chunk_ids_per_entity.get(entity_path).is_some() { - return true; - } - - if let Some(temporal_chunk_ids_per_timeline) = - self.temporal_chunk_ids_per_entity.get(entity_path) - { - for chunk_id_set in temporal_chunk_ids_per_timeline.values() { - if !chunk_id_set.per_start_time.is_empty() { - return true; - } - } - } - - false - } - pub fn num_events_for_static_component( &self, entity_path: &EntityPath, diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index ac53c440c04b..667861fcbf48 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -367,7 +367,10 @@ impl EntityDb { self.times_per_timeline.on_events(&store_events); self.query_caches.on_events(&store_events); self.tree.on_store_additions(&store_events); - self.tree.on_store_deletions(&store_events); + + // Tree deletions depend on data store, so data store must have been notified of deletions already. + self.tree + .on_store_deletions(&self.data_store, &store_events); // We inform the stats last, since it measures e2e latency. self.stats.on_events(&store_events); @@ -425,24 +428,11 @@ impl EntityDb { fn on_store_deletions(&mut self, store_events: &[ChunkStoreEvent]) { re_tracing::profile_function!(); - let Self { - data_source: _, - set_store_info: _, - last_modified_at: _, - latest_row_id: _, - entity_path_from_hash: _, - times_per_timeline, - tree, - data_store: _, - resolver: _, - query_caches, - stats: _, - } = self; - - times_per_timeline.on_events(store_events); - query_caches.on_events(store_events); + self.times_per_timeline.on_events(store_events); + self.query_caches.on_events(store_events); - tree.on_store_deletions(store_events); + // Tree deletions depend on data store, so data store must have been notified of deletions already. + self.tree.on_store_deletions(&self.data_store, store_events); } /// Key used for sorting recordings in the UI. diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index 5e3e68240b7e..c3c7bc098665 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -5,14 +5,10 @@ use itertools::Itertools; use nohash_hasher::IntMap; use re_chunk::RowId; -use re_chunk_store::{ChunkStoreDiff, ChunkStoreDiffKind, ChunkStoreEvent, ChunkStoreSubscriber}; -use re_log_types::{ComponentPath, EntityPath, EntityPathHash, EntityPathPart, TimeInt, Timeline}; +use re_chunk_store::{ChunkStore, ChunkStoreDiffKind, ChunkStoreEvent, ChunkStoreSubscriber}; +use re_log_types::{EntityPath, EntityPathHash, EntityPathPart, TimeInt, Timeline}; use re_types_core::ComponentName; -// Used all over in docstrings. -#[allow(unused_imports)] -use re_chunk_store::ChunkStore; - use crate::TimeHistogramPerTimeline; // ---------------------------------------------------------------------------- @@ -27,9 +23,6 @@ pub struct EntityTree { /// Direct descendants of this (sub)tree. pub children: BTreeMap, - /// Information about this specific entity (excluding children). - pub entity: EntityInfo, - /// Info about this subtree, including all children, recursively. pub subtree: SubtreeInfo, } @@ -206,7 +199,6 @@ impl EntityTree { Self { path, children: Default::default(), - entity: Default::default(), subtree: Default::default(), } } @@ -216,8 +208,10 @@ impl EntityTree { self.children.is_empty() } - pub fn num_children(&self) -> usize { - self.children.len() + /// Returns `false` if this entity has no children and no data. + pub fn is_empty(&self, chunk_store: &ChunkStore) -> bool { + self.children.is_empty() + && !chunk_store.entity_has_any_component_on_any_timeline(&self.path) } /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s. @@ -249,90 +243,38 @@ impl EntityTree { .or_insert_with(|| Self::new(entity_path.as_slice()[..=i].into())); tree.subtree.on_event(event); } - - // Finally book-keeping for the entity where data was actually added: - tree.on_added_data(&event.diff); - } - - /// Handles the addition of new data into the tree. - fn on_added_data(&mut self, store_diff: &ChunkStoreDiff) { - for component_name in store_diff.chunk.component_names() { - let component_path = - ComponentPath::new(store_diff.chunk.entity_path().clone(), component_name); - - let per_component = self - .entity - .components - .entry(component_path.component_name) - .or_default(); - per_component.add( - &store_diff - .chunk - .timelines() - .iter() - .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) - .collect_vec(), - 1, - ); - } } /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s. /// /// Only reacts to deletions (`event.kind == StoreDiffKind::Deletion`). - pub fn on_store_deletions(&mut self, store_events: &[ChunkStoreEvent]) { + pub fn on_store_deletions( + &mut self, + data_store: &ChunkStore, + store_events: &[ChunkStoreEvent], + ) { re_tracing::profile_function!(); - let Self { - path, - children, - entity, - subtree, - } = self; - // Only keep events relevant to this branch of the tree. let subtree_events = store_events .iter() .filter(|e| e.kind == ChunkStoreDiffKind::Deletion) - .filter(|&e| e.diff.chunk.entity_path().starts_with(path)) + .filter(|&e| e.diff.chunk.entity_path().starts_with(&self.path)) .cloned() .collect_vec(); - { - re_tracing::profile_scope!("entity"); - for event in subtree_events - .iter() - .filter(|e| e.chunk.entity_path() == path) - { - for component_name in event.chunk.component_names() { - if let Some(histo) = entity.components.get_mut(&component_name) { - histo.remove( - &event - .chunk - .timelines() - .iter() - .map(|(timeline, time_chunk)| (*timeline, time_chunk.times_raw())) - .collect_vec(), - 1, - ); - if histo.is_empty() { - entity.components.remove(&component_name); - } - } - } - } - } - { re_tracing::profile_scope!("subtree"); for event in &subtree_events { - subtree.on_event(event); + self.subtree.on_event(event); } } - children.retain(|_, child| { - child.on_store_deletions(&subtree_events); - child.num_children() > 0 + self.children.retain(|_, entity| { + // this is placed first, because we'll only know if the child entity is empty after telling it to clear itself. + entity.on_store_deletions(data_store, &subtree_events); + + !entity.is_empty(data_store) }); } diff --git a/crates/store/re_entity_db/tests/clear.rs b/crates/store/re_entity_db/tests/clear.rs index 3de66f176d17..dea8737effde 100644 --- a/crates/store/re_entity_db/tests/clear.rs +++ b/crates/store/re_entity_db/tests/clear.rs @@ -414,7 +414,7 @@ fn clear_and_gc() -> anyhow::Result<()> { // Insert a component, then clear it, then GC. { // EntityTree is Empty when we start - assert_eq!(db.tree().num_children(), 0); + assert!(db.tree().is_empty(db.store())); let point = MyPoint::new(1.0, 2.0); @@ -450,7 +450,7 @@ fn clear_and_gc() -> anyhow::Result<()> { assert_eq!(stats.temporal_chunks.total_num_rows, 0); // EntityTree should be empty again when we end since everything was GC'd - assert_eq!(db.tree().num_children(), 0); + assert!(db.tree().is_empty(db.store())); } Ok(()) From b5a140e25e620a53c8bfef57dce18e72b0631136 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:43:30 +0200 Subject: [PATCH 19/43] remove `SubtreeInfo` --- crates/store/re_entity_db/src/entity_db.rs | 14 ++- crates/store/re_entity_db/src/entity_tree.rs | 113 +----------------- .../src/time_histogram_per_timeline.rs | 23 +++- 3 files changed, 30 insertions(+), 120 deletions(-) diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index 667861fcbf48..8150bc600b3f 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -53,6 +53,9 @@ pub struct EntityDb { /// Used for time control. times_per_timeline: TimesPerTimeline, + /// A time histogram of all entities, for every timeline. + time_histogram_per_timeline: crate::TimeHistogramPerTimeline, + /// A tree-view (split on path components) of the entities. tree: crate::EntityTree, @@ -85,6 +88,7 @@ impl EntityDb { entity_path_from_hash: Default::default(), times_per_timeline: Default::default(), tree: crate::EntityTree::root(), + time_histogram_per_timeline: Default::default(), data_store, resolver: re_query::PromiseResolver::default(), query_caches, @@ -267,7 +271,7 @@ impl EntityDb { /// Histogram of all events on the timeeline, of all entities. pub fn time_histogram(&self, timeline: &Timeline) -> Option<&crate::TimeHistogram> { - self.tree().subtree.time_histogram.get(timeline) + self.time_histogram_per_timeline.get(timeline) } #[inline] @@ -365,12 +369,11 @@ impl EntityDb { { // Update our internal views by notifying them of resulting [`ChunkStoreEvent`]s. self.times_per_timeline.on_events(&store_events); + self.time_histogram_per_timeline.on_events(&store_events); self.query_caches.on_events(&store_events); self.tree.on_store_additions(&store_events); - // Tree deletions depend on data store, so data store must have been notified of deletions already. - self.tree - .on_store_deletions(&self.data_store, &store_events); + self.tree.on_store_deletions(&self.data_store); // We inform the stats last, since it measures e2e latency. self.stats.on_events(&store_events); @@ -430,9 +433,10 @@ impl EntityDb { self.times_per_timeline.on_events(store_events); self.query_caches.on_events(store_events); + self.time_histogram_per_timeline.on_events(store_events); // Tree deletions depend on data store, so data store must have been notified of deletions already. - self.tree.on_store_deletions(&self.data_store, store_events); + self.tree.on_store_deletions(&self.data_store); } /// Key used for sorting recordings in the UI. diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index c3c7bc098665..2d6826aaf42f 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -1,7 +1,6 @@ use std::collections::BTreeMap; use ahash::HashSet; -use itertools::Itertools; use nohash_hasher::IntMap; use re_chunk::RowId; @@ -9,8 +8,6 @@ use re_chunk_store::{ChunkStore, ChunkStoreDiffKind, ChunkStoreEvent, ChunkStore use re_log_types::{EntityPath, EntityPathHash, EntityPathPart, TimeInt, Timeline}; use re_types_core::ComponentName; -use crate::TimeHistogramPerTimeline; - // ---------------------------------------------------------------------------- /// A recursive, manually updated [`ChunkStoreSubscriber`] that maintains the entity hierarchy. @@ -22,9 +19,6 @@ pub struct EntityTree { /// Direct descendants of this (sub)tree. pub children: BTreeMap, - - /// Info about this subtree, including all children, recursively. - pub subtree: SubtreeInfo, } // NOTE: This is only to let people know that this is in fact a [`ChunkStoreSubscriber`], so they A) don't try @@ -50,86 +44,6 @@ impl ChunkStoreSubscriber for EntityTree { } } -/// Information about this specific entity (excluding children). -#[derive(Default)] -pub struct EntityInfo { - /// Flat time histograms for each component of this [`EntityTree`]. - /// - /// Keeps track of the _number of times a component is logged_ per time per timeline, only for - /// this specific [`EntityTree`]. - /// A component logged twice at the same timestamp is counted twice. - /// - /// ⚠ Auto-generated instance keys are _not_ accounted for. ⚠ - pub components: BTreeMap, -} - -/// Info about stuff at a given [`EntityPath`], including all of its children, recursively. -#[derive(Default)] -pub struct SubtreeInfo { - /// Recursive time histogram for this [`EntityTree`]. - /// - /// Keeps track of the _number of components logged_ per time per timeline, recursively across - /// all of the [`EntityTree`]'s children. - /// A component logged twice at the same timestamp is counted twice. - /// - /// ⚠ Auto-generated instance keys are _not_ accounted for. ⚠ - pub time_histogram: TimeHistogramPerTimeline, - - /// Number of bytes used by all arrow data - data_bytes: u64, -} - -impl SubtreeInfo { - /// Assumes the event has been filtered to be part of this subtree. - fn on_event(&mut self, event: &ChunkStoreEvent) { - use re_types_core::SizeBytes as _; - - match event.kind { - ChunkStoreDiffKind::Addition => { - let times = event - .chunk - .timelines() - .iter() - .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) - .collect_vec(); - self.time_histogram.add(×, event.num_components() as _); - - self.data_bytes += event.chunk.total_size_bytes(); - } - ChunkStoreDiffKind::Deletion => { - let times = event - .chunk - .timelines() - .iter() - .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) - .collect_vec(); - self.time_histogram - .remove(×, event.num_components() as _); - - let removed_bytes = event.chunk.total_size_bytes(); - self.data_bytes - .checked_sub(removed_bytes) - .unwrap_or_else(|| { - re_log::debug!( - store_id = %event.store_id, - entity_path = %event.chunk.entity_path(), - current = self.data_bytes, - removed = removed_bytes, - "book keeping underflowed" - ); - u64::MIN - }); - } - } - } - - /// Number of bytes used by all arrow data in this tree (including their schemas, but otherwise ignoring book-keeping overhead). - #[inline] - pub fn data_bytes(&self) -> u64 { - self.data_bytes - } -} - /// Maintains an optimized representation of a batch of [`ChunkStoreEvent`]s specifically designed to /// accelerate garbage collection of [`EntityTree`]s. /// @@ -199,7 +113,6 @@ impl EntityTree { Self { path, children: Default::default(), - subtree: Default::default(), } } @@ -234,45 +147,23 @@ impl EntityTree { // Book-keeping for each level in the hierarchy: let mut tree = self; - tree.subtree.on_event(event); - for (i, part) in entity_path.iter().enumerate() { tree = tree .children .entry(part.clone()) .or_insert_with(|| Self::new(entity_path.as_slice()[..=i].into())); - tree.subtree.on_event(event); } } /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s. /// /// Only reacts to deletions (`event.kind == StoreDiffKind::Deletion`). - pub fn on_store_deletions( - &mut self, - data_store: &ChunkStore, - store_events: &[ChunkStoreEvent], - ) { + pub fn on_store_deletions(&mut self, data_store: &ChunkStore) { re_tracing::profile_function!(); - // Only keep events relevant to this branch of the tree. - let subtree_events = store_events - .iter() - .filter(|e| e.kind == ChunkStoreDiffKind::Deletion) - .filter(|&e| e.diff.chunk.entity_path().starts_with(&self.path)) - .cloned() - .collect_vec(); - - { - re_tracing::profile_scope!("subtree"); - for event in &subtree_events { - self.subtree.on_event(event); - } - } - self.children.retain(|_, entity| { // this is placed first, because we'll only know if the child entity is empty after telling it to clear itself. - entity.on_store_deletions(data_store, &subtree_events); + entity.on_store_deletions(data_store); !entity.is_empty(data_store) }); diff --git a/crates/store/re_entity_db/src/time_histogram_per_timeline.rs b/crates/store/re_entity_db/src/time_histogram_per_timeline.rs index ebb06767a4c6..215a47e26c8b 100644 --- a/crates/store/re_entity_db/src/time_histogram_per_timeline.rs +++ b/crates/store/re_entity_db/src/time_histogram_per_timeline.rs @@ -1,5 +1,7 @@ use std::collections::BTreeMap; +use itertools::Itertools as _; +use re_chunk_store::ChunkStoreDiffKind; use re_chunk_store::{ChunkStoreEvent, ChunkStoreSubscriber}; use re_log_types::Timeline; @@ -132,9 +134,22 @@ impl ChunkStoreSubscriber for TimeHistogramPerTimeline { } #[allow(clippy::unimplemented)] - fn on_events(&mut self, _events: &[ChunkStoreEvent]) { - unimplemented!( - r"TimeHistogramPerTimeline view is maintained as a sub-view of `EntityTree`", - ); + fn on_events(&mut self, events: &[ChunkStoreEvent]) { + for event in events { + let times = event + .chunk + .timelines() + .iter() + .map(|(&timeline, time_chunk)| (timeline, time_chunk.times_raw())) + .collect_vec(); + match event.kind { + ChunkStoreDiffKind::Addition => { + self.add(×, event.num_components() as _); + } + ChunkStoreDiffKind::Deletion => { + self.remove(×, event.num_components() as _); + } + } + } } } From d4b65f1f5bcfe721c9422019b6e5fc3d7d8d456b Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 09:57:00 +0200 Subject: [PATCH 20/43] docs --- crates/store/re_chunk_store/src/query.rs | 38 +++++++----------------- crates/viewer/re_data_ui/src/item_ui.rs | 5 +++- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 557bb109eec0..adebf4f9dc5f 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -104,7 +104,7 @@ impl ChunkStore { } } - /// Check whether a given entity has a specific [`ComponentName`] either on the specified + /// Check whether an entity has a specific component either on the specified /// timeline, or in its static data. #[inline] pub fn entity_has_component( @@ -118,6 +118,7 @@ impl ChunkStore { .map_or(false, |components| components.contains(component_name)) } + /// Check whether an entity has a specific component on any timeline, or in its static data. pub fn entity_has_component_on_any_timeline( &self, entity_path: &EntityPath, @@ -152,7 +153,7 @@ impl ChunkStore { false } - /// Returns true if the given entity has a specific component with static data. + /// Check whether an entity has a specific component in its static data. #[inline] pub fn entity_has_static_component( &self, @@ -168,6 +169,7 @@ impl ChunkStore { static_components.contains_key(component_name) } + /// Check whether an entity has any data on a specific timeline, or any static data. pub fn entity_has_data_on_timeline( &self, timeline: &Timeline, @@ -193,28 +195,7 @@ impl ChunkStore { false } - pub fn entity_has_data_on_any_timeline(&self, entity_path: &EntityPath) -> bool { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - if self.static_chunk_ids_per_entity.get(entity_path).is_some() { - return true; - } - - if let Some(temporal_chunk_ids_per_timeline) = - self.temporal_chunk_ids_per_entity.get(entity_path) - { - for chunk_id_set in temporal_chunk_ids_per_timeline.values() { - if !chunk_id_set.per_start_time.is_empty() { - return true; - } - } - } - - false - } - + /// Check whether an entity has any data on any timeline, or any static data. pub fn entity_has_any_component_on_any_timeline(&self, entity_path: &EntityPath) -> bool { re_tracing::profile_function!(); @@ -601,7 +582,9 @@ impl ChunkStore { ) } - pub fn time_range_for_entity( + /// Returns the min and max times at which data was logged for an entity + /// on a specific timeline. + pub fn time_range_for_entity_on_timeline( &self, timeline: &Timeline, entity_path: &EntityPath, @@ -620,6 +603,7 @@ impl ChunkStore { Some(ResolvedTimeRange::new(*start, *end)) } + /// Returns the number of events logged for an entity on a specific timeline. pub fn num_events_on_timeline_for_all_components( &self, timeline: &Timeline, @@ -660,6 +644,7 @@ impl ChunkStore { total_events } + /// Returns the number of times a static component was logged to an entity. pub fn num_events_for_static_component( &self, entity_path: &EntityPath, @@ -677,7 +662,6 @@ impl ChunkStore { }) .and_then(|chunk_id| self.chunks_per_chunk_id.get(chunk_id)) { - // Static data overrides temporal data static_chunk .num_events_for_component(component_name) .unwrap_or(0) @@ -686,7 +670,7 @@ impl ChunkStore { } } - /// Returns the number of events logged for a given component on the given entity path across all timelines. + /// Returns the number of times a temporal component was logged to an entity path on a specific timeline. /// /// This ignores static data. pub fn num_events_on_timeline_for_temporal_component( diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index 870308aff1f6..251c40369931 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -320,7 +320,10 @@ fn entity_tree_stats_ui( .store() .num_events_on_timeline_for_all_components(timeline, &tree.path); - if let Some(time_range) = db.store().time_range_for_entity(timeline, &tree.path) { + if let Some(time_range) = db + .store() + .time_range_for_entity_on_timeline(timeline, &tree.path) + { let min_time = time_range.min(); let max_time = time_range.max(); if min_time < max_time && 1 < num_events { From 10dc774cc35c1482488e1f6d488b34dd901cf4b9 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 10:27:16 +0200 Subject: [PATCH 21/43] more docs --- crates/store/re_entity_db/src/entity_tree.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index 2d6826aaf42f..2691b40efac3 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -127,9 +127,10 @@ impl EntityTree { && !chunk_store.entity_has_any_component_on_any_timeline(&self.path) } - /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s. + /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s, + /// adding any new entities to the tree. /// - /// Only reacts to deletions (`event.kind == StoreDiffKind::Deletion`). + /// Only reacts to additions (`event.kind == StoreDiffKind::Addition`). pub fn on_store_additions(&mut self, events: &[ChunkStoreEvent]) { re_tracing::profile_function!(); for event in events @@ -155,9 +156,10 @@ impl EntityTree { } } - /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s. + /// Updates the [`EntityTree`] by removing any entities which have no data and no children. /// - /// Only reacts to deletions (`event.kind == StoreDiffKind::Deletion`). + /// ⚠ This depends on `data_store` having up-to-date records of which entities have no data, + /// so it must have already been notified of any chunk events prior to calling this method. pub fn on_store_deletions(&mut self, data_store: &ChunkStore) { re_tracing::profile_function!(); @@ -176,7 +178,10 @@ impl EntityTree { ) -> Option<&'tree EntityTree> { match path { [] => Some(this), - [first, rest @ ..] => subtree_recursive(this.children.get(first)?, rest), + [first, rest @ ..] => { + let child = this.children.get(first)?; + subtree_recursive(child, rest) + } } } From ee7509a21b8c6bb49e3c3cef1ec82775c540da79 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 10:28:31 +0200 Subject: [PATCH 22/43] docs docs --- crates/store/re_entity_db/src/entity_tree.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index 2691b40efac3..f868ab00db61 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -200,6 +200,8 @@ impl EntityTree { visit(self, &mut visitor); } + /// Invokes the `predicate` for `self` and all children recursively, + /// returning the subtree for which the `predicate` returns `true`. pub fn find_child_recursive( &self, mut predicate: impl FnMut(&EntityPath) -> bool, From 473dc594a93266fe6c26c914e889fe8ac449b5b3 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 11:29:05 +0200 Subject: [PATCH 23/43] simplify + docs --- crates/store/re_entity_db/src/entity_db.rs | 5 +++- crates/store/re_entity_db/src/entity_tree.rs | 29 +++++++++++--------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index 8150bc600b3f..4e75e64cddb1 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -570,6 +570,7 @@ impl EntityDb { size } + /// Returns true if an entity or any of its children have any data on the given timeline. pub fn subtree_has_data_on_timeline( &self, timeline: &Timeline, @@ -582,7 +583,9 @@ impl EntityDb { }; subtree - .find_child_recursive(|path| self.store().entity_has_data_on_timeline(timeline, path)) + .find_first_child_recursive(|path| { + self.store().entity_has_data_on_timeline(timeline, path) + }) .is_some() } } diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index f868ab00db61..20d97cb70c95 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -201,30 +201,33 @@ impl EntityTree { } /// Invokes the `predicate` for `self` and all children recursively, - /// returning the subtree for which the `predicate` returns `true`. - pub fn find_child_recursive( + /// returning the _first_ entity for which the `predicate` returns `true`. + /// + /// Note that this function has early return semantics, meaning if multiple + /// entities would return `true`, only the first is returned. + /// The entities are yielded in order of their entity paths. + pub fn find_first_child_recursive( &self, mut predicate: impl FnMut(&EntityPath) -> bool, ) -> Option<&Self> { - use std::ops::ControlFlow; - fn visit<'a>( this: &'a EntityTree, predicate: &mut impl FnMut(&EntityPath) -> bool, - ) -> ControlFlow<&'a EntityTree> { + ) -> Option<&'a EntityTree> { if predicate(&this.path) { - return ControlFlow::Break(this); + return Some(this); }; + for child in this.children.values() { - visit(child, predicate)?; + if let Some(subtree) = visit(child, predicate) { + // Early return + return Some(subtree); + }; } - ControlFlow::Continue(()) - } - let result = visit(self, &mut predicate); - match result { - ControlFlow::Continue(()) => None, - ControlFlow::Break(v) => Some(v), + None } + + visit(self, &mut predicate) } } From b153073ef267bc5206a20551c8dd51060cffcc33 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 13:43:20 +0200 Subject: [PATCH 24/43] btreemap is sorted by keys --- crates/store/re_chunk_store/src/query.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index adebf4f9dc5f..c474089edb67 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -597,8 +597,8 @@ impl ChunkStore { self.temporal_chunk_ids_per_entity.get(entity_path)?; let chunk_id_sets = temporal_chunk_ids_per_timeline.get(timeline)?; - let start = chunk_id_sets.per_start_time.keys().min()?; - let end = chunk_id_sets.per_end_time.keys().max()?; + let start = chunk_id_sets.per_start_time.first_key_value()?.0; + let end = chunk_id_sets.per_end_time.last_key_value()?.0; Some(ResolvedTimeRange::new(*start, *end)) } From b96a5e0b2e5304dc6f5854a28aae9c6e02e952c5 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:08:06 +0200 Subject: [PATCH 25/43] rename `all_components*` queries --- crates/store/re_chunk_store/src/query.rs | 9 +++------ crates/store/re_chunk_store/tests/reads.rs | 2 +- crates/viewer/re_data_ui/src/instance_path.rs | 2 +- crates/viewer/re_data_ui/src/item_ui.rs | 2 +- crates/viewer/re_selection_panel/src/defaults_ui.rs | 2 +- .../re_space_view_dataframe/src/latest_at_table.rs | 2 +- .../re_space_view_dataframe/src/time_range_table.rs | 2 +- crates/viewer/re_space_view_dataframe/src/utils.rs | 2 +- crates/viewer/re_time_panel/src/lib.rs | 2 +- crates/viewer/re_viewer_context/src/item.rs | 2 +- crates/viewer/re_viewport_blueprint/src/space_view.rs | 2 +- .../re_viewport_blueprint/src/space_view_contents.rs | 4 ++-- examples/rust/extend_viewer_ui/src/main.rs | 5 ++++- 13 files changed, 19 insertions(+), 19 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index c474089edb67..b41b21ae2d94 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -25,7 +25,7 @@ impl ChunkStore { /// Static components are always included in the results. /// /// Returns `None` if the entity doesn't exist at all on this `timeline`. - pub fn all_components( + pub fn all_components_on_timeline( &self, timeline: &Timeline, entity_path: &EntityPath, @@ -68,10 +68,7 @@ impl ChunkStore { /// Static components are always included in the results. /// /// Returns `None` if the entity has never had any data logged to it. - pub fn all_components_on_all_timelines( - &self, - entity_path: &EntityPath, - ) -> Option { + pub fn all_components(&self, entity_path: &EntityPath) -> Option { re_tracing::profile_function!(); self.query_id.fetch_add(1, Ordering::Relaxed); @@ -114,7 +111,7 @@ impl ChunkStore { component_name: &ComponentName, ) -> bool { re_tracing::profile_function!(); - self.all_components(timeline, entity_path) + self.all_components_on_timeline(timeline, entity_path) .map_or(false, |components| components.contains(component_name)) } diff --git a/crates/store/re_chunk_store/tests/reads.rs b/crates/store/re_chunk_store/tests/reads.rs index f7a579dc064b..fe5ff17d5c62 100644 --- a/crates/store/re_chunk_store/tests/reads.rs +++ b/crates/store/re_chunk_store/tests/reads.rs @@ -56,7 +56,7 @@ fn all_components() -> anyhow::Result<()> { |store: &ChunkStore, entity_path: &EntityPath, expected: Option<&[ComponentName]>| { let timeline = Timeline::new("frame_nr", TimeType::Sequence); - let component_names = store.all_components(&timeline, entity_path); + let component_names = store.all_components_on_timeline(&timeline, entity_path); let expected_component_names = expected.map(|expected| { let expected: ComponentNameSet = expected.iter().copied().collect(); diff --git a/crates/viewer/re_data_ui/src/instance_path.rs b/crates/viewer/re_data_ui/src/instance_path.rs index 3f6975c0f9bc..0b31446e2b5a 100644 --- a/crates/viewer/re_data_ui/src/instance_path.rs +++ b/crates/viewer/re_data_ui/src/instance_path.rs @@ -21,7 +21,7 @@ impl DataUi for InstancePath { let Some(components) = ctx .recording_store() - .all_components(&query.timeline(), entity_path) + .all_components_on_timeline(&query.timeline(), entity_path) else { if ctx.recording().is_known_entity(entity_path) { // This is fine - e.g. we're looking at `/world` and the user has only logged to `/world/car`. diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index 251c40369931..de9ea3bee181 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -144,7 +144,7 @@ pub fn instance_path_icon( // It is an entity path if db .store() - .all_components(timeline, &instance_path.entity_path) + .all_components_on_timeline(timeline, &instance_path.entity_path) .is_some() { &icons::ENTITY diff --git a/crates/viewer/re_selection_panel/src/defaults_ui.rs b/crates/viewer/re_selection_panel/src/defaults_ui.rs index f0c2313acea6..15163c250c42 100644 --- a/crates/viewer/re_selection_panel/src/defaults_ui.rs +++ b/crates/viewer/re_selection_panel/src/defaults_ui.rs @@ -187,7 +187,7 @@ fn active_defaults( // even if they are listed in `all_components`. ctx.blueprint_db() .store() - .all_components(&blueprint_timeline(), &view.defaults_path) + .all_components_on_timeline(&blueprint_timeline(), &view.defaults_path) .unwrap_or_default() .into_iter() .filter(|c| { diff --git a/crates/viewer/re_space_view_dataframe/src/latest_at_table.rs b/crates/viewer/re_space_view_dataframe/src/latest_at_table.rs index 2a3b02d65969..658ef78c470a 100644 --- a/crates/viewer/re_space_view_dataframe/src/latest_at_table.rs +++ b/crates/viewer/re_space_view_dataframe/src/latest_at_table.rs @@ -63,7 +63,7 @@ pub(crate) fn latest_at_table_ui( .iter() .flat_map(|entity_path| { ctx.recording_store() - .all_components(&query.timeline, entity_path) + .all_components_on_timeline(&query.timeline, entity_path) .unwrap_or_default() }) // TODO(#4466): make showing/hiding indicators components an explicit optional diff --git a/crates/viewer/re_space_view_dataframe/src/time_range_table.rs b/crates/viewer/re_space_view_dataframe/src/time_range_table.rs index f2be66d28cea..1d18cb17d22d 100644 --- a/crates/viewer/re_space_view_dataframe/src/time_range_table.rs +++ b/crates/viewer/re_space_view_dataframe/src/time_range_table.rs @@ -51,7 +51,7 @@ pub(crate) fn time_range_table_ui( .filter(|data_result| data_result.is_visible(ctx)) .flat_map(|data_result| { ctx.recording_store() - .all_components(&query.timeline, &data_result.entity_path) + .all_components_on_timeline(&query.timeline, &data_result.entity_path) .unwrap_or_default() }) // TODO(#4466): make showing/hiding indicators components an explicit optional diff --git a/crates/viewer/re_space_view_dataframe/src/utils.rs b/crates/viewer/re_space_view_dataframe/src/utils.rs index afcb2358cee2..49eeaa1f8a9d 100644 --- a/crates/viewer/re_space_view_dataframe/src/utils.rs +++ b/crates/viewer/re_space_view_dataframe/src/utils.rs @@ -28,7 +28,7 @@ pub(crate) fn sorted_instance_paths_for<'a>( re_tracing::profile_function!(); store - .all_components(timeline, entity_path) + .all_components_on_timeline(timeline, entity_path) .unwrap_or_default() .into_iter() .filter(|component_name| !component_name.is_indicator_component()) diff --git a/crates/viewer/re_time_panel/src/lib.rs b/crates/viewer/re_time_panel/src/lib.rs index d7d3767dbcab..06f4bc58608f 100644 --- a/crates/viewer/re_time_panel/src/lib.rs +++ b/crates/viewer/re_time_panel/src/lib.rs @@ -746,7 +746,7 @@ impl TimePanel { // If this is an entity: if let Some(components) = entity_db .store() - .all_components_on_all_timelines(&tree.path) + .all_components(&tree.path) { for component_name in sorted_component_list_for_ui(components.iter()) { let is_static = entity_db diff --git a/crates/viewer/re_viewer_context/src/item.rs b/crates/viewer/re_viewer_context/src/item.rs index 853abf0115a1..735e45fde509 100644 --- a/crates/viewer/re_viewer_context/src/item.rs +++ b/crates/viewer/re_viewer_context/src/item.rs @@ -183,7 +183,7 @@ pub fn resolve_mono_instance_path( // NOTE: While we normally frown upon direct queries to the datastore, `all_components` is fine. let Some(component_names) = entity_db .store() - .all_components(&query.timeline(), &instance.entity_path) + .all_components_on_timeline(&query.timeline(), &instance.entity_path) else { // No components at all, return unindexed entity. return re_entity_db::InstancePath::entity_all(instance.entity_path.clone()); diff --git a/crates/viewer/re_viewport_blueprint/src/space_view.rs b/crates/viewer/re_viewport_blueprint/src/space_view.rs index 46856545e282..3014233e3871 100644 --- a/crates/viewer/re_viewport_blueprint/src/space_view.rs +++ b/crates/viewer/re_viewport_blueprint/src/space_view.rs @@ -264,7 +264,7 @@ impl SpaceViewBlueprint { store_context.blueprint_timepoint_for_writes(), blueprint .store() - .all_components(&query.timeline(), path) + .all_components_on_timeline(&query.timeline(), path) .into_iter() .flat_map(|v| v.into_iter()) // It's important that we don't include the SpaceViewBlueprint's components diff --git a/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs b/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs index 283d78186621..8f319916493c 100644 --- a/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs +++ b/crates/viewer/re_viewport_blueprint/src/space_view_contents.rs @@ -430,7 +430,7 @@ impl DataQueryPropertyResolver<'_> { { for component_name in blueprint .store() - .all_components_on_all_timelines(&recursive_override_subtree.path) + .all_components(&recursive_override_subtree.path) .unwrap_or_default() { let results = blueprint.query_caches().latest_at( @@ -465,7 +465,7 @@ impl DataQueryPropertyResolver<'_> { { for component_name in blueprint .store() - .all_components_on_all_timelines(&individual_override_subtree.path) + .all_components(&individual_override_subtree.path) .unwrap_or_default() { let results = blueprint.query_caches().latest_at( diff --git a/examples/rust/extend_viewer_ui/src/main.rs b/examples/rust/extend_viewer_ui/src/main.rs index a7eb54f07476..924e4a92223b 100644 --- a/examples/rust/extend_viewer_ui/src/main.rs +++ b/examples/rust/extend_viewer_ui/src/main.rs @@ -130,7 +130,10 @@ fn entity_ui( entity_path: &re_log_types::EntityPath, ) { // Each entity can have many components (e.g. position, color, radius, …): - if let Some(components) = entity_db.store().all_components(&timeline, entity_path) { + if let Some(components) = entity_db + .store() + .all_components_on_timeline(&timeline, entity_path) + { for component in components { ui.collapsing(component.to_string(), |ui| { component_ui(ui, entity_db, timeline, entity_path, component); From 2aebfd021fab85a315b9b6bb11ff2d2fe56f1009 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:11:36 +0200 Subject: [PATCH 26/43] more renamings --- crates/store/re_chunk_store/src/query.rs | 6 +++--- crates/store/re_entity_db/src/entity_tree.rs | 3 +-- crates/viewer/re_data_ui/src/component_path.rs | 9 +++++---- crates/viewer/re_data_ui/src/image_meaning.rs | 8 ++++++-- crates/viewer/re_space_view_spatial/src/view_2d.rs | 2 +- crates/viewer/re_space_view_spatial/src/view_3d.rs | 2 +- crates/viewer/re_time_panel/src/lib.rs | 2 +- 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index b41b21ae2d94..2bca3e37d363 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -104,7 +104,7 @@ impl ChunkStore { /// Check whether an entity has a specific component either on the specified /// timeline, or in its static data. #[inline] - pub fn entity_has_component( + pub fn entity_has_component_on_timeline( &self, timeline: &Timeline, entity_path: &EntityPath, @@ -116,7 +116,7 @@ impl ChunkStore { } /// Check whether an entity has a specific component on any timeline, or in its static data. - pub fn entity_has_component_on_any_timeline( + pub fn entity_has_component( &self, entity_path: &EntityPath, component_name: &ComponentName, @@ -193,7 +193,7 @@ impl ChunkStore { } /// Check whether an entity has any data on any timeline, or any static data. - pub fn entity_has_any_component_on_any_timeline(&self, entity_path: &EntityPath) -> bool { + pub fn entity_has_data(&self, entity_path: &EntityPath) -> bool { re_tracing::profile_function!(); self.query_id.fetch_add(1, Ordering::Relaxed); diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index 20d97cb70c95..7d3b60bab766 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -123,8 +123,7 @@ impl EntityTree { /// Returns `false` if this entity has no children and no data. pub fn is_empty(&self, chunk_store: &ChunkStore) -> bool { - self.children.is_empty() - && !chunk_store.entity_has_any_component_on_any_timeline(&self.path) + self.children.is_empty() && !chunk_store.entity_has_data(&self.path) } /// Updates the [`EntityTree`] by applying a batch of [`ChunkStoreEvent`]s, diff --git a/crates/viewer/re_data_ui/src/component_path.rs b/crates/viewer/re_data_ui/src/component_path.rs index 4597468955a1..19f73a191826 100644 --- a/crates/viewer/re_data_ui/src/component_path.rs +++ b/crates/viewer/re_data_ui/src/component_path.rs @@ -33,10 +33,11 @@ impl DataUi for ComponentPath { } .data_ui(ctx, ui, ui_layout, query, db); } else if ctx.recording().tree().subtree(entity_path).is_some() { - if db - .store() - .entity_has_component(&query.timeline(), entity_path, component_name) - { + if db.store().entity_has_component_on_timeline( + &query.timeline(), + entity_path, + component_name, + ) { ui.label(""); } else { ui.label(format!( diff --git a/crates/viewer/re_data_ui/src/image_meaning.rs b/crates/viewer/re_data_ui/src/image_meaning.rs index 4b836b162468..2efcb895e1dd 100644 --- a/crates/viewer/re_data_ui/src/image_meaning.rs +++ b/crates/viewer/re_data_ui/src/image_meaning.rs @@ -11,9 +11,13 @@ pub fn image_meaning_for_entity( store: &re_chunk_store::ChunkStore, ) -> TensorDataMeaning { let timeline = &query.timeline(); - if store.entity_has_component(timeline, entity_path, &DepthImage::indicator().name()) { + if store.entity_has_component_on_timeline( + timeline, + entity_path, + &DepthImage::indicator().name(), + ) { TensorDataMeaning::Depth - } else if store.entity_has_component( + } else if store.entity_has_component_on_timeline( timeline, entity_path, &SegmentationImage::indicator().name(), diff --git a/crates/viewer/re_space_view_spatial/src/view_2d.rs b/crates/viewer/re_space_view_spatial/src/view_2d.rs index 97a66e75f253..4ae9bd70f5ee 100644 --- a/crates/viewer/re_space_view_spatial/src/view_2d.rs +++ b/crates/viewer/re_space_view_spatial/src/view_2d.rs @@ -282,7 +282,7 @@ fn count_non_nested_images_with_component( // bool true -> 1 entity_db .store() - .entity_has_component_on_any_timeline(&subtree.path, component_name) as usize + .entity_has_component(&subtree.path, component_name) as usize } else if !entity_bucket .iter() .any(|e| e.is_descendant_of(&subtree.path)) diff --git a/crates/viewer/re_space_view_spatial/src/view_3d.rs b/crates/viewer/re_space_view_spatial/src/view_3d.rs index 0b3a0a580441..93d59f5b4875 100644 --- a/crates/viewer/re_space_view_spatial/src/view_3d.rs +++ b/crates/viewer/re_space_view_spatial/src/view_3d.rs @@ -281,7 +281,7 @@ impl SpaceViewClass for SpatialSpaceView3D { if ctx .recording() .store() - .entity_has_component_on_any_timeline(path, &ViewCoordinates::name()) + .entity_has_component(path, &ViewCoordinates::name()) { indicated_entities.insert(path.clone()); } diff --git a/crates/viewer/re_time_panel/src/lib.rs b/crates/viewer/re_time_panel/src/lib.rs index 06f4bc58608f..900f28189639 100644 --- a/crates/viewer/re_time_panel/src/lib.rs +++ b/crates/viewer/re_time_panel/src/lib.rs @@ -760,7 +760,7 @@ impl TimePanel { let component_has_data_in_current_timeline = entity_db .store() - .entity_has_component(time_ctrl.timeline(), &tree.path, &component_name); + .entity_has_component_on_timeline(time_ctrl.timeline(), &tree.path, &component_name); let total_num_messages = entity_db.store().num_events_on_timeline_for_component( time_ctrl.timeline(), From e7022153a3721f353aa0d79fb5d05f2679a43e7c Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:31:35 +0200 Subject: [PATCH 27/43] invert --- crates/store/re_entity_db/src/entity_tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index 7d3b60bab766..356879803492 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -121,7 +121,7 @@ impl EntityTree { self.children.is_empty() } - /// Returns `false` if this entity has no children and no data. + /// Returns `true` if this entity has no children and no data. pub fn is_empty(&self, chunk_store: &ChunkStore) -> bool { self.children.is_empty() && !chunk_store.entity_has_data(&self.path) } From 7e6b5cc55f30ba2f3860ef164cd4499bdbcd5d30 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:07:10 +0200 Subject: [PATCH 28/43] refactor component/data query APIs --- crates/store/re_chunk_store/src/query.rs | 256 +++++++++++++++-------- crates/viewer/re_data_ui/src/item_ui.rs | 5 +- crates/viewer/re_time_panel/src/lib.rs | 14 +- 3 files changed, 177 insertions(+), 98 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 2bca3e37d363..60c25e71c6e1 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -101,8 +101,9 @@ impl ChunkStore { } } - /// Check whether an entity has a specific component either on the specified - /// timeline, or in its static data. + /// Check whether an entity has a static component or a temporal component on the specified timeline. + /// + /// This does _not_ check if the entity actually currently holds any data for that component. #[inline] pub fn entity_has_component_on_timeline( &self, @@ -111,11 +112,14 @@ impl ChunkStore { component_name: &ComponentName, ) -> bool { re_tracing::profile_function!(); - self.all_components_on_timeline(timeline, entity_path) - .map_or(false, |components| components.contains(component_name)) + + self.entity_has_static_component(entity_path, component_name) + || self.entity_has_temporal_component_on_timeline(timeline, entity_path, component_name) } - /// Check whether an entity has a specific component on any timeline, or in its static data. + /// Check whether an entity has a static component or a temporal component on any timeline. + /// + /// This does _not_ check if the entity actually currently holds any data for that component. pub fn entity_has_component( &self, entity_path: &EntityPath, @@ -123,50 +127,82 @@ impl ChunkStore { ) -> bool { re_tracing::profile_function!(); - if self - .static_chunk_ids_per_entity + self.entity_has_static_component(entity_path, component_name) + || self.entity_has_temporal_component(entity_path, component_name) + } + + /// Check whether an entity has a specific static component. + /// + /// This does _not_ check if the entity actually currently holds any data for that component. + #[inline] + pub fn entity_has_static_component( + &self, + entity_path: &EntityPath, + component_name: &ComponentName, + ) -> bool { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + self.static_chunk_ids_per_entity .get(entity_path) - .and_then(|static_chunks_per_component| static_chunks_per_component.get(component_name)) - .and_then(|id| self.chunks_per_chunk_id.get(id)) - .is_some() - { - return true; - } + .is_some_and(|static_chunk_ids_per_component| { + static_chunk_ids_per_component.contains_key(component_name) + }) + } - for temporal_chunk_ids_per_component in self - .temporal_chunk_ids_per_entity_per_component + /// Check whether an entity has a temporal component on any timeline. + /// + /// This does _not_ check if the entity actually currently holds any data for that component. + #[inline] + pub fn entity_has_temporal_component( + &self, + entity_path: &EntityPath, + component_name: &ComponentName, + ) -> bool { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + self.temporal_chunk_ids_per_entity_per_component .get(entity_path) .iter() .flat_map(|temporal_chunk_ids_per_timeline| temporal_chunk_ids_per_timeline.values()) - { - if temporal_chunk_ids_per_component - .get(component_name) - .is_some() - { - return true; - } - } - - false + .any(|temporal_chunk_ids_per_component| { + temporal_chunk_ids_per_component.contains_key(component_name) + }) } - /// Check whether an entity has a specific component in its static data. + /// Check whether an entity has a temporal component on a specific timeline. + /// + /// This does _not_ check if the entity actually currently holds any data for that component. #[inline] - pub fn entity_has_static_component( + pub fn entity_has_temporal_component_on_timeline( &self, + timeline: &Timeline, entity_path: &EntityPath, component_name: &ComponentName, ) -> bool { re_tracing::profile_function!(); - let Some(static_components) = self.static_chunk_ids_per_entity.get(entity_path) else { - return false; - }; + self.query_id.fetch_add(1, Ordering::Relaxed); - static_components.contains_key(component_name) + self.temporal_chunk_ids_per_entity_per_component + .get(entity_path) + .iter() + .filter_map(|temporal_chunk_ids_per_timeline| { + temporal_chunk_ids_per_timeline.get(timeline) + }) + .any(|temporal_chunk_ids_per_component| { + temporal_chunk_ids_per_component.contains_key(component_name) + }) } /// Check whether an entity has any data on a specific timeline, or any static data. + /// + /// This is different from checking if the entity has any component, it also ensures + /// that some _data_ currently exists in the store for this entity. + #[inline] pub fn entity_has_data_on_timeline( &self, timeline: &Timeline, @@ -174,49 +210,96 @@ impl ChunkStore { ) -> bool { re_tracing::profile_function!(); - self.query_id.fetch_add(1, Ordering::Relaxed); - - if self.static_chunk_ids_per_entity.get(entity_path).is_some() { - // Static data exists on all timelines - return true; - } + self.entity_has_static_data(entity_path) + || self.entity_has_temporal_data_on_timeline(timeline, entity_path) + } - if let Some(temporal_chunk_ids_per_timeline) = - self.temporal_chunk_ids_per_entity.get(entity_path) - { - if temporal_chunk_ids_per_timeline.contains_key(timeline) { - return true; - } - } + /// Check whether an entity has any static data or any temporal data on any timeline. + /// + /// This is different from checking if the entity has any component, it also ensures + /// that some _data_ currently exists in the store for this entity. + #[inline] + pub fn entity_has_data(&self, entity_path: &EntityPath) -> bool { + re_tracing::profile_function!(); - false + self.entity_has_static_data(entity_path) || self.entity_has_temporal_data(entity_path) } - /// Check whether an entity has any data on any timeline, or any static data. - pub fn entity_has_data(&self, entity_path: &EntityPath) -> bool { + /// Check whether an entity has any static data. + /// + /// This is different from checking if the entity has any component, it also ensures + /// that some _data_ currently exists in the store for this entity. + #[inline] + pub fn entity_has_static_data(&self, entity_path: &EntityPath) -> bool { re_tracing::profile_function!(); self.query_id.fetch_add(1, Ordering::Relaxed); - if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) - { - if !static_chunks_per_component.is_empty() { - return true; - } - } + self.static_chunk_ids_per_entity + .get(entity_path) + .is_some_and(|static_chunk_ids_per_component| { + static_chunk_ids_per_component + .values() + .any(|chunk_id| self.chunks_per_chunk_id.contains_key(chunk_id)) + }) + } - if let Some(temporal_chunks_per_timeline) = self - .temporal_chunk_ids_per_entity_per_component + /// Check whether an entity has any temporal data. + /// + /// This is different from checking if the entity has any component, it also ensures + /// that some _data_ currently exists in the store for this entity. + #[inline] + pub fn entity_has_temporal_data(&self, entity_path: &EntityPath) -> bool { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + self.temporal_chunk_ids_per_entity_per_component .get(entity_path) - { - for temporal_chunks_per_component in temporal_chunks_per_timeline.values() { - if !temporal_chunks_per_component.is_empty() { - return true; - } - } - } + .is_some_and(|temporal_chunks_per_timeline| { + temporal_chunks_per_timeline + .values() + .any(|temporal_chunks_per_component| { + temporal_chunks_per_component.values().any(|chunk_id_sets| { + chunk_id_sets.per_start_time.values().any(|chunk_id_set| { + chunk_id_set + .iter() + .any(|chunk_id| self.chunks_per_chunk_id.contains_key(chunk_id)) + }) + }) + }) + }) + } + + /// Check whether an entity has any temporal data. + /// + /// This is different from checking if the entity has any component, it also ensures + /// that some _data_ currently exists in the store for this entity. + #[inline] + pub fn entity_has_temporal_data_on_timeline( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> bool { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); - false + self.temporal_chunk_ids_per_entity_per_component + .get(entity_path) + .is_some_and(|temporal_chunks_per_timeline| { + temporal_chunks_per_timeline.get(timeline).is_some_and( + |temporal_chunks_per_component| { + temporal_chunks_per_component.values().any(|chunk_id_sets| { + chunk_id_sets.per_start_time.values().any(|chunk_id_set| { + chunk_id_set + .iter() + .any(|chunk_id| self.chunks_per_chunk_id.contains_key(chunk_id)) + }) + }) + }, + ) + }) } /// Find the earliest time at which something was logged for a given entity on the specified @@ -248,6 +331,26 @@ impl ChunkStore { (time_min != TimeInt::MAX).then_some(time_min) } + + /// Returns the min and max times at which data was logged for an entity on a specific timeline. + pub fn entity_time_range( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> Option { + re_tracing::profile_function!(); + + self.query_id.fetch_add(1, Ordering::Relaxed); + + let temporal_chunk_ids_per_timeline = + self.temporal_chunk_ids_per_entity.get(entity_path)?; + let chunk_id_sets = temporal_chunk_ids_per_timeline.get(timeline)?; + + let start = chunk_id_sets.per_start_time.first_key_value()?.0; + let end = chunk_id_sets.per_end_time.last_key_value()?.0; + + Some(ResolvedTimeRange::new(*start, *end)) + } } // LatestAt @@ -560,7 +663,9 @@ impl ChunkStore { // Queries returning `usize` and `bool` impl ChunkStore { - /// Returns the number of events logged for a given component on the given entity path across all timelines. + /// Returns the number of events of a specific component logged for an entity on a specific timeline. + /// + /// This counts both static and temporal components. pub fn num_events_on_timeline_for_component( &self, timeline: &Timeline, @@ -569,8 +674,6 @@ impl ChunkStore { ) -> usize { re_tracing::profile_function!(); - self.query_id.fetch_add(1, Ordering::Relaxed); - self.num_events_for_static_component(entity_path, component_name) + self.num_events_on_timeline_for_temporal_component( timeline, @@ -579,27 +682,6 @@ impl ChunkStore { ) } - /// Returns the min and max times at which data was logged for an entity - /// on a specific timeline. - pub fn time_range_for_entity_on_timeline( - &self, - timeline: &Timeline, - entity_path: &EntityPath, - ) -> Option { - re_tracing::profile_function!(); - - self.query_id.fetch_add(1, Ordering::Relaxed); - - let temporal_chunk_ids_per_timeline = - self.temporal_chunk_ids_per_entity.get(entity_path)?; - let chunk_id_sets = temporal_chunk_ids_per_timeline.get(timeline)?; - - let start = chunk_id_sets.per_start_time.first_key_value()?.0; - let end = chunk_id_sets.per_end_time.last_key_value()?.0; - - Some(ResolvedTimeRange::new(*start, *end)) - } - /// Returns the number of events logged for an entity on a specific timeline. pub fn num_events_on_timeline_for_all_components( &self, diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index de9ea3bee181..25c46a27ebdb 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -320,10 +320,7 @@ fn entity_tree_stats_ui( .store() .num_events_on_timeline_for_all_components(timeline, &tree.path); - if let Some(time_range) = db - .store() - .time_range_for_entity_on_timeline(timeline, &tree.path) - { + if let Some(time_range) = db.store().entity_time_range(timeline, &tree.path) { let min_time = time_range.min(); let max_time = time_range.max(); if min_time < max_time && 1 < num_events { diff --git a/crates/viewer/re_time_panel/src/lib.rs b/crates/viewer/re_time_panel/src/lib.rs index 900f28189639..a3933c921bbe 100644 --- a/crates/viewer/re_time_panel/src/lib.rs +++ b/crates/viewer/re_time_panel/src/lib.rs @@ -744,10 +744,7 @@ impl TimePanel { } // If this is an entity: - if let Some(components) = entity_db - .store() - .all_components(&tree.path) - { + if let Some(components) = entity_db.store().all_components(&tree.path) { for component_name in sorted_component_list_for_ui(components.iter()) { let is_static = entity_db .store() @@ -758,9 +755,12 @@ impl TimePanel { let item = TimePanelItem::component_path(component_path.clone()); let timeline = time_ctrl.timeline(); - let component_has_data_in_current_timeline = entity_db - .store() - .entity_has_component_on_timeline(time_ctrl.timeline(), &tree.path, &component_name); + let component_has_data_in_current_timeline = + entity_db.store().entity_has_component_on_timeline( + time_ctrl.timeline(), + &tree.path, + &component_name, + ); let total_num_messages = entity_db.store().num_events_on_timeline_for_component( time_ctrl.timeline(), From e7e57ea2f6d182f51108e0ee471ac6af83834ac9 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:10:51 +0200 Subject: [PATCH 29/43] use `u64` everywhere --- crates/store/re_chunk/src/chunk.rs | 16 +++++++------- crates/store/re_chunk_store/src/query.rs | 10 ++++----- .../re_time_panel/src/data_density_graph.rs | 22 +++++++++---------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/crates/store/re_chunk/src/chunk.rs b/crates/store/re_chunk/src/chunk.rs index 5c91f3e19507..40ec249852df 100644 --- a/crates/store/re_chunk/src/chunk.rs +++ b/crates/store/re_chunk/src/chunk.rs @@ -227,14 +227,14 @@ impl Chunk { // // TODO(cmc): This needs to be stored in chunk metadata and transported across IPC. #[inline] - pub fn num_events_cumulative(&self) -> usize { + pub fn num_events_cumulative(&self) -> u64 { // Reminder: component columns are sparse, we must take a look at the validity bitmaps. self.components .values() .map(|list_array| { list_array.validity().map_or_else( - || list_array.len(), - |validity| validity.len() - validity.unset_bits(), + || list_array.len() as u64, + |validity| validity.len() as u64 - validity.unset_bits() as u64, ) }) .sum() @@ -254,7 +254,7 @@ impl Chunk { re_tracing::profile_function!(); if self.is_static() { - return vec![(TimeInt::STATIC, self.num_events_cumulative() as u64)]; + return vec![(TimeInt::STATIC, self.num_events_cumulative())]; } let Some(time_chunk) = self.timelines().get(timeline) else { @@ -263,7 +263,7 @@ impl Chunk { let time_range = time_chunk.time_range(); if time_range.min() == time_range.max() { - return vec![(time_range.min(), self.num_events_cumulative() as u64)]; + return vec![(time_range.min(), self.num_events_cumulative())]; } let counts = if time_chunk.is_sorted() { @@ -363,12 +363,12 @@ impl Chunk { // // TODO(cmc): This needs to be stored in chunk metadata and transported across IPC. #[inline] - pub fn num_events_for_component(&self, component_name: ComponentName) -> Option { + pub fn num_events_for_component(&self, component_name: ComponentName) -> Option { // Reminder: component columns are sparse, we must check validity bitmap. self.components.get(&component_name).map(|list_array| { list_array.validity().map_or_else( - || list_array.len(), - |validity| validity.len() - validity.unset_bits(), + || list_array.len() as u64, + |validity| validity.len() as u64 - validity.unset_bits() as u64, ) }) } diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 60c25e71c6e1..e6ddd808cb4d 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -671,7 +671,7 @@ impl ChunkStore { timeline: &Timeline, entity_path: &EntityPath, component_name: ComponentName, - ) -> usize { + ) -> u64 { re_tracing::profile_function!(); self.num_events_for_static_component(entity_path, component_name) @@ -687,12 +687,12 @@ impl ChunkStore { &self, timeline: &Timeline, entity_path: &EntityPath, - ) -> usize { + ) -> u64 { re_tracing::profile_function!(); self.query_id.fetch_add(1, Ordering::Relaxed); - let mut total_events = 0; + let mut total_events = 0u64; if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) { @@ -728,7 +728,7 @@ impl ChunkStore { &self, entity_path: &EntityPath, component_name: ComponentName, - ) -> usize { + ) -> u64 { re_tracing::profile_function!(); self.query_id.fetch_add(1, Ordering::Relaxed); @@ -757,7 +757,7 @@ impl ChunkStore { timeline: &Timeline, entity_path: &EntityPath, component_name: ComponentName, - ) -> usize { + ) -> u64 { re_tracing::profile_function!(); self.query_id.fetch_add(1, Ordering::Relaxed); diff --git a/crates/viewer/re_time_panel/src/data_density_graph.rs b/crates/viewer/re_time_panel/src/data_density_graph.rs index affa3adbf549..255489430dd5 100644 --- a/crates/viewer/re_time_panel/src/data_density_graph.rs +++ b/crates/viewer/re_time_panel/src/data_density_graph.rs @@ -446,7 +446,7 @@ pub fn build_density_graph<'a>( .time_range_from_x_range((row_rect.left() - MARGIN_X)..=(row_rect.right() + MARGIN_X)); // NOTE: These chunks are guaranteed to have data on the current timeline - let mut chunk_ranges: Vec<(Arc, ResolvedTimeRange, usize)> = vec![]; + let mut chunk_ranges: Vec<(Arc, ResolvedTimeRange, u64)> = vec![]; let mut total_events = 0; { @@ -502,13 +502,13 @@ pub fn build_density_graph<'a>( #[derive(Clone, Copy)] pub struct DensityGraphBuilderConfig { /// If there are more chunks than this then we NEVER show individual events of any chunk. - pub max_total_chunk_events: usize, + pub max_total_chunk_events: u64, /// If a sorted chunk has fewer events than this we show its individual events. - pub max_events_in_sorted_chunk: usize, + pub max_events_in_sorted_chunk: u64, /// If an unsorted chunk has fewer events than this we show its individual events. - pub max_events_in_unsorted_chunk: usize, + pub max_events_in_unsorted_chunk: u64, } impl DensityGraphBuilderConfig { @@ -522,16 +522,16 @@ impl DensityGraphBuilderConfig { /// All sorted chunks will be rendered as individual events, /// and all unsorted chunks will be rendered whole. pub const ALWAYS_SPLIT_SORTED_CHUNKS: Self = Self { - max_total_chunk_events: usize::MAX, + max_total_chunk_events: u64::MAX, max_events_in_unsorted_chunk: 0, - max_events_in_sorted_chunk: usize::MAX, + max_events_in_sorted_chunk: u64::MAX, }; /// All chunks will be rendered as individual events. pub const ALWAYS_SPLIT_ALL_CHUNKS: Self = Self { - max_total_chunk_events: usize::MAX, - max_events_in_unsorted_chunk: usize::MAX, - max_events_in_sorted_chunk: usize::MAX, + max_total_chunk_events: u64::MAX, + max_events_in_unsorted_chunk: u64::MAX, + max_events_in_sorted_chunk: u64::MAX, }; } @@ -638,7 +638,7 @@ impl<'a> DensityGraphBuilder<'a> { } } - fn add_chunk_range(&mut self, time_range: ResolvedTimeRange, num_events: usize) { + fn add_chunk_range(&mut self, time_range: ResolvedTimeRange, num_events: u64) { if num_events == 0 { return; } @@ -693,7 +693,7 @@ fn visit_relevant_chunks( component_name: Option, timeline: Timeline, time_range: ResolvedTimeRange, - mut visitor: impl FnMut(Arc, ResolvedTimeRange, usize), + mut visitor: impl FnMut(Arc, ResolvedTimeRange, u64), ) { re_tracing::profile_function!(); From 0d88df1083417b3b862ce6de5f01cb21826dd214 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:32:54 +0200 Subject: [PATCH 30/43] more static/temporal semantics mess --- crates/store/re_chunk_store/src/query.rs | 64 ++++++++-------------- crates/store/re_entity_db/src/entity_db.rs | 24 ++++++++ crates/viewer/re_data_ui/src/item_ui.rs | 6 +- 3 files changed, 50 insertions(+), 44 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index e6ddd808cb4d..ecff4385d0f0 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -333,6 +333,8 @@ impl ChunkStore { } /// Returns the min and max times at which data was logged for an entity on a specific timeline. + /// + /// This ignores static data. pub fn entity_time_range( &self, timeline: &Timeline, @@ -683,27 +685,15 @@ impl ChunkStore { } /// Returns the number of events logged for an entity on a specific timeline. - pub fn num_events_on_timeline_for_all_components( - &self, - timeline: &Timeline, - entity_path: &EntityPath, - ) -> u64 { + /// + /// This ignores static data. + pub fn num_events_on_timeline(&self, timeline: &Timeline, entity_path: &EntityPath) -> u64 { re_tracing::profile_function!(); self.query_id.fetch_add(1, Ordering::Relaxed); let mut total_events = 0u64; - if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) - { - for chunk in static_chunks_per_component - .values() - .filter_map(|id| self.chunks_per_chunk_id.get(id)) - { - total_events += chunk.num_events_cumulative(); - } - } - if let Some(chunk_ids) = self .temporal_chunk_ids_per_entity .get(entity_path) @@ -762,32 +752,26 @@ impl ChunkStore { self.query_id.fetch_add(1, Ordering::Relaxed); - let Some(temporal_chunk_ids_per_timeline) = self - .temporal_chunk_ids_per_entity_per_component + self.temporal_chunk_ids_per_entity_per_component .get(entity_path) - else { - return 0; // no events logged for the entity path - }; - - let Some(temporal_chunk_ids_per_component) = temporal_chunk_ids_per_timeline.get(timeline) - else { - return 0; // no events logged on this timeline - }; - - let Some(chunk_ids) = temporal_chunk_ids_per_component.get(&component_name) else { - return 0; // no events logged for the component on this timeline - }; - - let mut num_events = 0; - for chunk in chunk_ids - .per_start_time - .values() - .flat_map(|ids| ids.iter().filter_map(|id| self.chunks_per_chunk_id.get(id))) - { - num_events += chunk.num_events_for_component(component_name).unwrap_or(0); - } - - num_events + .and_then(|temporal_chunk_ids_per_timeline| { + temporal_chunk_ids_per_timeline.get(timeline) + }) + .and_then(|temporal_chunk_ids_per_component| { + temporal_chunk_ids_per_component.get(&component_name) + }) + .map_or(0, |chunk_id_sets| { + chunk_id_sets + .per_start_time + .values() + .flat_map(|chunk_ids| { + chunk_ids + .iter() + .filter_map(|chunk_id| self.chunks_per_chunk_id.get(chunk_id)) + .filter_map(|chunk| chunk.num_events_for_component(component_name)) + }) + .sum() + }) } /// Returns the size of the entity on a specific timeline. diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index 06977804eaa1..6a1aaefd56f8 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -569,6 +569,8 @@ impl EntityDb { } /// Returns true if an entity or any of its children have any data on the given timeline. + /// + /// This includes static data. pub fn subtree_has_data_on_timeline( &self, timeline: &Timeline, @@ -586,6 +588,28 @@ impl EntityDb { }) .is_some() } + + /// Returns true if an entity or any of its children have any temporal data on the given timeline. + /// + /// This ignores static data. + pub fn subtree_has_temporal_data_on_timeline( + &self, + timeline: &Timeline, + entity_path: &EntityPath, + ) -> bool { + re_tracing::profile_function!(); + + let Some(subtree) = self.tree.subtree(entity_path) else { + return false; + }; + + subtree + .find_first_child_recursive(|path| { + self.store() + .entity_has_temporal_data_on_timeline(timeline, path) + }) + .is_some() + } } impl re_types_core::SizeBytes for EntityDb { diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index 25c46a27ebdb..3fa23bf05e29 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -315,10 +315,8 @@ fn entity_tree_stats_ui( let mut data_rate = None; // Try to estimate data-rate - if db.store().entity_has_data_on_timeline(timeline, &tree.path) { - let num_events = db - .store() - .num_events_on_timeline_for_all_components(timeline, &tree.path); + if db.subtree_has_temporal_data_on_timeline(timeline, &tree.path) { + let num_events = db.store().num_events_on_timeline(timeline, &tree.path); if let Some(time_range) = db.store().entity_time_range(timeline, &tree.path) { let min_time = time_range.min(); From 1bf8adefe2027d67ce34d0dfc3ad77b696260e0f Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:53:40 +0200 Subject: [PATCH 31/43] refactor into combinators --- crates/store/re_chunk_store/src/query.rs | 129 ++++++++++------------- 1 file changed, 54 insertions(+), 75 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index ecff4385d0f0..fb21fc56dfd6 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -259,15 +259,12 @@ impl ChunkStore { .is_some_and(|temporal_chunks_per_timeline| { temporal_chunks_per_timeline .values() - .any(|temporal_chunks_per_component| { - temporal_chunks_per_component.values().any(|chunk_id_sets| { - chunk_id_sets.per_start_time.values().any(|chunk_id_set| { - chunk_id_set - .iter() - .any(|chunk_id| self.chunks_per_chunk_id.contains_key(chunk_id)) - }) - }) + .flat_map(|temporal_chunks_per_component| { + temporal_chunks_per_component.values() }) + .flat_map(|chunk_id_sets| chunk_id_sets.per_start_time.values()) + .flat_map(|chunk_id_set| chunk_id_set.iter()) + .any(|chunk_id| self.chunks_per_chunk_id.contains_key(chunk_id)) }) } @@ -287,18 +284,13 @@ impl ChunkStore { self.temporal_chunk_ids_per_entity_per_component .get(entity_path) - .is_some_and(|temporal_chunks_per_timeline| { - temporal_chunks_per_timeline.get(timeline).is_some_and( - |temporal_chunks_per_component| { - temporal_chunks_per_component.values().any(|chunk_id_sets| { - chunk_id_sets.per_start_time.values().any(|chunk_id_set| { - chunk_id_set - .iter() - .any(|chunk_id| self.chunks_per_chunk_id.contains_key(chunk_id)) - }) - }) - }, - ) + .and_then(|temporal_chunks_per_timeline| temporal_chunks_per_timeline.get(timeline)) + .is_some_and(|temporal_chunks_per_component| { + temporal_chunks_per_component + .values() + .flat_map(|chunk_id_sets| chunk_id_sets.per_start_time.values()) + .flat_map(|chunk_id_set| chunk_id_set.iter()) + .any(|chunk_id| self.chunks_per_chunk_id.contains_key(chunk_id)) }) } @@ -663,11 +655,11 @@ impl ChunkStore { } } -// Queries returning `usize` and `bool` +// Counting impl ChunkStore { /// Returns the number of events of a specific component logged for an entity on a specific timeline. /// - /// This counts both static and temporal components. + /// This counts events for both static and temporal components. pub fn num_events_on_timeline_for_component( &self, timeline: &Timeline, @@ -692,25 +684,23 @@ impl ChunkStore { self.query_id.fetch_add(1, Ordering::Relaxed); - let mut total_events = 0u64; - - if let Some(chunk_ids) = self - .temporal_chunk_ids_per_entity + self.temporal_chunk_ids_per_entity .get(entity_path) .and_then(|temporal_chunks_events_per_timeline| { temporal_chunks_events_per_timeline.get(timeline) }) - { - for chunk in chunk_ids - .per_start_time - .values() - .flat_map(|ids| ids.iter().filter_map(|id| self.chunks_per_chunk_id.get(id))) - { - total_events += chunk.num_events_cumulative(); - } - } - - total_events + .map_or(0, |chunk_id_sets| { + chunk_id_sets + .per_start_time + .values() + .flat_map(|chunk_ids| { + chunk_ids + .iter() + .filter_map(|chunk_id| self.chunks_per_chunk_id.get(chunk_id)) + .map(|chunk| chunk.num_events_cumulative()) + }) + .sum() + }) } /// Returns the number of times a static component was logged to an entity. @@ -723,20 +713,14 @@ impl ChunkStore { self.query_id.fetch_add(1, Ordering::Relaxed); - if let Some(static_chunk) = self - .static_chunk_ids_per_entity + self.static_chunk_ids_per_entity .get(entity_path) .and_then(|static_chunks_per_component| { static_chunks_per_component.get(&component_name) }) .and_then(|chunk_id| self.chunks_per_chunk_id.get(chunk_id)) - { - static_chunk - .num_events_for_component(component_name) - .unwrap_or(0) - } else { - 0 - } + .and_then(|chunk| chunk.num_events_for_component(component_name)) + .unwrap_or(0) } /// Returns the number of times a temporal component was logged to an entity path on a specific timeline. @@ -764,12 +748,9 @@ impl ChunkStore { chunk_id_sets .per_start_time .values() - .flat_map(|chunk_ids| { - chunk_ids - .iter() - .filter_map(|chunk_id| self.chunks_per_chunk_id.get(chunk_id)) - .filter_map(|chunk| chunk.num_events_for_component(component_name)) - }) + .flat_map(|chunk_ids| chunk_ids.iter()) + .filter_map(|chunk_id| self.chunks_per_chunk_id.get(chunk_id)) + .filter_map(|chunk| chunk.num_events_for_component(component_name)) .sum() }) } @@ -789,35 +770,33 @@ impl ChunkStore { self.query_id.fetch_add(1, Ordering::Relaxed); - let mut total_size = 0; - - if let Some(static_chunks_per_component) = self.static_chunk_ids_per_entity.get(entity_path) - { - for chunk in static_chunks_per_component - .values() - .filter_map(|id| self.chunks_per_chunk_id.get(id)) - { - total_size += Chunk::total_size_bytes(chunk) as usize; - } - } + let static_data_size = self.static_chunk_ids_per_entity.get(entity_path).map_or( + 0, + |static_chunks_per_component| { + static_chunks_per_component + .values() + .filter_map(|id| self.chunks_per_chunk_id.get(id)) + .map(|chunk| Chunk::total_size_bytes(chunk) as usize) + .sum() + }, + ); - if let Some(chunk_id_sets) = self + let temporal_data_size = self .temporal_chunk_ids_per_entity .get(entity_path) .and_then(|temporal_chunk_ids_per_timeline| { temporal_chunk_ids_per_timeline.get(timeline) }) - { - for chunk in chunk_id_sets - .per_start_time - .values() - .flat_map(|v| v.iter()) - .filter_map(|id| self.chunks_per_chunk_id.get(id)) - { - total_size += Chunk::total_size_bytes(chunk) as usize; - } - } + .map_or(0, |chunk_id_sets| { + chunk_id_sets + .per_start_time + .values() + .flat_map(|chunk_ids| chunk_ids.iter()) + .filter_map(|id| self.chunks_per_chunk_id.get(id)) + .map(|chunk| Chunk::total_size_bytes(chunk) as usize) + .sum() + }); - total_size + static_data_size + temporal_data_size } } From 5ffb0901a0626f541fdc00a8fa6d0a7e11b0698b Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 18:02:57 +0200 Subject: [PATCH 32/43] clarify temporal/static in `num_events` APIs --- crates/store/re_chunk_store/src/query.rs | 25 +++----------- crates/viewer/re_data_ui/src/component.rs | 4 +-- crates/viewer/re_data_ui/src/item_ui.rs | 4 ++- crates/viewer/re_time_panel/src/lib.rs | 41 ++++++++++++++--------- 4 files changed, 36 insertions(+), 38 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index fb21fc56dfd6..4a63db0c5681 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -657,31 +657,16 @@ impl ChunkStore { // Counting impl ChunkStore { - /// Returns the number of events of a specific component logged for an entity on a specific timeline. + /// Returns the number of events logged for an entity on a specific timeline. /// - /// This counts events for both static and temporal components. - pub fn num_events_on_timeline_for_component( + /// This ignores static data. + pub fn num_temporal_events_on_timeline( &self, timeline: &Timeline, entity_path: &EntityPath, - component_name: ComponentName, ) -> u64 { re_tracing::profile_function!(); - self.num_events_for_static_component(entity_path, component_name) - + self.num_events_on_timeline_for_temporal_component( - timeline, - entity_path, - component_name, - ) - } - - /// Returns the number of events logged for an entity on a specific timeline. - /// - /// This ignores static data. - pub fn num_events_on_timeline(&self, timeline: &Timeline, entity_path: &EntityPath) -> u64 { - re_tracing::profile_function!(); - self.query_id.fetch_add(1, Ordering::Relaxed); self.temporal_chunk_ids_per_entity @@ -704,7 +689,7 @@ impl ChunkStore { } /// Returns the number of times a static component was logged to an entity. - pub fn num_events_for_static_component( + pub fn num_static_events_for_component( &self, entity_path: &EntityPath, component_name: ComponentName, @@ -726,7 +711,7 @@ impl ChunkStore { /// Returns the number of times a temporal component was logged to an entity path on a specific timeline. /// /// This ignores static data. - pub fn num_events_on_timeline_for_temporal_component( + pub fn num_temporal_events_for_component_on_timeline( &self, timeline: &Timeline, entity_path: &EntityPath, diff --git a/crates/viewer/re_data_ui/src/component.rs b/crates/viewer/re_data_ui/src/component.rs index e721b7f28c78..bcd5de12d949 100644 --- a/crates/viewer/re_data_ui/src/component.rs +++ b/crates/viewer/re_data_ui/src/component.rs @@ -74,7 +74,7 @@ impl<'a> DataUi for EntityLatestAtResults<'a> { if self.results.is_static() { let static_message_count = db .store() - .num_events_for_static_component(&self.entity_path, component_name); + .num_static_events_for_component(&self.entity_path, component_name); if static_message_count > 1 { ui.label(ui.ctx().warning_text(format!( "Static component value was overridden {} times", @@ -88,7 +88,7 @@ impl<'a> DataUi for EntityLatestAtResults<'a> { } let temporal_message_count = - db.store().num_events_on_timeline_for_temporal_component( + db.store().num_temporal_events_for_component_on_timeline( &query.timeline(), &self.entity_path, component_name, diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index 3fa23bf05e29..4006ddb8ca8d 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -316,7 +316,9 @@ fn entity_tree_stats_ui( // Try to estimate data-rate if db.subtree_has_temporal_data_on_timeline(timeline, &tree.path) { - let num_events = db.store().num_events_on_timeline(timeline, &tree.path); + let num_events = db + .store() + .num_temporal_events_on_timeline(timeline, &tree.path); if let Some(time_range) = db.store().entity_time_range(timeline, &tree.path) { let min_time = time_range.min(); diff --git a/crates/viewer/re_time_panel/src/lib.rs b/crates/viewer/re_time_panel/src/lib.rs index a3933c921bbe..37238e25fc27 100644 --- a/crates/viewer/re_time_panel/src/lib.rs +++ b/crates/viewer/re_time_panel/src/lib.rs @@ -762,11 +762,17 @@ impl TimePanel { &component_name, ); - let total_num_messages = entity_db.store().num_events_on_timeline_for_component( - time_ctrl.timeline(), - &tree.path, - component_name, - ); + let num_static_messages = entity_db + .store() + .num_static_events_for_component(&tree.path, component_name); + let num_temporal_messages = entity_db + .store() + .num_temporal_events_for_component_on_timeline( + time_ctrl.timeline(), + &tree.path, + component_name, + ); + let total_num_messages = num_static_messages + num_temporal_messages; let response = ui .list_item() @@ -806,19 +812,24 @@ impl TimePanel { ))); } else { list_item::list_item_scope(ui, "hover tooltip", |ui| { + let kind = if is_static { "Static" } else { "Temporal" }; + + let num_messages = if is_static { + num_static_messages + } else { + num_temporal_messages + }; + + let num_messages = if num_messages == 1 { + "once".to_owned() + } else { + format!("{} times", re_format::format_uint(num_messages)) + }; + ui.list_item().interactive(false).show_flat( ui, list_item::LabelContent::new(format!( - "{} component, logged {}", - if is_static { "Static" } else { "Temporal" }, - if total_num_messages == 1 { - "once".to_owned() - } else { - format!( - "{} times", - re_format::format_uint(total_num_messages) - ) - }, + "{kind} component, logged {num_messages}" )) .truncate(false) .with_icon(if is_static { From 0291f807748060a3a56eaf98d98b18266f7ac627 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 18:09:22 +0200 Subject: [PATCH 33/43] rationale --- crates/store/re_chunk_store/src/query.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 4a63db0c5681..8cb3b00248b2 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -18,6 +18,11 @@ use crate::RowId; // --- +// These APIs often have `temporal` and `static` variants. +// It is sometimes useful to be able to separately query either, +// such as when we want to tell the user that they logged a component +// as both static and temporal, which is probably wrong. + impl ChunkStore { /// Retrieve all the [`ComponentName`]s that have been written to for a given [`EntityPath`] on /// the specified [`Timeline`]. @@ -711,6 +716,8 @@ impl ChunkStore { /// Returns the number of times a temporal component was logged to an entity path on a specific timeline. /// /// This ignores static data. + /// + /// An example use case of this is detecting when static data is logged more than once. pub fn num_temporal_events_for_component_on_timeline( &self, timeline: &Timeline, From e3083b67a418d6beda6ec369a6c8061ab823dfb5 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 18:13:29 +0200 Subject: [PATCH 34/43] specify units --- crates/store/re_chunk_store/src/query.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 8cb3b00248b2..a04f75b0e7c4 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -747,7 +747,7 @@ impl ChunkStore { }) } - /// Returns the size of the entity on a specific timeline. + /// Returns number of bytes used for an entity's data on a specific timeline. /// /// This includes static data. /// @@ -762,7 +762,7 @@ impl ChunkStore { self.query_id.fetch_add(1, Ordering::Relaxed); - let static_data_size = self.static_chunk_ids_per_entity.get(entity_path).map_or( + let static_data_size_bytes = self.static_chunk_ids_per_entity.get(entity_path).map_or( 0, |static_chunks_per_component| { static_chunks_per_component @@ -773,7 +773,7 @@ impl ChunkStore { }, ); - let temporal_data_size = self + let temporal_data_size_bytes = self .temporal_chunk_ids_per_entity .get(entity_path) .and_then(|temporal_chunk_ids_per_timeline| { @@ -789,6 +789,6 @@ impl ChunkStore { .sum() }); - static_data_size + temporal_data_size + static_data_size_bytes + temporal_data_size_bytes } } From 4082b9cdb23b4b9532ecd833c25f237175a364af Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 18:17:58 +0200 Subject: [PATCH 35/43] clarify `size_of_entity_on_timeline` --- crates/store/re_chunk_store/src/query.rs | 11 ++++++++--- crates/store/re_entity_db/src/entity_db.rs | 6 ++++-- crates/viewer/re_data_ui/src/item_ui.rs | 16 +++++++--------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index a04f75b0e7c4..970501c7adc4 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -747,13 +747,18 @@ impl ChunkStore { }) } - /// Returns number of bytes used for an entity's data on a specific timeline. + /// Returns number of bytes used for an entity on a specific timeline. /// - /// This includes static data. + /// This is an approximation of the actual storage cost of the entity, + /// as the measurement includes the overhead of various data structures + /// we use in the database. + /// It is imprecise, because it does not account for every possible place + /// someone may be storing something related to the entity, only most of + /// what is accessible inside this chunk store. /// /// ⚠ This does not return the _total_ size of the entity and all its children! /// For that, use `entity_db.size_of_subtree_on_timeline`. - pub fn size_of_entity_on_timeline( + pub fn approx_size_of_entity_on_timeline( &self, timeline: &Timeline, entity_path: &EntityPath, diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index 6a1aaefd56f8..069d46bd6757 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -549,7 +549,7 @@ impl EntityDb { /// Returns the byte size of an entity and all its children on the given timeline, recursively. /// /// This includes static data. - pub fn size_of_subtree_on_timeline( + pub fn approx_size_of_subtree_on_timeline( &self, timeline: &Timeline, entity_path: &EntityPath, @@ -562,7 +562,9 @@ impl EntityDb { let mut size = 0; subtree.visit_children_recursively(|path| { - size += self.store().size_of_entity_on_timeline(timeline, path); + size += self + .store() + .approx_size_of_entity_on_timeline(timeline, path); }); size diff --git a/crates/viewer/re_data_ui/src/item_ui.rs b/crates/viewer/re_data_ui/src/item_ui.rs index 4006ddb8ca8d..6bdb7f5e4261 100644 --- a/crates/viewer/re_data_ui/src/item_ui.rs +++ b/crates/viewer/re_data_ui/src/item_ui.rs @@ -299,16 +299,14 @@ fn entity_tree_stats_ui( ) { use re_format::format_bytes; - // Show total bytes used in whole subtree - let total_bytes = db.size_of_subtree_on_timeline(timeline, &tree.path); - let subtree_caveat = if tree.children.is_empty() { "" } else { " (including subtree)" }; - if total_bytes == 0 { + let approx_num_bytes = db.approx_size_of_subtree_on_timeline(timeline, &tree.path); + if approx_num_bytes == 0 { return; } @@ -335,7 +333,7 @@ fn entity_tree_stats_ui( let duration = max_time.as_f64() - min_time.as_f64(); - let mut bytes_per_time = total_bytes as f64 / duration; + let mut bytes_per_time = approx_num_bytes as f64 / duration; // Fencepost adjustment: bytes_per_time *= (num_events - 1) as f64 / num_events as f64; @@ -361,14 +359,14 @@ fn entity_tree_stats_ui( if let Some(data_rate) = data_rate { ui.label(format!( - "Using {}{subtree_caveat} ≈ {}", - format_bytes(total_bytes as f64), + "Using ~{}{subtree_caveat} ≈ {}", + format_bytes(approx_num_bytes as f64), data_rate )); } else { ui.label(format!( - "Using {}{subtree_caveat}", - format_bytes(total_bytes as f64) + "Using ~{}{subtree_caveat}", + format_bytes(approx_num_bytes as f64) )); } } From 5d21b1cdcc0058bbc2c9c5b7f83d948f2dfc190d Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 18:18:38 +0200 Subject: [PATCH 36/43] typo --- crates/store/re_chunk_store/src/query.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 970501c7adc4..0de76fc6bdb0 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -757,7 +757,7 @@ impl ChunkStore { /// what is accessible inside this chunk store. /// /// ⚠ This does not return the _total_ size of the entity and all its children! - /// For that, use `entity_db.size_of_subtree_on_timeline`. + /// For that, use `entity_db.approx_size_of_subtree_on_timeline`. pub fn approx_size_of_entity_on_timeline( &self, timeline: &Timeline, From 5ef30e9292272307ecb449279b8e6d812d1156cb Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 18:19:40 +0200 Subject: [PATCH 37/43] use `u64` for entity size --- crates/store/re_chunk_store/src/query.rs | 6 +++--- crates/store/re_entity_db/src/entity_db.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 0de76fc6bdb0..873d768114ca 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -762,7 +762,7 @@ impl ChunkStore { &self, timeline: &Timeline, entity_path: &EntityPath, - ) -> usize { + ) -> u64 { re_tracing::profile_function!(); self.query_id.fetch_add(1, Ordering::Relaxed); @@ -773,7 +773,7 @@ impl ChunkStore { static_chunks_per_component .values() .filter_map(|id| self.chunks_per_chunk_id.get(id)) - .map(|chunk| Chunk::total_size_bytes(chunk) as usize) + .map(|chunk| Chunk::total_size_bytes(chunk)) .sum() }, ); @@ -790,7 +790,7 @@ impl ChunkStore { .values() .flat_map(|chunk_ids| chunk_ids.iter()) .filter_map(|id| self.chunks_per_chunk_id.get(id)) - .map(|chunk| Chunk::total_size_bytes(chunk) as usize) + .map(|chunk| Chunk::total_size_bytes(chunk)) .sum() }); diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index 069d46bd6757..ce59466fada1 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -553,7 +553,7 @@ impl EntityDb { &self, timeline: &Timeline, entity_path: &EntityPath, - ) -> usize { + ) -> u64 { re_tracing::profile_function!(); let Some(subtree) = self.tree.subtree(entity_path) else { From b2001e5202bb498385118cb04d276c8a28fd2cb5 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Mon, 22 Jul 2024 18:24:46 +0200 Subject: [PATCH 38/43] add back unused param --- crates/store/re_entity_db/src/entity_db.rs | 4 +--- crates/store/re_entity_db/src/entity_tree.rs | 12 +++++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index ce59466fada1..c4634f39e813 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -432,9 +432,7 @@ impl EntityDb { self.times_per_timeline.on_events(store_events); self.query_caches.on_events(store_events); self.time_histogram_per_timeline.on_events(store_events); - - // Tree deletions depend on data store, so data store must have been notified of deletions already. - self.tree.on_store_deletions(&self.data_store); + self.tree.on_store_deletions(&self.data_store, store_events); } /// Key used for sorting recordings in the UI. diff --git a/crates/store/re_entity_db/src/entity_tree.rs b/crates/store/re_entity_db/src/entity_tree.rs index 356879803492..4d5d4a30f8de 100644 --- a/crates/store/re_entity_db/src/entity_tree.rs +++ b/crates/store/re_entity_db/src/entity_tree.rs @@ -156,15 +156,17 @@ impl EntityTree { } /// Updates the [`EntityTree`] by removing any entities which have no data and no children. - /// - /// ⚠ This depends on `data_store` having up-to-date records of which entities have no data, - /// so it must have already been notified of any chunk events prior to calling this method. - pub fn on_store_deletions(&mut self, data_store: &ChunkStore) { + pub fn on_store_deletions(&mut self, data_store: &ChunkStore, events: &[ChunkStoreEvent]) { re_tracing::profile_function!(); + // We don't actually use the events for anything, we just want to + // have a direct dependency on the chunk store which must have + // produced them by the time this function was called. + let _ = events; + self.children.retain(|_, entity| { // this is placed first, because we'll only know if the child entity is empty after telling it to clear itself. - entity.on_store_deletions(data_store); + entity.on_store_deletions(data_store, events); !entity.is_empty(data_store) }); From eb98e819fcca665a862dba6778cac2f69cfc8ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Proch=C3=A1zka?= Date: Tue, 23 Jul 2024 09:54:06 +0200 Subject: [PATCH 39/43] Update crates/store/re_chunk_store/src/query.rs Co-authored-by: Clement Rey --- crates/store/re_chunk_store/src/query.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 873d768114ca..e1fae139b88a 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -662,7 +662,7 @@ impl ChunkStore { // Counting impl ChunkStore { - /// Returns the number of events logged for an entity on a specific timeline. + /// Returns the number of temporal events logged for an entity on a specific timeline. /// /// This ignores static data. pub fn num_temporal_events_on_timeline( From 9eea627e079a04a5eeb8762279514d3a5672b213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Proch=C3=A1zka?= Date: Tue, 23 Jul 2024 09:54:17 +0200 Subject: [PATCH 40/43] Update crates/store/re_chunk_store/src/query.rs Co-authored-by: Clement Rey --- crates/store/re_chunk_store/src/query.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index e1fae139b88a..50e6eefe3663 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -693,7 +693,7 @@ impl ChunkStore { }) } - /// Returns the number of times a static component was logged to an entity. + /// Returns the number of static events logged for an entity for a specific component. pub fn num_static_events_for_component( &self, entity_path: &EntityPath, From 12dc3397ed8f855b471c5aac166f98fdbe5c7226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Proch=C3=A1zka?= Date: Tue, 23 Jul 2024 09:54:24 +0200 Subject: [PATCH 41/43] Update crates/store/re_chunk_store/src/query.rs Co-authored-by: Clement Rey --- crates/store/re_chunk_store/src/query.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 50e6eefe3663..17a6118a1b40 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -713,7 +713,7 @@ impl ChunkStore { .unwrap_or(0) } - /// Returns the number of times a temporal component was logged to an entity path on a specific timeline. + /// Returns the number of temporal events logged for an entity for a specific component on a given timeline. /// /// This ignores static data. /// From c95c1822be4f4781117860eecfa8b5a1fc50c56b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Proch=C3=A1zka?= Date: Tue, 23 Jul 2024 09:54:35 +0200 Subject: [PATCH 42/43] Update crates/store/re_chunk_store/src/query.rs Co-authored-by: Clement Rey --- crates/store/re_chunk_store/src/query.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 17a6118a1b40..8cf6684be7d6 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -749,6 +749,7 @@ impl ChunkStore { /// Returns number of bytes used for an entity on a specific timeline. /// + /// This always includes static data. /// This is an approximation of the actual storage cost of the entity, /// as the measurement includes the overhead of various data structures /// we use in the database. From 82127ba0b1eecca4a3d93eb3eb3e1033c6099088 Mon Sep 17 00:00:00 2001 From: jprochazk <1665677+jprochazk@users.noreply.github.com> Date: Tue, 23 Jul 2024 09:55:55 +0200 Subject: [PATCH 43/43] docs --- crates/store/re_chunk_store/src/query.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/store/re_chunk_store/src/query.rs b/crates/store/re_chunk_store/src/query.rs index 8cf6684be7d6..ae43f6596db0 100644 --- a/crates/store/re_chunk_store/src/query.rs +++ b/crates/store/re_chunk_store/src/query.rs @@ -694,6 +694,8 @@ impl ChunkStore { } /// Returns the number of static events logged for an entity for a specific component. + /// + /// This ignores temporal events. pub fn num_static_events_for_component( &self, entity_path: &EntityPath, @@ -715,9 +717,7 @@ impl ChunkStore { /// Returns the number of temporal events logged for an entity for a specific component on a given timeline. /// - /// This ignores static data. - /// - /// An example use case of this is detecting when static data is logged more than once. + /// This ignores static events. pub fn num_temporal_events_for_component_on_timeline( &self, timeline: &Timeline,