Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement ChunkStore and integrate it everywhere #6570

Merged
merged 10 commits into from
Jul 4, 2024
Prev Previous commit
Next Next commit
address review comments
teh-cmc committed Jul 4, 2024
commit e4977d136a99e2132c061831dc323b584a938962
7 changes: 4 additions & 3 deletions crates/re_chunk_store/src/query.rs
Original file line number Diff line number Diff line change
@@ -115,7 +115,7 @@ impl ChunkStore {
///
/// If the entity has static component data associated with it, it will unconditionally
/// override any temporal component data.
pub fn latest_at(
pub fn latest_at_relevant_chunks(
&self,
query: &LatestAtQuery,
entity_path: &EntityPath,
@@ -197,14 +197,15 @@ impl ChunkStore {

/// Returns the most-relevant chunk(s) for the given [`RangeQuery`].
///
/// It is possible that some or all of the returned [`Chunk`]s overlap in time.
/// The criteria for returning a chunk is only that it may contain data that overlaps with
/// the queried range.
///
/// The caller should filter the returned chunks further (see [`Chunk::range`]) in order to
/// determine how exactly each row of data fit with the rest.
///
/// If the entity has static component data associated with it, it will unconditionally
/// override any temporal component data.
pub fn range(
pub fn range_relevant_chunks(
&self,
query: &RangeQuery,
entity_path: &EntityPath,
12 changes: 6 additions & 6 deletions crates/re_chunk_store/tests/correctness.rs
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@ fn query_latest_component<C: re_types_core::Component>(
re_tracing::profile_function!();

let (data_time, row_id, array) = store
.latest_at(query, entity_path, C::name())
.latest_at_relevant_chunks(query, entity_path, C::name())
.into_iter()
.flat_map(|chunk| {
chunk
@@ -270,7 +270,7 @@ fn latest_at_emptiness_edge_cases() -> anyhow::Result<()> {

// empty frame_nr
{
let chunks = store.latest_at(
let chunks = store.latest_at_relevant_chunks(
&LatestAtQuery::new(timeline_frame_nr, frame39),
&entity_path,
MyIndex::name(),
@@ -280,7 +280,7 @@ fn latest_at_emptiness_edge_cases() -> anyhow::Result<()> {

// empty log_time
{
let chunks = store.latest_at(
let chunks = store.latest_at_relevant_chunks(
&LatestAtQuery::new(timeline_log_time, now_minus_1s_nanos),
&entity_path,
MyIndex::name(),
@@ -290,7 +290,7 @@ fn latest_at_emptiness_edge_cases() -> anyhow::Result<()> {

// wrong entity path
{
let chunks = store.latest_at(
let chunks = store.latest_at_relevant_chunks(
&LatestAtQuery::new(timeline_frame_nr, frame40),
&EntityPath::from("does/not/exist"),
MyIndex::name(),
@@ -300,7 +300,7 @@ fn latest_at_emptiness_edge_cases() -> anyhow::Result<()> {

// wrong timeline name
{
let chunks = store.latest_at(
let chunks = store.latest_at_relevant_chunks(
&LatestAtQuery::new(timeline_wrong_name, frame40),
&EntityPath::from("does/not/exist"),
MyIndex::name(),
@@ -310,7 +310,7 @@ fn latest_at_emptiness_edge_cases() -> anyhow::Result<()> {

// wrong timeline kind
{
let chunks = store.latest_at(
let chunks = store.latest_at_relevant_chunks(
&LatestAtQuery::new(timeline_wrong_kind, frame40),
&EntityPath::from("does/not/exist"),
MyIndex::name(),
2 changes: 1 addition & 1 deletion crates/re_chunk_store/tests/gc.rs
Original file line number Diff line number Diff line change
@@ -25,7 +25,7 @@ fn query_latest_array(
re_tracing::profile_function!();

let (data_time, row_id, array) = store
.latest_at(query, entity_path, component_name)
.latest_at_relevant_chunks(query, entity_path, component_name)
.into_iter()
.flat_map(|chunk| {
chunk
4 changes: 2 additions & 2 deletions crates/re_chunk_store/tests/reads.rs
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ fn query_latest_array(
re_tracing::profile_function!();

let (data_time, row_id, array) = store
.latest_at(query, entity_path, component_name)
.latest_at_relevant_chunks(query, entity_path, component_name)
.into_iter()
.flat_map(|chunk| {
chunk
@@ -580,7 +580,7 @@ fn range() -> anyhow::Result<()> {
let timeline_frame_nr = Timeline::new("frame_nr", TimeType::Sequence);

let query = RangeQuery::new(timeline_frame_nr, time_range);
let results = store.range(&query, &entity_path, component_name);
let results = store.range_relevant_chunks(&query, &entity_path, component_name);

eprintln!("================= {component_name} @ {query:?} ===============");
let mut results_processed = 0usize;
1 change: 1 addition & 0 deletions crates/re_entity_db/src/entity_db.rs
Original file line number Diff line number Diff line change
@@ -486,6 +486,7 @@ impl EntityDb {
return true;
};

// TODO(cmc): chunk.slice_time_selection(time_selection)
chunk
.timelines()
.get(&timeline)
2 changes: 1 addition & 1 deletion crates/re_query/src/latest_at/query.rs
Original file line number Diff line number Diff line change
@@ -298,7 +298,7 @@ pub fn latest_at(
component_name: ComponentName,
) -> Option<(TimeInt, RowId, Box<dyn ArrowArray>)> {
store
.latest_at(query, entity_path, component_name)
.latest_at_relevant_chunks(query, entity_path, component_name)
.into_iter()
.flat_map(|chunk| {
chunk
2 changes: 1 addition & 1 deletion crates/re_query/src/range/query.rs
Original file line number Diff line number Diff line change
@@ -211,7 +211,7 @@ pub fn range<'a>(
component_name: ComponentName,
) -> impl Iterator<Item = (TimeInt, RowId, Box<dyn Array>)> + 'a {
store
.range(query, entity_path, component_name)
.range_relevant_chunks(query, entity_path, component_name)
.into_iter()
.map(move |chunk| chunk.range(query, component_name))
.filter(|chunk| !chunk.is_empty())
2 changes: 1 addition & 1 deletion crates/re_space_view_dataframe/src/space_view_class.rs
Original file line number Diff line number Diff line change
@@ -232,7 +232,7 @@ fn sorted_instance_paths_for<'a>(
.filter(|component_name| !component_name.is_indicator_component())
.flat_map(|component_name| {
let num_instances = store
.latest_at(latest_at_query, entity_path, component_name)
.latest_at_relevant_chunks(latest_at_query, entity_path, component_name)
.into_iter()
.filter_map(|chunk| {
let (data_time, row_id, batch) = chunk