Skip to content

Commit 5377dfd

Browse files
authored
feat: try load page indexes (#1425)
## Rationale See #1040 ## Detailed Changes - Try load page indexes ## Test Plan CI
1 parent 94673bd commit 5377dfd

File tree

1 file changed

+22
-2
lines changed

1 file changed

+22
-2
lines changed

analytic_engine/src/sst/parquet/async_reader.rs

+22-2
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ use datafusion::{
3737
};
3838
use futures::{Stream, StreamExt};
3939
use generic_error::{BoxError, GenericResult};
40-
use logger::{debug, error};
40+
use logger::{debug, error, warn};
4141
use object_store::{ObjectStoreRef, Path};
4242
use parquet::{
4343
arrow::{arrow_reader::RowSelection, ParquetRecordBatchStreamBuilder, ProjectionMask},
@@ -397,7 +397,27 @@ impl<'a> Reader<'a> {
397397
file_path: self.path.to_string(),
398398
})?;
399399

400-
// TODO: Support page index until https://github.com/CeresDB/ceresdb/issues/1040 is fixed.
400+
let mut parquet_meta_data = Arc::new(parquet_meta_data);
401+
let object_store_reader = parquet_ext::reader::ObjectStoreReader::new(
402+
self.store.clone(),
403+
self.path.clone(),
404+
parquet_meta_data.clone(),
405+
);
406+
407+
if let Ok(meta_data) = parquet_ext::meta_data::meta_with_page_indexes(object_store_reader)
408+
.await
409+
.map_err(|e| {
410+
// When loading page indexes failed, we just log the error and continue querying
411+
// TODO: Fix this in stream. https://github.com/apache/incubator-horaedb/issues/1040
412+
warn!(
413+
"Fail to load page indexes, path:{}, err:{:?}.",
414+
self.path, e
415+
);
416+
e
417+
})
418+
{
419+
parquet_meta_data = meta_data;
420+
}
401421

402422
MetaData::try_new(&parquet_meta_data, ignore_sst_filter, self.store.clone())
403423
.await

0 commit comments

Comments
 (0)