Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: use encoded_size as memory usage #1213

Merged
merged 3 commits into from
Sep 14, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 9 additions & 10 deletions analytic_engine/src/memtable/skiplist/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,7 @@
pub mod factory;
pub mod iter;

use std::{
convert::TryInto,
sync::atomic::{self, AtomicU64, AtomicUsize},
};
use std::sync::atomic::{self, AtomicU64, AtomicUsize};

use arena::{Arena, BasicStats};
use bytes_ext::Bytes;
Expand Down Expand Up @@ -153,12 +150,14 @@ impl<A: Arena<Stats = BasicStats> + Clone + Sync + Send + 'static> MemTable
}

fn approximate_memory_usage(&self) -> usize {
// Mem size of skiplist is u32, need to cast to usize
match self.skiplist.mem_size().try_into() {
Ok(v) => v,
// The skiplist already use bytes larger than usize
Err(_) => usize::MAX,
}
let encoded_size = self
.metrics
.row_encoded_size
.load(atomic::Ordering::Relaxed);
let arena_block_size = self.skiplist.arena_block_size();

// Ceil to block_size
(encoded_size + arena_block_size - 1) / arena_block_size * arena_block_size
}

fn set_last_sequence(&self, sequence: SequenceNumber) -> Result<()> {
Expand Down
3 changes: 3 additions & 0 deletions components/arena/src/arena_trait.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ pub trait Arena {
/// Get arena's statistics.
fn stats(&self) -> Self::Stats;

/// Get arena's block size.
fn block_size(&self) -> usize;

// provided methods

/// Allocate required memory. Panic if failed.
Expand Down
15 changes: 11 additions & 4 deletions components/arena/src/mono_inc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,21 +38,24 @@ const DEFAULT_ALIGN: usize = 8;
#[derive(Clone)]
pub struct MonoIncArena {
core: Arc<RwLock<ArenaCore>>,
block_size: usize,
}

impl MonoIncArena {
pub fn new(regular_block_size: usize) -> Self {
pub fn new(block_size: usize) -> Self {
Self {
core: Arc::new(RwLock::new(ArenaCore::new(
regular_block_size,
block_size,
Arc::new(NoopCollector {}),
))),
block_size,
}
}

pub fn with_collector(regular_block_size: usize, collector: CollectorRef) -> Self {
pub fn with_collector(block_size: usize, collector: CollectorRef) -> Self {
Self {
core: Arc::new(RwLock::new(ArenaCore::new(regular_block_size, collector))),
core: Arc::new(RwLock::new(ArenaCore::new(block_size, collector))),
block_size,
}
}
}
Expand All @@ -71,6 +74,10 @@ impl Arena for MonoIncArena {
fn alloc(&self, layout: Layout) -> NonNull<u8> {
self.core.write().unwrap().alloc(layout)
}

fn block_size(&self) -> usize {
self.block_size
}
}

struct ArenaCore {
Expand Down
4 changes: 4 additions & 0 deletions components/skiplist/src/list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,10 @@ impl<C, A: Arena<Stats = BasicStats> + Clone> Skiplist<C, A> {
fn height(&self) -> usize {
self.core.height.load(Ordering::SeqCst)
}

pub fn arena_block_size(&self) -> usize {
self.core.arena.block_size()
}
}

impl<C: KeyComparator, A: Arena<Stats = BasicStats> + Clone> Skiplist<C, A> {
Expand Down