Skip to content
This repository has been archived by the owner on Jan 22, 2025. It is now read-only.

[wip] Add rocksdb's kv dump subcommand #26330

Closed
wants to merge 1 commit into from
Closed

[wip] Add rocksdb's kv dump subcommand #26330

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ledger-tool/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,10 @@ fn output_slot(
verbose_level: u64,
all_program_ids: &mut HashMap<Pubkey, u64>,
) -> Result<(), String> {
for (key, value) in blockstore.dump(slot) {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

obviously, this isn't good to merge as-is.

Ideally, we should add new ledger-tool subcommand like this:

solana-ledger-tool rocksdb-dump --slot NNNN --column-family CF

println!("{} {}", key.iter().map(|k| format!("{:02X}", k)).join(""), value.iter().map(|v| format!("{:02X}", v)).join(""));
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally, this should be matched to sst_dump

this pr's output:

00000000000000000000000000000000 15C369BE58D00A57BB4486AC19C228E68761BDB54F02656FF2DB117B9AB105CCF0343DFA931138B0DBA7E2FCD823880666BEC8E315B593167D06A81586CC720BA50000000000000000000000002E6A00000000000000730440000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D
710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A
00000000000000000000000000000001 B1AB8EDACCA615E51906C839ADDD81F33F6289B6F21EB3A9875A285FC9C24C3058906CC0E75A76FD9F6778500F52DB074EA32F39580C0128D6E779A80B98010BA50000000000000000010000002E6A0000000000000073043E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710E7E69ED04A3E866793A700000000000000000000000000000000C424C1E165A2FE409E0D6A15CFC1021EAE979282D710

sst_dump:

'000000000841543600000000000000A0' seq:400178018, type:1 => 50DBC8268E0EEAFA970F695B33F15B64BAF1F60873EDC195A6F507B430C9D4B1C987B4DE68A9F191F9B124981169D15A1897E5D4D39DBFA4E87ABD8BAD68480EA53654410800000000A00000000C83A00000000100037304B7620000000001572BCC03029D1962E6A12C2A922F868249B6B988627E25ECA54D3649598C2C6FDF3EBB5E9898848ABDD105E725C42D47088CE61D77E82E24B775591C94F08E0A010001033221A191210482CD561F7593E37BFEFDE4651A4CC752FAD07F25FD9572D09403CA549D9EDD75F98C6489ADBA06AFCF8B740B92D8D98ECA72772032F52F2CC2A40761481D357474BB7C4D7624EBD3BDB3D8355E73D11043FC0DA3538000000000BEFB8F9D18A7D18AAA3E3837387C964E7CE8BF59735A2C3181782420F79BA4A101020201009A03080000001D00000000000000D6534108000000001F000000D7534108000000001E000000D9534108000000001D000000DA534108000000001C000000DB534108000000001B000000DC534108000000001A000000EC5341080000000019000000ED5341080000000018000000EE5341080000000017000000EF5341080000000016000000F05341080000000015000000F15341080000000014000000F8534108000000001300000008544108000000001200000009544108000000001100000010544108000000001000000011544108000000000F00000012544108000000000E00000013544108000000000D00000014544108000000000C00000015544108000000000B0000001C544108000000000A0000001D544108000000000900000024544108000000000800000025544108000000000700000026544108000000000600000027544108000000000500000028544108000000000400000034544108000000000100000001D5534108000000008E4ED6F93B3B1C37A3E266C7D50A8B77009C6CC7C5A65B5E52DABD77F4DD1EA301F326B76200000000014EF31CC949267C995545A115597597B163E53D2E4742F57BECD3AB1FA69D7BEEBF30DEEBB73BD907D55C802EF11A25F0A5212387F79B019E2EFB882766B92B0D010001033940D575B267151E4D4E47DD3524CD3D6B8234524297867D6290E4B9B8A373EF241D4213B50996C0D790ABE7F2175BD0166C874CB8DF88CA5D35DE7BF6886BB70761481D357474BB7C4D7624EBD3BDB3D8355E73D11043FC0DA3538000000000BEFB8F9D18A7D18AAA3E3837387C964E7CE8BF59735A2C3181782420F79BA4A101020201009A03080000001D00000000000000D6534108000000001F000000D7534108000000001E000000D9534108000000001D000000DA534108000000001C000000DB534108000000001B000000DC534108000000001A000000EC5341080000000019000000ED5341080000000018000000EE5341080000000017000000EF5341080000000016000000F05341080000000015000000F15341080000000014000000F8534108000000001300000008544108000000001200000009544108000000001100000010544108000000001000000011544108000000000F00000012544108000000000E0000
'000000000841543600000000000000A1' seq:400178015, type:1 => 399E73DD8EF046524AC0BA55674923138207C57343123E5FDED05FCB81103777E328B629DBB6515E76F53F860B9FBC5C305345AD2F6C4E2411CCAB858B2C2D0AA53654410800000000A10000000C83A000000001000373040013544108000000000D00000014544108000000000C00000015544108000000000B0000001C544108000000000A0000001D544108000000000900000024544108000000000800000025544108000000000700000026544108000000000600000027544108000000000500000028544108000000000400000034544108000000000100000001D5534108000000008E4ED6F93B3B1C37A3E266C7D50A8B77009C6CC7C5A65B5E52DABD77F4DD1EA301F326B76200000000012CDDC2A4A3FCE585546A3489717827D936B83264B87F0249196A7E11C2504277242C6944FF5E8CD529BC82FBB26BC45111E49424D13970CA43C37D139C18EC06010001036E84D2E3E54FF3E3C05C3E4643E0F7030D96E91F89ADB34451F69E2C50615CF1BE0947C3303E94400D86BF80AE7E0A3500127A6C2E9076B1B228FBEB90601D0B0761481D357474BB7C4D7624EBD3BDB3D8355E73D11043FC0DA3538000000000BEFB8F9D18A7D18AAA3E3837387C964E7CE8BF59735A2C3181782420F79BA4A101020201009A03080000001D00000000000000D6534108000000001F000000D7534108000000001E000000D9534108000000001D000000DA534108000000001C000000DB534108000000001B000000DC534108000000001A000000EC5341080000000019000000ED5341080000000018000000EE5341080000000017000000EF5341080000000016000000F05341080000000015000000F15341080000000014000000F8534108000000001300000008544108000000001200000009544108000000001100000010544108000000001000000011544108000000000F00000012544108000000000E00000013544108000000000D00000014544108000000000C00000015544108000000000B0000001C544108000000000A0000001D544108000000000900000024544108000000000800000025544108000000000700000026544108000000000600000027544108000000000500000028544108000000000400000034544108000000000100000001D5534108000000008E4ED6F93B3B1C37A3E266C7D50A8B77009C6CC7C5A65B5E52DABD77F4DD1EA301F326B762000000000177C2D3CE4815167D8313BD832649E2132800F3A665B2E687D036023C23F0D7E1F8EF11B6778181FB9CCA072EE0F96F3DA33D3E07237DAD7DCB53E660E4B0BA00010001039CBC46C9BC80020BA5AE0F1329B044F2A929C3B4C589DBE0809A564C1AB3C648C573350CFD7D601FEB6B359DCB9085B0B3CE16E81D15F23450DB0C6E965B3FA10761481D357474BB7C4D7624EBD3BDB3D8355E73D11043FC0DA3538000000000BEFB8F9D18A7D18AAA3E3837387C964E7CE8BF59735A2C3181782420F79BA4A101020201009A03080000001D00000000000000CD534108000000001F000000D4534108000000001E000000D5534108000000001D000000D653

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm, my memory is vague. but iirc, there was some way to adjust sst_dump's output to match to this pr's format....

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or, i just awk-ed the sst_dump's output... xD

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

anyway, the basic idea is that using the same format makes detecting bitrot a piece of case by diff

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the below is the extended diff result using the same files with #9009 (comment):

image

}

if blockstore.is_dead(slot) {
if allow_dead_slots {
if *method == LedgerOutputMethod::Print {
Expand Down
45 changes: 45 additions & 0 deletions ledger/src/blockstore.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1844,6 +1844,33 @@ impl Blockstore {
missing_indexes
}

fn dump_slot<C>(
db_iterator: &mut DBRawIterator,
slot: Slot,
) -> Vec<(Vec<u8>, Vec<u8>)>
where
C: Column<Index = (u64, u64)>,
{
db_iterator.seek(&C::key((slot, 0)));
let mut key_value_pairs = vec![];

loop {
if !db_iterator.valid() {
break;
}
let (current_slot, _index) = C::index(db_iterator.key().expect("Expect a valid key"));

if current_slot > slot {
break;
}
key_value_pairs.push((db_iterator.key().unwrap().to_vec(), db_iterator.value().unwrap().to_vec()));

db_iterator.next();
}

key_value_pairs
}

pub fn find_missing_data_indexes(
&self,
slot: Slot,
Expand All @@ -1869,6 +1896,24 @@ impl Blockstore {
}
}

pub fn dump(
&self,
slot: Slot,
) -> Vec<(Vec<u8>, Vec<u8>)> {
if let Ok(mut db_iterator) = self
.db
.raw_iterator_cf(self.db.cf_handle::<cf::ShredData>())
{
Self::dump_slot::<cf::ShredData>(
&mut db_iterator,
slot,
)
} else {
vec![]
}
}


pub fn get_block_time(&self, slot: Slot) -> Result<Option<UnixTimestamp>> {
datapoint_info!("blockstore-rpc-api", ("method", "get_block_time", String));
let _lock = self.check_lowest_cleanup_slot(slot)?;
Expand Down