Skip to content

Commit 90729f8

Browse files
liukailiukai
committed
Extract metaindex block from block-based table
Summary: This change will allow other table to reuse the code for meta blocks. Test Plan: all existing unit tests passed Reviewers: dhruba, haobo, sdong CC: leveldb Differential Revision: https://reviews.facebook.net/D14475
1 parent e1d92df commit 90729f8

11 files changed

+432
-293
lines changed

‎db/table_properties_collector.cc

+5-86
Original file line numberDiff line numberDiff line change
@@ -10,87 +10,6 @@
1010

1111
namespace rocksdb {
1212

13-
namespace {
14-
void AppendProperty(
15-
std::string& props,
16-
const std::string& key,
17-
const std::string& value,
18-
const std::string& prop_delim,
19-
const std::string& kv_delim) {
20-
props.append(key);
21-
props.append(kv_delim);
22-
props.append(value);
23-
props.append(prop_delim);
24-
}
25-
26-
template <class TValue>
27-
void AppendProperty(
28-
std::string& props,
29-
const std::string& key,
30-
const TValue& value,
31-
const std::string& prop_delim,
32-
const std::string& kv_delim) {
33-
AppendProperty(
34-
props, key, std::to_string(value), prop_delim, kv_delim
35-
);
36-
}
37-
}
38-
39-
std::string TableProperties::ToString(
40-
const std::string& prop_delim,
41-
const std::string& kv_delim) const {
42-
std::string result;
43-
result.reserve(1024);
44-
45-
// Basic Info
46-
AppendProperty(
47-
result, "# data blocks", num_data_blocks, prop_delim, kv_delim
48-
);
49-
AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim);
50-
51-
AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim);
52-
AppendProperty(
53-
result,
54-
"raw average key size",
55-
num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0,
56-
prop_delim,
57-
kv_delim
58-
);
59-
AppendProperty(
60-
result, "raw value size", raw_value_size, prop_delim, kv_delim
61-
);
62-
AppendProperty(
63-
result,
64-
"raw average value size",
65-
num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0,
66-
prop_delim,
67-
kv_delim
68-
);
69-
70-
AppendProperty(result, "data block size", data_size, prop_delim, kv_delim);
71-
AppendProperty(result, "index block size", index_size, prop_delim, kv_delim);
72-
AppendProperty(
73-
result, "filter block size", filter_size, prop_delim, kv_delim
74-
);
75-
AppendProperty(
76-
result,
77-
"(estimated) table size",
78-
data_size + index_size + filter_size,
79-
prop_delim,
80-
kv_delim
81-
);
82-
83-
AppendProperty(
84-
result,
85-
"filter policy name",
86-
filter_policy_name.empty() ? std::string("N/A") : filter_policy_name,
87-
prop_delim,
88-
kv_delim
89-
);
90-
91-
return result;
92-
}
93-
9413
Status InternalKeyPropertiesCollector::Add(
9514
const Slice& key, const Slice& value) {
9615
ParsedInternalKey ikey;
@@ -106,7 +25,7 @@ Status InternalKeyPropertiesCollector::Add(
10625
}
10726

10827
Status InternalKeyPropertiesCollector::Finish(
109-
TableProperties::UserCollectedProperties* properties) {
28+
UserCollectedProperties* properties) {
11029
assert(properties);
11130
assert(properties->find(
11231
InternalKeyTablePropertiesNames::kDeletedKeys) == properties->end());
@@ -118,7 +37,7 @@ Status InternalKeyPropertiesCollector::Finish(
11837
return Status::OK();
11938
}
12039

121-
TableProperties::UserCollectedProperties
40+
UserCollectedProperties
12241
InternalKeyPropertiesCollector::GetReadableProperties() const {
12342
return {
12443
{ "kDeletedKeys", std::to_string(deleted_keys_) }
@@ -137,11 +56,11 @@ Status UserKeyTablePropertiesCollector::Add(
13756
}
13857

13958
Status UserKeyTablePropertiesCollector::Finish(
140-
TableProperties::UserCollectedProperties* properties) {
59+
UserCollectedProperties* properties) {
14160
return collector_->Finish(properties);
14261
}
14362

144-
TableProperties::UserCollectedProperties
63+
UserCollectedProperties
14564
UserKeyTablePropertiesCollector::GetReadableProperties() const {
14665
return collector_->GetReadableProperties();
14766
}
@@ -151,7 +70,7 @@ const std::string InternalKeyTablePropertiesNames::kDeletedKeys
15170
= "rocksdb.deleted.keys";
15271

15372
uint64_t GetDeletedKeys(
154-
const TableProperties::UserCollectedProperties& props) {
73+
const UserCollectedProperties& props) {
15574
auto pos = props.find(InternalKeyTablePropertiesNames::kDeletedKeys);
15675
if (pos == props.end()) {
15776
return 0;

‎db/table_properties_collector.h

+4-8
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,13 @@ class InternalKeyPropertiesCollector : public TablePropertiesCollector {
2424
public:
2525
virtual Status Add(const Slice& key, const Slice& value) override;
2626

27-
virtual Status Finish(
28-
TableProperties::UserCollectedProperties* properties) override;
27+
virtual Status Finish(UserCollectedProperties* properties) override;
2928

3029
virtual const char* Name() const override {
3130
return "InternalKeyPropertiesCollector";
3231
}
3332

34-
TableProperties::UserCollectedProperties
35-
GetReadableProperties() const override;
33+
UserCollectedProperties GetReadableProperties() const override;
3634

3735
private:
3836
uint64_t deleted_keys_ = 0;
@@ -61,13 +59,11 @@ class UserKeyTablePropertiesCollector : public TablePropertiesCollector {
6159

6260
virtual Status Add(const Slice& key, const Slice& value) override;
6361

64-
virtual Status Finish(
65-
TableProperties::UserCollectedProperties* properties) override;
62+
virtual Status Finish(UserCollectedProperties* properties) override;
6663

6764
virtual const char* Name() const override { return collector_->Name(); }
6865

69-
TableProperties::UserCollectedProperties
70-
GetReadableProperties() const override;
66+
UserCollectedProperties GetReadableProperties() const override;
7167

7268
protected:
7369
std::shared_ptr<TablePropertiesCollector> collector_;

‎db/table_properties_collector_test.cc

+3-4
Original file line numberDiff line numberDiff line change
@@ -114,10 +114,10 @@ class RegularKeysStartWithA: public TablePropertiesCollector {
114114
public:
115115
const char* Name() const { return "RegularKeysStartWithA"; }
116116

117-
Status Finish(TableProperties::UserCollectedProperties* properties) {
117+
Status Finish(UserCollectedProperties* properties) {
118118
std::string encoded;
119119
PutVarint32(&encoded, count_);
120-
*properties = TableProperties::UserCollectedProperties {
120+
*properties = UserCollectedProperties {
121121
{ "TablePropertiesTest", "Rocksdb" },
122122
{ "Count", encoded }
123123
};
@@ -132,8 +132,7 @@ class RegularKeysStartWithA: public TablePropertiesCollector {
132132
return Status::OK();
133133
}
134134

135-
virtual TableProperties::UserCollectedProperties
136-
GetReadableProperties() const {
135+
virtual UserCollectedProperties GetReadableProperties() const {
137136
return {};
138137
}
139138

‎include/rocksdb/options.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -611,8 +611,9 @@ struct Options {
611611
// the tables.
612612
// Default: emtpy vector -- no user-defined statistics collection will be
613613
// performed.
614-
std::vector<std::shared_ptr<TablePropertiesCollector>>
615-
table_properties_collectors;
614+
typedef std::vector<std::shared_ptr<TablePropertiesCollector>>
615+
TablePropertiesCollectors;
616+
TablePropertiesCollectors table_properties_collectors;
616617

617618
// Allows thread-safe inplace updates. Requires Updates iff
618619
// * key exists in current memtable

‎include/rocksdb/table_properties.h

+24-14
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,18 @@
1111

1212
namespace rocksdb {
1313

14+
// Other than basic table properties, each table may also have the user
15+
// collected properties.
16+
// The value of the user-collected properties are encoded as raw bytes --
17+
// users have to interprete these values by themselves.
18+
typedef
19+
std::unordered_map<std::string, std::string>
20+
UserCollectedProperties;
21+
1422
// TableProperties contains a bunch of read-only properties of its associated
1523
// table.
1624
struct TableProperties {
1725
public:
18-
// Other than basic table properties, each table may also have the user
19-
// collected properties.
20-
// The value of the user-collected properties are encoded as raw bytes --
21-
// users have to interprete these values by themselves.
22-
typedef
23-
std::unordered_map<std::string, std::string>
24-
UserCollectedProperties;
25-
2626
// the total size of all data blocks.
2727
uint64_t data_size = 0;
2828
// the size of index block.
@@ -52,6 +52,19 @@ struct TableProperties {
5252
const std::string& kv_delim = "=") const;
5353
};
5454

55+
// table properties' human-readable names in the property block.
56+
struct TablePropertiesNames {
57+
static const std::string kDataSize;
58+
static const std::string kIndexSize;
59+
static const std::string kFilterSize;
60+
static const std::string kRawKeySize;
61+
static const std::string kRawValueSize;
62+
static const std::string kNumDataBlocks;
63+
static const std::string kNumEntries;
64+
static const std::string kFilterPolicy;
65+
};
66+
67+
5568
// `TablePropertiesCollector` provides the mechanism for users to collect
5669
// their own interested properties. This class is essentially a collection
5770
// of callback functions that will be invoked during table building.
@@ -68,23 +81,20 @@ class TablePropertiesCollector {
6881
// for writing the properties block.
6982
// @params properties User will add their collected statistics to
7083
// `properties`.
71-
virtual Status Finish(
72-
TableProperties::UserCollectedProperties* properties) = 0;
84+
virtual Status Finish(UserCollectedProperties* properties) = 0;
7385

7486
// The name of the properties collector can be used for debugging purpose.
7587
virtual const char* Name() const = 0;
7688

7789
// Return the human-readable properties, where the key is property name and
7890
// the value is the human-readable form of value.
79-
virtual TableProperties::UserCollectedProperties
80-
GetReadableProperties() const = 0;
91+
virtual UserCollectedProperties GetReadableProperties() const = 0;
8192
};
8293

8394
// Extra properties
8495
// Below is a list of non-basic properties that are collected by database
8596
// itself. Especially some properties regarding to the internal keys (which
8697
// is unknown to `table`).
87-
extern uint64_t GetDeletedKeys(
88-
const TableProperties::UserCollectedProperties& props);
98+
extern uint64_t GetDeletedKeys(const UserCollectedProperties& props);
8999

90100
} // namespace rocksdb

0 commit comments

Comments
 (0)