Skip to content

Commit a062e1f

Browse files
author
Lei Jin
committed
SetOptions() for memtable related options
Summary: as title Test Plan: make all check I will think a way to set up stress test for this Reviewers: sdong, yhchiang, igor Reviewed By: igor Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D23055
1 parent e4eca6a commit a062e1f

14 files changed

+219
-56
lines changed

‎db/column_family.cc

+26-6
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "db/write_controller.h"
2828
#include "util/autovector.h"
2929
#include "util/hash_skiplist_rep.h"
30+
#include "util/options_helper.h"
3031

3132
namespace rocksdb {
3233

@@ -212,7 +213,7 @@ void SuperVersionUnrefHandle(void* ptr) {
212213

213214
ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
214215
Version* dummy_versions, Cache* table_cache,
215-
const ColumnFamilyOptions& options,
216+
const ColumnFamilyOptions& cf_options,
216217
const DBOptions* db_options,
217218
const EnvOptions& env_options,
218219
ColumnFamilySet* column_family_set)
@@ -222,9 +223,10 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
222223
current_(nullptr),
223224
refs_(0),
224225
dropped_(false),
225-
internal_comparator_(options.comparator),
226-
options_(*db_options, SanitizeOptions(&internal_comparator_, options)),
226+
internal_comparator_(cf_options.comparator),
227+
options_(*db_options, SanitizeOptions(&internal_comparator_, cf_options)),
227228
ioptions_(options_),
229+
mutable_cf_options_(options_),
228230
mem_(nullptr),
229231
imm_(options_.min_write_buffer_number_to_merge),
230232
super_version_(nullptr),
@@ -378,13 +380,12 @@ const EnvOptions* ColumnFamilyData::soptions() const {
378380

379381
void ColumnFamilyData::SetCurrent(Version* current) { current_ = current; }
380382

381-
void ColumnFamilyData::CreateNewMemtable() {
383+
void ColumnFamilyData::CreateNewMemtable(const MemTableOptions& moptions) {
382384
assert(current_ != nullptr);
383385
if (mem_ != nullptr) {
384386
delete mem_->Unref();
385387
}
386-
mem_ = new MemTable(internal_comparator_, ioptions_,
387-
MemTableOptions(options_));
388+
mem_ = new MemTable(internal_comparator_, ioptions_, moptions);
388389
mem_->Ref();
389390
}
390391

@@ -486,7 +487,15 @@ bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
486487

487488
SuperVersion* ColumnFamilyData::InstallSuperVersion(
488489
SuperVersion* new_superversion, port::Mutex* db_mutex) {
490+
db_mutex->AssertHeld();
491+
return InstallSuperVersion(new_superversion, db_mutex, mutable_cf_options_);
492+
}
493+
494+
SuperVersion* ColumnFamilyData::InstallSuperVersion(
495+
SuperVersion* new_superversion, port::Mutex* db_mutex,
496+
const MutableCFOptions& mutable_cf_options) {
489497
new_superversion->db_mutex = db_mutex;
498+
new_superversion->mutable_cf_options = mutable_cf_options;
490499
new_superversion->Init(mem_, imm_.current(), current_);
491500
SuperVersion* old_superversion = super_version_;
492501
super_version_ = new_superversion;
@@ -522,6 +531,17 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
522531
}
523532
}
524533

534+
bool ColumnFamilyData::SetOptions(
535+
const std::unordered_map<std::string, std::string>& options_map) {
536+
MutableCFOptions new_mutable_cf_options;
537+
if (GetMutableOptionsFromStrings(mutable_cf_options_, options_map,
538+
&new_mutable_cf_options)) {
539+
mutable_cf_options_ = new_mutable_cf_options;
540+
return true;
541+
}
542+
return false;
543+
}
544+
525545
ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
526546
const DBOptions* db_options,
527547
const EnvOptions& env_options,

‎db/column_family.h

+22-3
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "db/table_cache.h"
2424
#include "util/thread_local.h"
2525
#include "db/flush_scheduler.h"
26+
#include "util/mutable_cf_options.h"
2627

2728
namespace rocksdb {
2829

@@ -80,6 +81,7 @@ struct SuperVersion {
8081
MemTable* mem;
8182
MemTableListVersion* imm;
8283
Version* current;
84+
MutableCFOptions mutable_cf_options;
8385
std::atomic<uint32_t> refs;
8486
// We need to_delete because during Cleanup(), imm->Unref() returns
8587
// all memtables that we need to free through this vector. We then
@@ -168,11 +170,24 @@ class ColumnFamilyData {
168170
void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
169171
uint64_t GetLogNumber() const { return log_number_; }
170172

171-
// TODO(ljin): make this API thread-safe once we allow updating options_
172-
const Options* options() const { return &options_; }
173173
// thread-safe
174+
const Options* options() const { return &options_; }
174175
const EnvOptions* soptions() const;
175176
const ImmutableCFOptions* ioptions() const { return &ioptions_; }
177+
// REQUIRES: DB mutex held
178+
// This returns the MutableCFOptions used by current SuperVersion
179+
// You shoul use this API to reference MutableCFOptions most of the time.
180+
const MutableCFOptions* mutable_cf_options() const {
181+
return &(super_version_->mutable_cf_options);
182+
}
183+
// REQUIRES: DB mutex held
184+
// This returns the latest MutableCFOptions, which may be not in effect yet.
185+
const MutableCFOptions* GetLatestMutableCFOptions() const {
186+
return &mutable_cf_options_;
187+
}
188+
// REQUIRES: DB mutex held
189+
bool SetOptions(
190+
const std::unordered_map<std::string, std::string>& options_map);
176191

177192
InternalStats* internal_stats() { return internal_stats_.get(); }
178193

@@ -182,7 +197,7 @@ class ColumnFamilyData {
182197
Version* dummy_versions() { return dummy_versions_; }
183198
void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
184199
void SetCurrent(Version* current);
185-
void CreateNewMemtable();
200+
void CreateNewMemtable(const MemTableOptions& moptions);
186201

187202
TableCache* table_cache() const { return table_cache_.get(); }
188203

@@ -223,6 +238,9 @@ class ColumnFamilyData {
223238
// if its reference count is zero and needs deletion or nullptr if not
224239
// As argument takes a pointer to allocated SuperVersion to enable
225240
// the clients to allocate SuperVersion outside of mutex.
241+
SuperVersion* InstallSuperVersion(SuperVersion* new_superversion,
242+
port::Mutex* db_mutex,
243+
const MutableCFOptions& mutable_cf_options);
226244
SuperVersion* InstallSuperVersion(SuperVersion* new_superversion,
227245
port::Mutex* db_mutex);
228246

@@ -255,6 +273,7 @@ class ColumnFamilyData {
255273

256274
const Options options_;
257275
const ImmutableCFOptions ioptions_;
276+
MutableCFOptions mutable_cf_options_;
258277

259278
std::unique_ptr<TableCache> table_cache_;
260279

‎db/db_impl.cc

+25-6
Original file line numberDiff line numberDiff line change
@@ -1228,7 +1228,8 @@ Status DBImpl::Recover(
12281228
if (!s.ok()) {
12291229
// Clear memtables if recovery failed
12301230
for (auto cfd : *versions_->GetColumnFamilySet()) {
1231-
cfd->CreateNewMemtable();
1231+
cfd->CreateNewMemtable(MemTableOptions(
1232+
*cfd->GetLatestMutableCFOptions(), *cfd->options()));
12321233
}
12331234
}
12341235
}
@@ -1356,7 +1357,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
13561357
// file-systems cause the DB::Open() to fail.
13571358
return status;
13581359
}
1359-
cfd->CreateNewMemtable();
1360+
cfd->CreateNewMemtable(MemTableOptions(
1361+
*cfd->GetLatestMutableCFOptions(), *cfd->options()));
13601362
}
13611363
}
13621364
}
@@ -1393,7 +1395,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
13931395
// Recovery failed
13941396
break;
13951397
}
1396-
cfd->CreateNewMemtable();
1398+
cfd->CreateNewMemtable(MemTableOptions(
1399+
*cfd->GetLatestMutableCFOptions(), *cfd->options()));
13971400
}
13981401

13991402
// write MANIFEST with update
@@ -1623,6 +1626,7 @@ Status DBImpl::FlushMemTableToOutputFile(ColumnFamilyData* cfd,
16231626
}
16241627

16251628
if (s.ok()) {
1629+
// Use latest MutableCFOptions
16261630
InstallSuperVersion(cfd, deletion_state);
16271631
if (madeProgress) {
16281632
*madeProgress = 1;
@@ -1714,6 +1718,13 @@ Status DBImpl::CompactRange(ColumnFamilyHandle* column_family,
17141718
return s;
17151719
}
17161720

1721+
bool DBImpl::SetOptions(ColumnFamilyHandle* column_family,
1722+
const std::unordered_map<std::string, std::string>& options_map) {
1723+
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
1724+
MutexLock l(&mutex_);
1725+
return cfh->cfd()->SetOptions(options_map);
1726+
}
1727+
17171728
// return the same level if it cannot be moved
17181729
int DBImpl::FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd, int level) {
17191730
mutex_.AssertHeld();
@@ -1784,6 +1795,7 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
17841795
cfd->GetName().c_str(), edit.DebugString().data());
17851796

17861797
status = versions_->LogAndApply(cfd, &edit, &mutex_, db_directory_.get());
1798+
// Use latest MutableCFOptions
17871799
superversion_to_free = cfd->InstallSuperVersion(new_superversion, &mutex_);
17881800
new_superversion = nullptr;
17891801

@@ -2322,6 +2334,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
23222334
}
23232335
status = versions_->LogAndApply(c->column_family_data(), c->edit(), &mutex_,
23242336
db_directory_.get());
2337+
// Use latest MutableCFOptions
23252338
InstallSuperVersion(c->column_family_data(), deletion_state);
23262339
LogToBuffer(log_buffer, "[%s] Deleted %d files\n",
23272340
c->column_family_data()->GetName().c_str(),
@@ -2338,6 +2351,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
23382351
f->smallest_seqno, f->largest_seqno);
23392352
status = versions_->LogAndApply(c->column_family_data(), c->edit(), &mutex_,
23402353
db_directory_.get());
2354+
// Use latest MutableCFOptions
23412355
InstallSuperVersion(c->column_family_data(), deletion_state);
23422356

23432357
Version::LevelSummaryStorage tmp;
@@ -3322,6 +3336,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact,
33223336

33233337
if (status.ok()) {
33243338
status = InstallCompactionResults(compact, log_buffer);
3339+
// Use latest MutableCFOptions
33253340
InstallSuperVersion(cfd, deletion_state);
33263341
}
33273342
Version::LevelSummaryStorage tmp;
@@ -3426,6 +3441,7 @@ void DBImpl::InstallSuperVersion(ColumnFamilyData* cfd,
34263441
SuperVersion* new_superversion =
34273442
(deletion_state.new_superversion != nullptr) ?
34283443
deletion_state.new_superversion : new SuperVersion();
3444+
// Use latest MutableCFOptions
34293445
SuperVersion* old_superversion =
34303446
cfd->InstallSuperVersion(new_superversion, &mutex_);
34313447
deletion_state.new_superversion = nullptr;
@@ -3618,6 +3634,7 @@ Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& options,
36183634
auto cfd =
36193635
versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name);
36203636
assert(cfd != nullptr);
3637+
// Use latest MutableCFOptions
36213638
delete cfd->InstallSuperVersion(new SuperVersion(), &mutex_);
36223639
*handle = new ColumnFamilyHandleImpl(cfd, this, &mutex_);
36233640
Log(db_options_.info_log, "Created column family [%s] (ID %u)",
@@ -4138,6 +4155,7 @@ Status DBImpl::SetNewMemtableAndNewLogFile(ColumnFamilyData* cfd,
41384155
uint64_t new_log_number =
41394156
creating_new_log ? versions_->NewFileNumber() : logfile_number_;
41404157
SuperVersion* new_superversion = nullptr;
4158+
const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
41414159
mutex_.Unlock();
41424160
Status s;
41434161
{
@@ -4156,8 +4174,8 @@ Status DBImpl::SetNewMemtableAndNewLogFile(ColumnFamilyData* cfd,
41564174

41574175
if (s.ok()) {
41584176
new_mem = new MemTable(cfd->internal_comparator(),
4159-
*cfd->ioptions(),
4160-
MemTableOptions(*cfd->options()));
4177+
*cfd->ioptions(), MemTableOptions(mutable_cf_options,
4178+
*cfd->options()));
41614179
new_superversion = new SuperVersion();
41624180
}
41634181
}
@@ -4197,7 +4215,7 @@ Status DBImpl::SetNewMemtableAndNewLogFile(ColumnFamilyData* cfd,
41974215
"[%s] New memtable created with log file: #%" PRIu64 "\n",
41984216
cfd->GetName().c_str(), logfile_number_);
41994217
context->superversions_to_free_.push_back(
4200-
cfd->InstallSuperVersion(new_superversion, &mutex_));
4218+
cfd->InstallSuperVersion(new_superversion, &mutex_, mutable_cf_options));
42014219
return s;
42024220
}
42034221

@@ -4672,6 +4690,7 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
46724690
}
46734691
if (s.ok()) {
46744692
for (auto cfd : *impl->versions_->GetColumnFamilySet()) {
4693+
// Use latest MutableCFOptions
46754694
delete cfd->InstallSuperVersion(new SuperVersion(), &impl->mutex_);
46764695
}
46774696
impl->alive_log_files_.push_back(

‎db/db_impl.h

+4
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,10 @@ class DBImpl : public DB {
112112
bool reduce_level = false, int target_level = -1,
113113
uint32_t target_path_id = 0);
114114

115+
using DB::SetOptions;
116+
bool SetOptions(ColumnFamilyHandle* column_family,
117+
const std::unordered_map<std::string, std::string>& options_map);
118+
115119
using DB::NumberLevels;
116120
virtual int NumberLevels(ColumnFamilyHandle* column_family);
117121
using DB::MaxMemCompactionLevel;

‎db/memtable.cc

+10-8
Original file line numberDiff line numberDiff line change
@@ -31,18 +31,20 @@
3131

3232
namespace rocksdb {
3333

34-
MemTableOptions::MemTableOptions(const Options& options)
35-
: write_buffer_size(options.write_buffer_size),
36-
arena_block_size(options.arena_block_size),
37-
memtable_prefix_bloom_bits(options.memtable_prefix_bloom_bits),
38-
memtable_prefix_bloom_probes(options.memtable_prefix_bloom_probes),
34+
MemTableOptions::MemTableOptions(
35+
const MutableCFOptions& mutable_cf_options, const Options& options)
36+
: write_buffer_size(mutable_cf_options.write_buffer_size),
37+
arena_block_size(mutable_cf_options.arena_block_size),
38+
memtable_prefix_bloom_bits(mutable_cf_options.memtable_prefix_bloom_bits),
39+
memtable_prefix_bloom_probes(
40+
mutable_cf_options.memtable_prefix_bloom_probes),
3941
memtable_prefix_bloom_huge_page_tlb_size(
40-
options.memtable_prefix_bloom_huge_page_tlb_size),
42+
mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size),
4143
inplace_update_support(options.inplace_update_support),
4244
inplace_update_num_locks(options.inplace_update_num_locks),
4345
inplace_callback(options.inplace_callback),
44-
max_successive_merges(options.max_successive_merges),
45-
filter_deletes(options.filter_deletes) {}
46+
max_successive_merges(mutable_cf_options.max_successive_merges),
47+
filter_deletes(mutable_cf_options.filter_deletes) {}
4648

4749
MemTable::MemTable(const InternalKeyComparator& cmp,
4850
const ImmutableCFOptions& ioptions,

‎db/memtable.h

+4-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "rocksdb/immutable_options.h"
2222
#include "util/arena.h"
2323
#include "util/dynamic_bloom.h"
24+
#include "util/mutable_cf_options.h"
2425

2526
namespace rocksdb {
2627

@@ -30,7 +31,9 @@ class MemTableIterator;
3031
class MergeContext;
3132

3233
struct MemTableOptions {
33-
explicit MemTableOptions(const Options& options);
34+
explicit MemTableOptions(
35+
const MutableCFOptions& mutable_cf_options,
36+
const Options& options);
3437
size_t write_buffer_size;
3538
size_t arena_block_size;
3639
uint32_t memtable_prefix_bloom_bits;

‎db/repair.cc

+2-1
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,8 @@ class Repairer {
219219
std::string scratch;
220220
Slice record;
221221
WriteBatch batch;
222-
MemTable* mem = new MemTable(icmp_, ioptions_, MemTableOptions(options_));
222+
MemTable* mem = new MemTable(icmp_, ioptions_,
223+
MemTableOptions(MutableCFOptions(options_), options_));
223224
auto cf_mems_default = new ColumnFamilyMemTablesDefault(mem, &options_);
224225
mem->Ref();
225226
int counter = 0;

‎db/version_set.cc

+7-3
Original file line numberDiff line numberDiff line change
@@ -2962,17 +2962,21 @@ void VersionSet::GetObsoleteFiles(std::vector<FileMetaData*>* files) {
29622962
}
29632963

29642964
ColumnFamilyData* VersionSet::CreateColumnFamily(
2965-
const ColumnFamilyOptions& options, VersionEdit* edit) {
2965+
const ColumnFamilyOptions& cf_options, VersionEdit* edit) {
29662966
assert(edit->is_column_family_add_);
29672967

29682968
Version* dummy_versions = new Version(nullptr, this);
29692969
auto new_cfd = column_family_set_->CreateColumnFamily(
2970-
edit->column_family_name_, edit->column_family_, dummy_versions, options);
2970+
edit->column_family_name_, edit->column_family_, dummy_versions,
2971+
cf_options);
29712972

29722973
Version* v = new Version(new_cfd, this, current_version_number_++);
29732974

29742975
AppendVersion(new_cfd, v);
2975-
new_cfd->CreateNewMemtable();
2976+
// GetLatestMutableCFOptions() is safe here without mutex since the
2977+
// cfd is not available to client
2978+
new_cfd->CreateNewMemtable(MemTableOptions(
2979+
*new_cfd->GetLatestMutableCFOptions(), *new_cfd->options()));
29762980
new_cfd->SetLogNumber(edit->log_number_);
29772981
return new_cfd;
29782982
}

‎db/write_batch_test.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ static std::string PrintContents(WriteBatch* b) {
2828
Options options;
2929
options.memtable_factory = factory;
3030
MemTable* mem = new MemTable(cmp, ImmutableCFOptions(options),
31-
MemTableOptions(options));
31+
MemTableOptions(MutableCFOptions(options), options));
3232
mem->Ref();
3333
std::string state;
3434
ColumnFamilyMemTablesDefault cf_mems_default(mem, &options);

0 commit comments

Comments
 (0)