Skip to content

Commit e20fa3f

Browse files
igorcanadiigorcanadi
committed
Merge branch 'master' into columnfamilies
Conflicts: db/db_impl.cc db/internal_stats.cc db/internal_stats.h db/version_set.cc
2 parents 159928d + fcd5c5e commit e20fa3f

14 files changed

+562
-322
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,4 @@ build_tools/VALGRIND_LOGS/
2222
coverage/COVERAGE_REPORT
2323
.gdbhistory
2424
.phutil_module_cache
25+
tags

‎db/compaction.cc

+2-3
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ static void FileSizeSummary(unsigned long long sz, char* output, int len) {
213213

214214
static int InputSummary(std::vector<FileMetaData*>& files, char* output,
215215
int len) {
216+
*output = '\0';
216217
int write = 0;
217218
for (unsigned int i = 0; i < files.size(); i++) {
218219
int sz = len - write;
@@ -249,9 +250,7 @@ void Compaction::Summary(char* output, int len) {
249250
return;
250251
}
251252

252-
if (inputs_[1].size()) {
253-
write += InputSummary(inputs_[1], output+write, len-write);
254-
}
253+
write += InputSummary(inputs_[1], output+write, len-write);
255254
if (write < 0 || write >= len) {
256255
return;
257256
}

‎db/compaction_picker.cc

+11-1
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,11 @@ bool CompactionPicker::ExpandWhileOverlapping(Compaction* c) {
178178
// If, after the expansion, there are files that are already under
179179
// compaction, then we must drop/cancel this compaction.
180180
int parent_index = -1;
181-
if (FilesInCompaction(c->inputs_[0]) ||
181+
if (c->inputs_[0].empty()) {
182+
Log(options_->info_log,
183+
"ExpandWhileOverlapping() failure because zero input files");
184+
}
185+
if (c->inputs_[0].empty() || FilesInCompaction(c->inputs_[0]) ||
182186
(c->level() != c->output_level() &&
183187
ParentRangeInCompaction(c->input_version_, &smallest, &largest, level,
184188
&parent_index))) {
@@ -369,6 +373,12 @@ Compaction* LevelCompactionPicker::PickCompaction(Version* version,
369373
Compaction* c = nullptr;
370374
int level = -1;
371375

376+
// Compute the compactions needed. It is better to do it here
377+
// and also in LogAndApply(), otherwise the values could be stale.
378+
std::vector<uint64_t> size_being_compacted(NumberLevels() - 1);
379+
SizeBeingCompacted(size_being_compacted);
380+
version->ComputeCompactionScore(size_being_compacted);
381+
372382
// We prefer compactions triggered by too much data in a level over
373383
// the compactions triggered by seeks.
374384
//

‎db/db_impl.cc

+21-8
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
#include "db/db_impl.h"
1111

1212
#define __STDC_FORMAT_MACROS
13-
1413
#include <inttypes.h>
1514
#include <algorithm>
1615
#include <climits>
@@ -1711,8 +1710,10 @@ Status DBImpl::WaitForFlushMemTable(ColumnFamilyData* cfd) {
17111710
return s;
17121711
}
17131712

1714-
Status DBImpl::TEST_FlushMemTable() {
1715-
return FlushMemTable(default_cf_handle_->cfd(), FlushOptions());
1713+
Status DBImpl::TEST_FlushMemTable(bool wait) {
1714+
FlushOptions fo;
1715+
fo.wait = wait;
1716+
return FlushMemTable(default_cf_handle_->cfd(), fo);
17161717
}
17171718

17181719
Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
@@ -1851,10 +1852,15 @@ void DBImpl::BackgroundCallFlush() {
18511852
// case this is an environmental problem and we do not want to
18521853
// chew up resources for failed compactions for the duration of
18531854
// the problem.
1855+
uint64_t error_cnt = default_cf_handle_->cfd()
1856+
->internal_stats()
1857+
->BumpAndGetBackgroundErrorCount();
18541858
bg_cv_.SignalAll(); // In case a waiter can proceed despite the error
1855-
Log(options_.info_log, "Waiting after background flush error: %s",
1856-
s.ToString().c_str());
18571859
mutex_.Unlock();
1860+
Log(options_.info_log,
1861+
"Waiting after background flush error: %s"
1862+
"Accumulated background error counts: %" PRIu64,
1863+
s.ToString().c_str(), error_cnt);
18581864
log_buffer.FlushBufferToLog();
18591865
LogFlush(options_.info_log);
18601866
env_->SleepForMicroseconds(1000000);
@@ -1925,11 +1931,16 @@ void DBImpl::BackgroundCallCompaction() {
19251931
// case this is an environmental problem and we do not want to
19261932
// chew up resources for failed compactions for the duration of
19271933
// the problem.
1934+
uint64_t error_cnt = default_cf_handle_->cfd()
1935+
->internal_stats()
1936+
->BumpAndGetBackgroundErrorCount();
19281937
bg_cv_.SignalAll(); // In case a waiter can proceed despite the error
19291938
mutex_.Unlock();
19301939
log_buffer.FlushBufferToLog();
1931-
Log(options_.info_log, "Waiting after background compaction error: %s",
1932-
s.ToString().c_str());
1940+
Log(options_.info_log,
1941+
"Waiting after background compaction error: %s, "
1942+
"Accumulated background error counts: %" PRIu64,
1943+
s.ToString().c_str(), error_cnt);
19331944
LogFlush(options_.info_log);
19341945
env_->SleepForMicroseconds(1000000);
19351946
mutex_.Lock();
@@ -3820,8 +3831,10 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
38203831
value->clear();
38213832
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
38223833
auto cfd = cfh->cfd();
3834+
DBPropertyType property_type = GetPropertyType(property);
38233835
MutexLock l(&mutex_);
3824-
return cfd->internal_stats()->GetProperty(property, value, cfd);
3836+
return cfd->internal_stats()->GetProperty(property_type, property, value,
3837+
cfd);
38253838
}
38263839

38273840
void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,

‎db/db_impl.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ class DBImpl : public DB {
143143
ColumnFamilyHandle* column_family = nullptr);
144144

145145
// Force current memtable contents to be flushed.
146-
Status TEST_FlushMemTable();
146+
Status TEST_FlushMemTable(bool wait = true);
147147

148148
// Wait for memtable compaction
149149
Status TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family = nullptr);

‎db/db_test.cc

+128
Original file line numberDiff line numberDiff line change
@@ -2206,6 +2206,92 @@ TEST(DBTest, NumImmutableMemTable) {
22062206
} while (ChangeCompactOptions());
22072207
}
22082208

2209+
class SleepingBackgroundTask {
2210+
public:
2211+
SleepingBackgroundTask() : bg_cv_(&mutex_), should_sleep_(true) {}
2212+
void DoSleep() {
2213+
MutexLock l(&mutex_);
2214+
while (should_sleep_) {
2215+
bg_cv_.Wait();
2216+
}
2217+
}
2218+
void WakeUp() {
2219+
MutexLock l(&mutex_);
2220+
should_sleep_ = false;
2221+
bg_cv_.SignalAll();
2222+
}
2223+
2224+
static void DoSleepTask(void* arg) {
2225+
reinterpret_cast<SleepingBackgroundTask*>(arg)->DoSleep();
2226+
}
2227+
2228+
private:
2229+
port::Mutex mutex_;
2230+
port::CondVar bg_cv_; // Signalled when background work finishes
2231+
bool should_sleep_;
2232+
};
2233+
2234+
TEST(DBTest, GetProperty) {
2235+
// Set sizes to both background thread pool to be 1 and block them.
2236+
env_->SetBackgroundThreads(1, Env::HIGH);
2237+
env_->SetBackgroundThreads(1, Env::LOW);
2238+
SleepingBackgroundTask sleeping_task_low;
2239+
env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
2240+
Env::Priority::LOW);
2241+
SleepingBackgroundTask sleeping_task_high;
2242+
env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task_high,
2243+
Env::Priority::HIGH);
2244+
2245+
Options options = CurrentOptions();
2246+
WriteOptions writeOpt = WriteOptions();
2247+
writeOpt.disableWAL = true;
2248+
options.compaction_style = kCompactionStyleUniversal;
2249+
options.level0_file_num_compaction_trigger = 1;
2250+
options.compaction_options_universal.size_ratio = 50;
2251+
options.max_background_compactions = 1;
2252+
options.max_background_flushes = 1;
2253+
options.max_write_buffer_number = 10;
2254+
options.min_write_buffer_number_to_merge = 1;
2255+
options.write_buffer_size = 1000000;
2256+
Reopen(&options);
2257+
2258+
std::string big_value(1000000 * 2, 'x');
2259+
std::string num;
2260+
SetPerfLevel(kEnableTime);
2261+
2262+
ASSERT_OK(dbfull()->Put(writeOpt, "k1", big_value));
2263+
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
2264+
ASSERT_EQ(num, "0");
2265+
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
2266+
ASSERT_EQ(num, "0");
2267+
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
2268+
ASSERT_EQ(num, "0");
2269+
perf_context.Reset();
2270+
2271+
ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value));
2272+
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
2273+
ASSERT_EQ(num, "1");
2274+
ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value));
2275+
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
2276+
ASSERT_EQ(num, "2");
2277+
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
2278+
ASSERT_EQ(num, "1");
2279+
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
2280+
ASSERT_EQ(num, "0");
2281+
2282+
sleeping_task_high.WakeUp();
2283+
dbfull()->TEST_WaitForFlushMemTable();
2284+
2285+
ASSERT_OK(dbfull()->Put(writeOpt, "k4", big_value));
2286+
ASSERT_OK(dbfull()->Put(writeOpt, "k5", big_value));
2287+
dbfull()->TEST_WaitForFlushMemTable();
2288+
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
2289+
ASSERT_EQ(num, "0");
2290+
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
2291+
ASSERT_EQ(num, "1");
2292+
sleeping_task_low.WakeUp();
2293+
}
2294+
22092295
TEST(DBTest, FLUSH) {
22102296
do {
22112297
CreateAndReopenWithCF({"pikachu"});
@@ -4286,6 +4372,11 @@ TEST(DBTest, NoSpace) {
42864372
dbfull()->TEST_CompactRange(level, nullptr, nullptr);
42874373
}
42884374
}
4375+
4376+
std::string property_value;
4377+
ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value));
4378+
ASSERT_EQ("5", property_value);
4379+
42894380
env_->no_space_.Release_Store(nullptr);
42904381
ASSERT_LT(CountFiles(), num_files + 3);
42914382

@@ -4294,6 +4385,43 @@ TEST(DBTest, NoSpace) {
42944385
} while (ChangeCompactOptions());
42954386
}
42964387

4388+
// Check background error counter bumped on flush failures.
4389+
TEST(DBTest, NoSpaceFlush) {
4390+
do {
4391+
Options options = CurrentOptions();
4392+
options.env = env_;
4393+
options.max_background_flushes = 1;
4394+
Reopen(&options);
4395+
4396+
ASSERT_OK(Put("foo", "v1"));
4397+
env_->no_space_.Release_Store(env_); // Force out-of-space errors
4398+
4399+
std::string property_value;
4400+
// Background error count is 0 now.
4401+
ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value));
4402+
ASSERT_EQ("0", property_value);
4403+
4404+
dbfull()->TEST_FlushMemTable(false);
4405+
4406+
// Wait 300 milliseconds or background-errors turned 1 from 0.
4407+
int time_to_sleep_limit = 300000;
4408+
while (time_to_sleep_limit > 0) {
4409+
int to_sleep = (time_to_sleep_limit > 1000) ? 1000 : time_to_sleep_limit;
4410+
time_to_sleep_limit -= to_sleep;
4411+
env_->SleepForMicroseconds(to_sleep);
4412+
4413+
ASSERT_TRUE(
4414+
db_->GetProperty("rocksdb.background-errors", &property_value));
4415+
if (property_value == "1") {
4416+
break;
4417+
}
4418+
}
4419+
ASSERT_EQ("1", property_value);
4420+
4421+
env_->no_space_.Release_Store(nullptr);
4422+
} while (ChangeCompactOptions());
4423+
}
4424+
42974425
TEST(DBTest, NonWritableFileSystem) {
42984426
do {
42994427
Options options = CurrentOptions();

0 commit comments

Comments
 (0)