Skip to content

Commit 0143abd

Browse files
igorcanadiigorcanadi
committed
Merge branch 'master' into columnfamilies
Conflicts: HISTORY.md db/db_impl.cc db/db_impl.h db/db_iter.cc db/db_test.cc db/dbformat.h db/memtable.cc db/memtable_list.cc db/memtable_list.h db/table_cache.cc db/table_cache.h db/version_edit.h db/version_set.cc db/version_set.h db/write_batch.cc db/write_batch_test.cc include/rocksdb/options.h util/options.cc
2 parents 0b4ccf7 + 4564b2e commit 0143abd

File tree

104 files changed

+6208
-2341
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

104 files changed

+6208
-2341
lines changed

‎HISTORY.md

+5-1
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,12 @@
66
executed in high priority thread pool.
77

88
## Unreleased (will be relased in 2.8)
9-
* By default, checksums are verified on every read from database
9+
## Unreleased
10+
11+
### Public API changes
1012

13+
* Removed arena.h from public header files.
14+
* By default, checksums are verified on every read from database
1115

1216
## 2.7.0 (01/28/2014)
1317

‎Makefile

+13-8
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,7 @@
66
INSTALL_PATH ?= $(CURDIR)
77

88
#-----------------------------------------------
9-
# Uncomment exactly one of the lines labelled (A), (B), and (C) below
10-
# to switch between compilation modes.
11-
12-
# OPT ?= -DNDEBUG # (A) Production use (optimized mode)
13-
OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
9+
OPT += -fno-omit-frame-pointer -momit-leaf-frame-pointer
1410
#-----------------------------------------------
1511

1612
# detect what platform we're building on
@@ -57,6 +53,7 @@ TESTS = \
5753
auto_roll_logger_test \
5854
block_test \
5955
bloom_test \
56+
dynamic_bloom_test \
6057
c_test \
6158
cache_test \
6259
coding_test \
@@ -75,6 +72,7 @@ TESTS = \
7572
merge_test \
7673
redis_test \
7774
reduce_levels_test \
75+
plain_table_db_test \
7876
simple_table_db_test \
7977
skiplist_test \
8078
stringappend_test \
@@ -93,6 +91,7 @@ TOOLS = \
9391
db_repl_stress \
9492
blob_store_bench
9593

94+
9695
PROGRAMS = db_bench signal_test $(TESTS) $(TOOLS)
9796
BENCHMARKS = db_bench_sqlite3 db_bench_tree_db table_reader_bench
9897

@@ -143,11 +142,11 @@ all: $(LIBRARY) $(PROGRAMS)
143142
# Will also generate shared libraries.
144143
release:
145144
$(MAKE) clean
146-
OPT=-DNDEBUG $(MAKE) all -j32
145+
OPT="-DNDEBUG -O2" $(MAKE) all -j32
147146

148147
coverage:
149148
$(MAKE) clean
150-
COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) all check
149+
COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) all check -j32
151150
(cd coverage; ./coverage_test.sh)
152151
# Delete intermediate files
153152
find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
@@ -248,6 +247,9 @@ table_properties_collector_test: db/table_properties_collector_test.o $(LIBOBJEC
248247
bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
249248
$(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
250249

250+
dynamic_bloom_test: util/dynamic_bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
251+
$(CXX) util/dynamic_bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
252+
251253
c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS)
252254
$(CXX) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
253255

@@ -278,11 +280,14 @@ crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
278280
db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS)
279281
$(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
280282

283+
plain_table_db_test: db/plain_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
284+
$(CXX) db/plain_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
285+
281286
simple_table_db_test: db/simple_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
282287
$(CXX) db/simple_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
283288

284289
table_reader_bench: table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS)
285-
$(CXX) table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
290+
$(CXX) table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) -pg
286291

287292
perf_context_test: db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
288293
$(CXX) db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)

‎build_tools/format-diff.sh

-2
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,13 @@ fi
4747
# ln -s `git rev-parse --show-toplevel`/build_tools/format-diff.sh $PRE_COMMIT_SCRIPT_PATH
4848
# fi
4949
# fi
50-
5150
set -e
5251

5352
uncommitted_code=`git diff HEAD`
5453

5554
# If there's no uncommitted changes, we assume user are doing post-commit
5655
# format check, in which case we'll check the modified lines from latest commit.
5756
# Otherwise, we'll check format of the uncommitted code only.
58-
format_last_commit=0
5957
if [ -z "$uncommitted_code" ]
6058
then
6159
# Check the format of last commit

‎coverage/coverage_test.sh

+5
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
4444
tee -a $RECENT_REPORT &&
4545
echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"
4646

47+
# Unless otherwise specified, we'll not generate html report by default
48+
if [ -z "$HTML" ]; then
49+
exit 0
50+
fi
51+
4752
# Generate the html report. If we cannot find lcov in this machine, we'll simply
4853
# skip this step.
4954
echo "Generating the html coverage report..."

‎db/builder.cc

+19-23
Original file line numberDiff line numberDiff line change
@@ -9,37 +9,35 @@
99

1010
#include "db/builder.h"
1111

12-
#include "db/filename.h"
1312
#include "db/dbformat.h"
13+
#include "db/filename.h"
1414
#include "db/merge_helper.h"
1515
#include "db/table_cache.h"
1616
#include "db/version_edit.h"
1717
#include "rocksdb/db.h"
18-
#include "rocksdb/table.h"
1918
#include "rocksdb/env.h"
2019
#include "rocksdb/iterator.h"
2120
#include "rocksdb/options.h"
21+
#include "rocksdb/table.h"
2222
#include "table/block_based_table_builder.h"
2323
#include "util/stop_watch.h"
2424

2525
namespace rocksdb {
2626

2727
class TableFactory;
2828

29-
TableBuilder* GetTableBuilder(const Options& options, WritableFile* file,
29+
TableBuilder* NewTableBuilder(const Options& options,
30+
const InternalKeyComparator& internal_comparator,
31+
WritableFile* file,
3032
CompressionType compression_type) {
31-
return options.table_factory->GetTableBuilder(options, file,
32-
compression_type);
33+
return options.table_factory->NewTableBuilder(options, internal_comparator,
34+
file, compression_type);
3335
}
3436

35-
Status BuildTable(const std::string& dbname,
36-
Env* env,
37-
const Options& options,
38-
const EnvOptions& soptions,
39-
TableCache* table_cache,
40-
Iterator* iter,
41-
FileMetaData* meta,
42-
const Comparator* user_comparator,
37+
Status BuildTable(const std::string& dbname, Env* env, const Options& options,
38+
const EnvOptions& soptions, TableCache* table_cache,
39+
Iterator* iter, FileMetaData* meta,
40+
const InternalKeyComparator& internal_comparator,
4341
const SequenceNumber newest_snapshot,
4442
const SequenceNumber earliest_seqno_in_memtable,
4543
const CompressionType compression) {
@@ -64,17 +62,17 @@ Status BuildTable(const std::string& dbname,
6462
return s;
6563
}
6664

67-
TableBuilder* builder = GetTableBuilder(options, file.get(),
68-
compression);
65+
TableBuilder* builder =
66+
NewTableBuilder(options, internal_comparator, file.get(), compression);
6967

7068
// the first key is the smallest key
7169
Slice key = iter->key();
7270
meta->smallest.DecodeFrom(key);
7371
meta->smallest_seqno = GetInternalKeySeqno(key);
7472
meta->largest_seqno = meta->smallest_seqno;
7573

76-
MergeHelper merge(user_comparator, options.merge_operator.get(),
77-
options.info_log.get(),
74+
MergeHelper merge(internal_comparator.user_comparator(),
75+
options.merge_operator.get(), options.info_log.get(),
7876
true /* internal key corruption is not ok */);
7977

8078
if (purge) {
@@ -103,8 +101,8 @@ Status BuildTable(const std::string& dbname,
103101
// If the key is the same as the previous key (and it is not the
104102
// first key), then we skip it, since it is an older version.
105103
// Otherwise we output the key and mark it as the "new" previous key.
106-
if (!is_first_key && !user_comparator->Compare(prev_ikey.user_key,
107-
this_ikey.user_key)) {
104+
if (!is_first_key && !internal_comparator.user_comparator()->Compare(
105+
prev_ikey.user_key, this_ikey.user_key)) {
108106
// seqno within the same key are in decreasing order
109107
assert(this_ikey.sequence < prev_ikey.sequence);
110108
} else {
@@ -202,10 +200,8 @@ Status BuildTable(const std::string& dbname,
202200

203201
if (s.ok()) {
204202
// Verify that the table is usable
205-
Iterator* it = table_cache->NewIterator(ReadOptions(),
206-
soptions,
207-
meta->number,
208-
meta->file_size);
203+
Iterator* it = table_cache->NewIterator(ReadOptions(), soptions,
204+
internal_comparator, *meta);
209205
s = it->status();
210206
delete it;
211207
}

‎db/builder.h

+7-10
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,20 @@ class VersionEdit;
2424
class TableBuilder;
2525
class WritableFile;
2626

27-
28-
extern TableBuilder* GetTableBuilder(const Options& options, WritableFile* file,
29-
CompressionType compression_type);
27+
extern TableBuilder* NewTableBuilder(
28+
const Options& options, const InternalKeyComparator& internal_comparator,
29+
WritableFile* file, CompressionType compression_type);
3030

3131
// Build a Table file from the contents of *iter. The generated file
3232
// will be named according to meta->number. On success, the rest of
3333
// *meta will be filled with metadata about the generated table.
3434
// If no data is present in *iter, meta->file_size will be set to
3535
// zero, and no Table file will be produced.
36-
extern Status BuildTable(const std::string& dbname,
37-
Env* env,
38-
const Options& options,
39-
const EnvOptions& soptions,
40-
TableCache* table_cache,
41-
Iterator* iter,
36+
extern Status BuildTable(const std::string& dbname, Env* env,
37+
const Options& options, const EnvOptions& soptions,
38+
TableCache* table_cache, Iterator* iter,
4239
FileMetaData* meta,
43-
const Comparator* user_comparator,
40+
const InternalKeyComparator& internal_comparator,
4441
const SequenceNumber newest_snapshot,
4542
const SequenceNumber earliest_seqno_in_memtable,
4643
const CompressionType compression);

‎db/column_family.cc

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include "db/internal_stats.h"
1818
#include "db/compaction_picker.h"
1919
#include "db/table_properties_collector.h"
20+
#include "util/autovector.h"
2021
#include "util/hash_skiplist_rep.h"
2122

2223
namespace rocksdb {
@@ -184,7 +185,7 @@ ColumnFamilyData::~ColumnFamilyData() {
184185
if (mem_ != nullptr) {
185186
delete mem_->Unref();
186187
}
187-
std::vector<MemTable*> to_delete;
188+
autovector<MemTable*> to_delete;
188189
imm_.current()->Unref(&to_delete);
189190
for (MemTable* m : to_delete) {
190191
delete m;

‎db/column_family.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
#include "rocksdb/options.h"
1818
#include "rocksdb/env.h"
19-
#include "db/memtablelist.h"
19+
#include "db/memtable_list.h"
2020
#include "db/write_batch_internal.h"
2121
#include "db/table_cache.h"
2222

@@ -40,7 +40,7 @@ struct SuperVersion {
4040
// We need to_delete because during Cleanup(), imm->Unref() returns
4141
// all memtables that we need to free through this vector. We then
4242
// delete all those memtables outside of mutex, during destruction
43-
std::vector<MemTable*> to_delete;
43+
autovector<MemTable*> to_delete;
4444

4545
// should be called outside the mutex
4646
SuperVersion();

‎db/db_bench.cc

+7-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "rocksdb/slice.h"
2525
#include "rocksdb/slice_transform.h"
2626
#include "rocksdb/statistics.h"
27+
#include "rocksdb/perf_context.h"
2728
#include "port/port.h"
2829
#include "util/bit_set.h"
2930
#include "util/crc32c.h"
@@ -389,6 +390,8 @@ DEFINE_int64(stats_interval, 0, "Stats are reported every N operations when "
389390
DEFINE_int32(stats_per_interval, 0, "Reports additional stats per interval when"
390391
" this is greater than 0.");
391392

393+
DEFINE_int32(perf_level, 0, "Level of perf collection");
394+
392395
static bool ValidateRateLimit(const char* flagname, double value) {
393396
static constexpr double EPSILON = 1e-10;
394397
if ( value < -EPSILON ) {
@@ -728,6 +731,7 @@ struct SharedState {
728731
port::Mutex mu;
729732
port::CondVar cv;
730733
int total;
734+
int perf_level;
731735

732736
// Each thread goes through the following states:
733737
// (1) initializing
@@ -739,7 +743,7 @@ struct SharedState {
739743
long num_done;
740744
bool start;
741745

742-
SharedState() : cv(&mu) { }
746+
SharedState() : cv(&mu), perf_level(FLAGS_perf_level) { }
743747
};
744748

745749
// Per-thread state for concurrent executions of the same benchmark.
@@ -847,6 +851,7 @@ class Benchmark {
847851
fprintf(stdout, "Memtablerep: vector\n");
848852
break;
849853
}
854+
fprintf(stdout, "Perf Level: %d\n", FLAGS_perf_level);
850855

851856
PrintWarnings();
852857
fprintf(stdout, "------------------------------------------------\n");
@@ -1202,6 +1207,7 @@ class Benchmark {
12021207
}
12031208
}
12041209

1210+
SetPerfLevel(static_cast<PerfLevel> (shared->perf_level));
12051211
thread->stats.Start(thread->tid);
12061212
(arg->bm->*(arg->method))(thread);
12071213
thread->stats.Stop();

0 commit comments

Comments
 (0)