Skip to content

Commit 3995e80

Browse files
author
Lei Jin
committed
kill ReadOptions.prefix and .prefix_seek
Summary: also add an override option total_order_iteration if you want to use full iterator with prefix_extractor Test Plan: make all check Reviewers: igor, haobo, sdong, yhchiang Reviewed By: haobo CC: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D17805
1 parent 8ce5492 commit 3995e80

32 files changed

+102
-464
lines changed

‎HISTORY.md

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
* Column family support
1010

1111
### Public API changes
12+
* Deprecated ReadOptions.prefix and ReadOptions.prefix_seek. Seek() defaults to prefix-based seek when Options.prefix_extractor is supplied. More detail is documented in https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes
1213

1314
## 2.8.0 (04/04/2014)
1415

‎db/c.cc

-11
Original file line numberDiff line numberDiff line change
@@ -1230,23 +1230,12 @@ void rocksdb_readoptions_set_fill_cache(
12301230
opt->rep.fill_cache = v;
12311231
}
12321232

1233-
void rocksdb_readoptions_set_prefix_seek(
1234-
rocksdb_readoptions_t* opt, unsigned char v) {
1235-
opt->rep.prefix_seek = v;
1236-
}
1237-
12381233
void rocksdb_readoptions_set_snapshot(
12391234
rocksdb_readoptions_t* opt,
12401235
const rocksdb_snapshot_t* snap) {
12411236
opt->rep.snapshot = (snap ? snap->rep : nullptr);
12421237
}
12431238

1244-
void rocksdb_readoptions_set_prefix(
1245-
rocksdb_readoptions_t* opt, const char* key, size_t keylen) {
1246-
Slice prefix = Slice(key, keylen);
1247-
opt->rep.prefix = &prefix;
1248-
}
1249-
12501239
void rocksdb_readoptions_set_read_tier(
12511240
rocksdb_readoptions_t* opt, int v) {
12521241
opt->rep.read_tier = static_cast<rocksdb::ReadTier>(v);

‎db/c_test.c

-2
Original file line numberDiff line numberDiff line change
@@ -461,8 +461,6 @@ int main(int argc, char** argv) {
461461
rocksdb_put(db, woptions, "bar3", 4, "bar", 3, &err);
462462
CheckNoError(err);
463463

464-
rocksdb_readoptions_set_prefix_seek(roptions, 1);
465-
466464
rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
467465
CheckCondition(!rocksdb_iter_valid(iter));
468466

‎db/db_bench.cc

-2
Original file line numberDiff line numberDiff line change
@@ -1944,7 +1944,6 @@ class Benchmark {
19441944
void IteratorCreation(ThreadState* thread) {
19451945
Duration duration(FLAGS_duration, reads_);
19461946
ReadOptions options(FLAGS_verify_checksum, true);
1947-
options.prefix_seek = (FLAGS_prefix_size > 0);
19481947
while (!duration.Done(1)) {
19491948
DB* db = SelectDB(thread);
19501949
Iterator* iter = db->NewIterator(options);
@@ -1966,7 +1965,6 @@ class Benchmark {
19661965
int64_t found = 0;
19671966
ReadOptions options(FLAGS_verify_checksum, true);
19681967
options.tailing = FLAGS_use_tailing_iterator;
1969-
options.prefix_seek = (FLAGS_prefix_size > 0);
19701968

19711969
Iterator* single_iter = nullptr;
19721970
std::vector<Iterator*> multi_iters;

‎db/db_impl.cc

+2-15
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
#include "db/memtable_list.h"
3434
#include "db/merge_context.h"
3535
#include "db/merge_helper.h"
36-
#include "db/prefix_filter_iterator.h"
3736
#include "db/table_cache.h"
3837
#include "db/table_properties_collector.h"
3938
#include "db/tailing_iter.h"
@@ -1339,7 +1338,7 @@ Status DBImpl::WriteLevel0TableForRecovery(ColumnFamilyData* cfd, MemTable* mem,
13391338
FileMetaData meta;
13401339
meta.number = versions_->NewFileNumber();
13411340
pending_outputs_.insert(meta.number);
1342-
Iterator* iter = mem->NewIterator();
1341+
Iterator* iter = mem->NewIterator(ReadOptions(), true);
13431342
const SequenceNumber newest_snapshot = snapshots_.GetNewest();
13441343
const SequenceNumber earliest_seqno_in_memtable =
13451344
mem->GetFirstSequenceNumber();
@@ -1405,7 +1404,7 @@ Status DBImpl::WriteLevel0Table(ColumnFamilyData* cfd,
14051404
for (MemTable* m : mems) {
14061405
Log(options_.info_log, "[%s] Flushing memtable with next log file: %lu\n",
14071406
cfd->GetName().c_str(), (unsigned long)m->GetNextLogNumber());
1408-
memtables.push_back(m->NewIterator());
1407+
memtables.push_back(m->NewIterator(ReadOptions(), true));
14091408
}
14101409
Iterator* iter = NewMergingIterator(&cfd->internal_comparator(),
14111410
&memtables[0], memtables.size());
@@ -3494,25 +3493,13 @@ Iterator* DBImpl::NewIterator(const ReadOptions& options,
34943493
cfd->user_comparator(), iter, snapshot);
34953494
}
34963495

3497-
if (options.prefix) {
3498-
// use extra wrapper to exclude any keys from the results which
3499-
// don't begin with the prefix
3500-
iter = new PrefixFilterIterator(iter, *options.prefix,
3501-
cfd->options()->prefix_extractor.get());
3502-
}
35033496
return iter;
35043497
}
35053498

35063499
Status DBImpl::NewIterators(
35073500
const ReadOptions& options,
35083501
const std::vector<ColumnFamilyHandle*>& column_families,
35093502
std::vector<Iterator*>* iterators) {
3510-
3511-
if (options.prefix) {
3512-
return Status::NotSupported(
3513-
"NewIterators doesn't support ReadOptions::prefix");
3514-
}
3515-
35163503
iterators->clear();
35173504
iterators->reserve(column_families.size());
35183505
SequenceNumber latest_snapshot = 0;

‎db/db_impl_debug.cc

-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ Iterator* DBImpl::TEST_NewInternalIterator(ColumnFamilyHandle* column_family) {
3333
SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
3434
mutex_.Unlock();
3535
ReadOptions roptions;
36-
roptions.prefix_seek = true;
3736
return NewInternalIterator(roptions, cfd, super_version);
3837
}
3938

‎db/db_test.cc

+40-63
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,8 @@ class DBTest {
306306
kSkipUniversalCompaction = 2,
307307
kSkipMergePut = 4,
308308
kSkipPlainTable = 8,
309-
kSkipHashIndex = 16
309+
kSkipHashIndex = 16,
310+
kSkipNoSeekToLast = 32
310311
};
311312

312313
DBTest() : option_config_(kDefault),
@@ -341,6 +342,11 @@ class DBTest {
341342
if ((skip_mask & kSkipMergePut) && option_config_ == kMergePut) {
342343
continue;
343344
}
345+
if ((skip_mask & kSkipNoSeekToLast) &&
346+
(option_config_ == kHashLinkList ||
347+
option_config_ == kHashSkipList)) {;
348+
continue;
349+
}
344350
if ((skip_mask & kSkipPlainTable)
345351
&& (option_config_ == kPlainTableAllBytesPrefix
346352
|| option_config_ == kPlainTableFirstBytePrefix)) {
@@ -862,10 +868,11 @@ class DBTest {
862868

863869
void VerifyIterLast(std::string expected_key, int cf = 0) {
864870
Iterator* iter;
871+
ReadOptions ro;
865872
if (cf == 0) {
866-
iter = db_->NewIterator(ReadOptions());
873+
iter = db_->NewIterator(ro);
867874
} else {
868-
iter = db_->NewIterator(ReadOptions(), handles_[cf]);
875+
iter = db_->NewIterator(ro, handles_[cf]);
869876
}
870877
iter->SeekToLast();
871878
ASSERT_EQ(IterStatus(iter), expected_key);
@@ -1463,7 +1470,7 @@ TEST(DBTest, NonBlockingIteration) {
14631470

14641471
// This test verifies block cache behaviors, which is not used by plain
14651472
// table format.
1466-
} while (ChangeOptions(kSkipPlainTable));
1473+
} while (ChangeOptions(kSkipPlainTable | kSkipNoSeekToLast));
14671474
}
14681475

14691476
// A delete is skipped for key if KeyMayExist(key) returns False
@@ -1907,19 +1914,23 @@ TEST(DBTest, IterSmallAndLargeMix) {
19071914
TEST(DBTest, IterMultiWithDelete) {
19081915
do {
19091916
CreateAndReopenWithCF({"pikachu"});
1910-
ASSERT_OK(Put(1, "a", "va"));
1911-
ASSERT_OK(Put(1, "b", "vb"));
1912-
ASSERT_OK(Put(1, "c", "vc"));
1913-
ASSERT_OK(Delete(1, "b"));
1914-
ASSERT_EQ("NOT_FOUND", Get(1, "b"));
1917+
ASSERT_OK(Put(1, "ka", "va"));
1918+
ASSERT_OK(Put(1, "kb", "vb"));
1919+
ASSERT_OK(Put(1, "kc", "vc"));
1920+
ASSERT_OK(Delete(1, "kb"));
1921+
ASSERT_EQ("NOT_FOUND", Get(1, "kb"));
19151922

19161923
Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
1917-
iter->Seek("c");
1918-
ASSERT_EQ(IterStatus(iter), "c->vc");
1924+
iter->Seek("kc");
1925+
ASSERT_EQ(IterStatus(iter), "kc->vc");
19191926
if (!CurrentOptions().merge_operator) {
19201927
// TODO: merge operator does not support backward iteration yet
1921-
iter->Prev();
1922-
ASSERT_EQ(IterStatus(iter), "a->va");
1928+
if (kPlainTableAllBytesPrefix != option_config_&&
1929+
kBlockBasedTableWithWholeKeyHashIndex != option_config_ &&
1930+
kHashLinkList != option_config_) {
1931+
iter->Prev();
1932+
ASSERT_EQ(IterStatus(iter), "ka->va");
1933+
}
19231934
}
19241935
delete iter;
19251936
} while (ChangeOptions());
@@ -1952,7 +1963,7 @@ TEST(DBTest, IterPrevMaxSkip) {
19521963

19531964
ASSERT_OK(Delete(1, "key1"));
19541965
VerifyIterLast("(invalid)", 1);
1955-
} while (ChangeOptions(kSkipMergePut));
1966+
} while (ChangeOptions(kSkipMergePut | kSkipNoSeekToLast));
19561967
}
19571968

19581969
TEST(DBTest, IterWithSnapshot) {
@@ -1977,15 +1988,19 @@ TEST(DBTest, IterWithSnapshot) {
19771988
ASSERT_EQ(IterStatus(iter), "key5->val5");
19781989
if (!CurrentOptions().merge_operator) {
19791990
// TODO: merge operator does not support backward iteration yet
1980-
iter->Prev();
1981-
ASSERT_EQ(IterStatus(iter), "key4->val4");
1982-
iter->Prev();
1983-
ASSERT_EQ(IterStatus(iter), "key3->val3");
1991+
if (kPlainTableAllBytesPrefix != option_config_&&
1992+
kBlockBasedTableWithWholeKeyHashIndex != option_config_ &&
1993+
kHashLinkList != option_config_) {
1994+
iter->Prev();
1995+
ASSERT_EQ(IterStatus(iter), "key4->val4");
1996+
iter->Prev();
1997+
ASSERT_EQ(IterStatus(iter), "key3->val3");
19841998

1985-
iter->Next();
1986-
ASSERT_EQ(IterStatus(iter), "key4->val4");
1987-
iter->Next();
1988-
ASSERT_EQ(IterStatus(iter), "key5->val5");
1999+
iter->Next();
2000+
ASSERT_EQ(IterStatus(iter), "key4->val4");
2001+
iter->Next();
2002+
ASSERT_EQ(IterStatus(iter), "key5->val5");
2003+
}
19892004
iter->Next();
19902005
ASSERT_TRUE(!iter->Valid());
19912006
}
@@ -5944,7 +5959,7 @@ TEST(DBTest, GroupCommitTest) {
59445959
ASSERT_TRUE(!itr->Valid());
59455960
delete itr;
59465961

5947-
} while (ChangeOptions());
5962+
} while (ChangeOptions(kSkipNoSeekToLast));
59485963
}
59495964

59505965
namespace {
@@ -6313,7 +6328,7 @@ TEST(DBTest, Randomized) {
63136328
}
63146329
if (model_snap != nullptr) model.ReleaseSnapshot(model_snap);
63156330
if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap);
6316-
} while (ChangeOptions(kSkipDeletesFilterFirst));
6331+
} while (ChangeOptions(kSkipDeletesFilterFirst | kSkipNoSeekToLast));
63176332
}
63186333

63196334
TEST(DBTest, MultiGetSimple) {
@@ -6429,7 +6444,6 @@ void PrefixScanInit(DBTest *dbtest) {
64296444
} // namespace
64306445

64316446
TEST(DBTest, PrefixScan) {
6432-
ReadOptions ro = ReadOptions();
64336447
int count;
64346448
Slice prefix;
64356449
Slice key;
@@ -6450,45 +6464,9 @@ TEST(DBTest, PrefixScan) {
64506464
options.max_background_compactions = 2;
64516465
options.create_if_missing = true;
64526466
options.disable_seek_compaction = true;
6453-
// Tricky: options.prefix_extractor will be released by
6454-
// NewHashSkipListRepFactory after use.
64556467
options.memtable_factory.reset(NewHashSkipListRepFactory());
64566468

6457-
// prefix specified, with blooms: 2 RAND I/Os
6458-
// SeekToFirst
6459-
DestroyAndReopen(&options);
6460-
PrefixScanInit(this);
6461-
count = 0;
6462-
env_->random_read_counter_.Reset();
6463-
ro.prefix = &prefix;
6464-
iter = db_->NewIterator(ro);
6465-
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
6466-
assert(iter->key().starts_with(prefix));
6467-
count++;
6468-
}
6469-
ASSERT_OK(iter->status());
6470-
delete iter;
6471-
ASSERT_EQ(count, 2);
6472-
ASSERT_EQ(env_->random_read_counter_.Read(), 2);
6473-
6474-
// prefix specified, with blooms: 2 RAND I/Os
6475-
// Seek
6476-
DestroyAndReopen(&options);
6477-
PrefixScanInit(this);
6478-
count = 0;
6479-
env_->random_read_counter_.Reset();
6480-
ro.prefix = &prefix;
6481-
iter = db_->NewIterator(ro);
6482-
for (iter->Seek(key); iter->Valid(); iter->Next()) {
6483-
assert(iter->key().starts_with(prefix));
6484-
count++;
6485-
}
6486-
ASSERT_OK(iter->status());
6487-
delete iter;
6488-
ASSERT_EQ(count, 2);
6489-
ASSERT_EQ(env_->random_read_counter_.Read(), 2);
6490-
6491-
// no prefix specified: 11 RAND I/Os
6469+
// 11 RAND I/Os
64926470
DestroyAndReopen(&options);
64936471
PrefixScanInit(this);
64946472
count = 0;
@@ -6652,7 +6630,6 @@ TEST(DBTest, TailingIteratorDeletes) {
66526630
TEST(DBTest, TailingIteratorPrefixSeek) {
66536631
ReadOptions read_options;
66546632
read_options.tailing = true;
6655-
read_options.prefix_seek = true;
66566633

66576634
Options options = CurrentOptions();
66586635
options.env = env_;

‎db/memtable.cc

+6-6
Original file line numberDiff line numberDiff line change
@@ -159,13 +159,12 @@ const char* EncodeKey(std::string* scratch, const Slice& target) {
159159

160160
class MemTableIterator: public Iterator {
161161
public:
162-
MemTableIterator(const MemTable& mem, const ReadOptions& options)
162+
MemTableIterator(const MemTable& mem, const ReadOptions& options,
163+
bool enforce_total_order)
163164
: bloom_(nullptr),
164165
prefix_extractor_(mem.prefix_extractor_),
165166
valid_(false) {
166-
if (options.prefix) {
167-
iter_.reset(mem.table_->GetPrefixIterator(*options.prefix));
168-
} else if (options.prefix_seek) {
167+
if (prefix_extractor_ != nullptr && !enforce_total_order) {
169168
bloom_ = mem.prefix_bloom_.get();
170169
iter_.reset(mem.table_->GetDynamicPrefixIterator());
171170
} else {
@@ -224,8 +223,9 @@ class MemTableIterator: public Iterator {
224223
void operator=(const MemTableIterator&);
225224
};
226225

227-
Iterator* MemTable::NewIterator(const ReadOptions& options) {
228-
return new MemTableIterator(*this, options);
226+
Iterator* MemTable::NewIterator(const ReadOptions& options,
227+
bool enforce_total_order) {
228+
return new MemTableIterator(*this, options, enforce_total_order);
229229
}
230230

231231
port::RWMutex* MemTable::GetLock(const Slice& key) {

‎db/memtable.h

+4-8
Original file line numberDiff line numberDiff line change
@@ -75,14 +75,10 @@ class MemTable {
7575
// iterator are internal keys encoded by AppendInternalKey in the
7676
// db/dbformat.{h,cc} module.
7777
//
78-
// If options.prefix is supplied, it is passed to the underlying MemTableRep
79-
// as a hint that the iterator only need to support access to keys with that
80-
// specific prefix.
81-
// If options.prefix is not supplied and options.prefix_seek is set, the
82-
// iterator is not bound to a specific prefix. However, the semantics of
83-
// Seek is changed - the result might only include keys with the same prefix
84-
// as the seek-key.
85-
Iterator* NewIterator(const ReadOptions& options = ReadOptions());
78+
// By default, it returns an iterator for prefix seek if prefix_extractor
79+
// is configured in Options.
80+
Iterator* NewIterator(const ReadOptions& options,
81+
bool enforce_total_order = false);
8682

8783
// Add an entry into memtable that maps key to value at the
8884
// specified sequence number and with the specified type.

0 commit comments

Comments
 (0)