Skip to content

Commit 58b0f9d

Browse files
siyingsiying
committed
Support purging logs from separate log directory
Summary: 1. Support purging info logs from a separate paths from DB path. Refactor the codes of generating info log prefixes so that it can be called when generating new files and scanning log directory. 2. Fix the bug of not scanning multiple DB paths (should only impact multiple DB paths) Test Plan: Add unit test for generating and parsing info log files Add end-to-end test in db_test Reviewers: yhchiang, ljin Reviewed By: ljin Subscribers: leveldb, igor, dhruba Differential Revision: https://reviews.facebook.net/D21801
1 parent 2da53b1 commit 58b0f9d

File tree

5 files changed

+172
-57
lines changed

5 files changed

+172
-57
lines changed

‎db/db_impl.cc

+19-9
Original file line numberDiff line numberDiff line change
@@ -585,7 +585,8 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state,
585585
// set of all files in the directory. We'll exclude files that are still
586586
// alive in the subsequent processings.
587587
std::vector<std::string> files;
588-
env_->GetChildren(dbname_, &files); // Ignore errors
588+
env_->GetChildren(options_.db_paths[path_id].path,
589+
&files); // Ignore errors
589590
for (std::string file : files) {
590591
deletion_state.candidate_files.emplace_back(file, path_id);
591592
}
@@ -599,6 +600,14 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state,
599600
deletion_state.candidate_files.emplace_back(log_file, 0);
600601
}
601602
}
603+
// Add info log files in db_log_dir
604+
if (!options_.db_log_dir.empty() && options_.db_log_dir != dbname_) {
605+
std::vector<std::string> info_log_files;
606+
env_->GetChildren(options_.db_log_dir, &info_log_files); // Ignore errors
607+
for (std::string log_file : info_log_files) {
608+
deletion_state.candidate_files.emplace_back(log_file, 0);
609+
}
610+
}
602611
}
603612
}
604613

@@ -665,14 +674,14 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
665674
candidate_files.end());
666675

667676
std::vector<std::string> old_info_log_files;
668-
677+
InfoLogPrefix info_log_prefix(!options_.db_log_dir.empty(), dbname_);
669678
for (const auto& candidate_file : candidate_files) {
670679
std::string to_delete = candidate_file.file_name;
671680
uint32_t path_id = candidate_file.path_id;
672681
uint64_t number;
673682
FileType type;
674683
// Ignore file if we cannot recognize it.
675-
if (!ParseFileName(to_delete, &number, &type)) {
684+
if (!ParseFileName(to_delete, &number, info_log_prefix.prefix, &type)) {
676685
continue;
677686
}
678687

@@ -747,16 +756,17 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
747756

748757
// Delete old info log files.
749758
size_t old_info_log_file_count = old_info_log_files.size();
750-
// NOTE: Currently we only support log purge when options_.db_log_dir is
751-
// located in `dbname` directory.
752-
if (old_info_log_file_count >= options_.keep_log_file_num &&
753-
options_.db_log_dir.empty()) {
759+
if (old_info_log_file_count >= options_.keep_log_file_num) {
754760
std::sort(old_info_log_files.begin(), old_info_log_files.end());
755761
size_t end = old_info_log_file_count - options_.keep_log_file_num;
756762
for (unsigned int i = 0; i <= end; i++) {
757763
std::string& to_delete = old_info_log_files.at(i);
758-
Log(options_.info_log, "Delete info log file %s\n", to_delete.c_str());
759-
Status s = env_->DeleteFile(dbname_ + "/" + to_delete);
764+
std::string full_path_to_delete =
765+
(options_.db_log_dir.empty() ? dbname_ : options_.db_log_dir) + "/" +
766+
to_delete;
767+
Log(options_.info_log, "Delete info log file %s\n",
768+
full_path_to_delete.c_str());
769+
Status s = env_->DeleteFile(full_path_to_delete);
760770
if (!s.ok()) {
761771
Log(options_.info_log, "Delete info log file %s FAILED -- %s\n",
762772
to_delete.c_str(), s.ToString().c_str());

‎db/db_test.cc

+31
Original file line numberDiff line numberDiff line change
@@ -6054,6 +6054,37 @@ TEST(DBTest, WALArchivalSizeLimit) {
60546054
} while (ChangeCompactOptions());
60556055
}
60566056

6057+
TEST(DBTest, PurgeInfoLogs) {
6058+
Options options = CurrentOptions();
6059+
options.keep_log_file_num = 5;
6060+
options.create_if_missing = true;
6061+
for (int mode = 0; mode <= 1; mode++) {
6062+
if (mode == 1) {
6063+
options.db_log_dir = dbname_ + "_logs";
6064+
env_->CreateDirIfMissing(options.db_log_dir);
6065+
} else {
6066+
options.db_log_dir = "";
6067+
}
6068+
for (int i = 0; i < 8; i++) {
6069+
Reopen(&options);
6070+
}
6071+
6072+
std::vector<std::string> files;
6073+
env_->GetChildren(options.db_log_dir.empty() ? dbname_ : options.db_log_dir,
6074+
&files);
6075+
int info_log_count = 0;
6076+
for (std::string file : files) {
6077+
if (file.find("LOG") != std::string::npos) {
6078+
if (mode == 1) {
6079+
env_->DeleteFile(options.db_log_dir + "/" + file);
6080+
}
6081+
info_log_count++;
6082+
}
6083+
}
6084+
ASSERT_EQ(5, info_log_count);
6085+
}
6086+
}
6087+
60576088
namespace {
60586089
SequenceNumber ReadRecords(
60596090
std::unique_ptr<TransactionLogIterator>& iter,

‎db/filename.cc

+52-27
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,16 @@
2020
namespace rocksdb {
2121

2222
// Given a path, flatten the path name by replacing all chars not in
23-
// {[0-9,a-z,A-Z,-,_,.]} with _. And append '\0' at the end.
23+
// {[0-9,a-z,A-Z,-,_,.]} with _. And append '_LOG\0' at the end.
2424
// Return the number of chars stored in dest not including the trailing '\0'.
25-
static int FlattenPath(const std::string& path, char* dest, int len) {
26-
int write_idx = 0;
27-
int i = 0;
28-
int src_len = path.size();
25+
static size_t GetInfoLogPrefix(const std::string& path, char* dest, int len) {
26+
const char suffix[] = "_LOG";
2927

30-
while (i < src_len && write_idx < len - 1) {
28+
size_t write_idx = 0;
29+
size_t i = 0;
30+
size_t src_len = path.size();
31+
32+
while (i < src_len && write_idx < len - sizeof(suffix)) {
3133
if ((path[i] >= 'a' && path[i] <= 'z') ||
3234
(path[i] >= '0' && path[i] <= '9') ||
3335
(path[i] >= 'A' && path[i] <= 'Z') ||
@@ -41,8 +43,10 @@ static int FlattenPath(const std::string& path, char* dest, int len) {
4143
}
4244
i++;
4345
}
44-
45-
dest[write_idx] = '\0';
46+
assert(sizeof(suffix) <= len - write_idx);
47+
// "\0" is automatically added by snprintf
48+
snprintf(dest + write_idx, len - write_idx, suffix);
49+
write_idx += sizeof(suffix) - 1;
4650
return write_idx;
4751
}
4852

@@ -118,14 +122,26 @@ std::string TempFileName(const std::string& dbname, uint64_t number) {
118122
return MakeFileName(dbname, number, "dbtmp");
119123
}
120124

125+
InfoLogPrefix::InfoLogPrefix(bool has_log_dir,
126+
const std::string& db_absolute_path) {
127+
if (!has_log_dir) {
128+
const char kInfoLogPrefix[] = "LOG";
129+
// "\0" is automatically added to the end
130+
snprintf(buf, sizeof(buf), kInfoLogPrefix);
131+
prefix = Slice(buf, sizeof(kInfoLogPrefix) - 1);
132+
} else {
133+
size_t len = GetInfoLogPrefix(db_absolute_path, buf, sizeof(buf));
134+
prefix = Slice(buf, len);
135+
}
136+
}
137+
121138
std::string InfoLogFileName(const std::string& dbname,
122139
const std::string& db_path, const std::string& log_dir) {
123140
if (log_dir.empty())
124141
return dbname + "/LOG";
125142

126-
char flatten_db_path[256];
127-
FlattenPath(db_path, flatten_db_path, 256);
128-
return log_dir + "/" + flatten_db_path + "_LOG";
143+
InfoLogPrefix info_log_prefix(true, db_path);
144+
return log_dir + "/" + info_log_prefix.buf;
129145
}
130146

131147
// Return the name of the old info log file for "dbname".
@@ -137,9 +153,8 @@ std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
137153
if (log_dir.empty())
138154
return dbname + "/LOG.old." + buf;
139155

140-
char flatten_db_path[256];
141-
FlattenPath(db_path, flatten_db_path, 256);
142-
return log_dir + "/" + flatten_db_path + "_LOG.old." + buf;
156+
InfoLogPrefix info_log_prefix(true, db_path);
157+
return log_dir + "/" + info_log_prefix.buf + ".old." + buf;
143158
}
144159

145160
std::string MetaDatabaseName(const std::string& dbname, uint64_t number) {
@@ -157,8 +172,8 @@ std::string IdentityFileName(const std::string& dbname) {
157172
// dbname/IDENTITY
158173
// dbname/CURRENT
159174
// dbname/LOCK
160-
// dbname/LOG
161-
// dbname/LOG.old.[0-9]+
175+
// dbname/<info_log_name_prefix>
176+
// dbname/<info_log_name_prefix>.old.[0-9]+
162177
// dbname/MANIFEST-[0-9]+
163178
// dbname/[0-9]+.(log|sst)
164179
// dbname/METADB-[0-9]+
@@ -167,6 +182,12 @@ bool ParseFileName(const std::string& fname,
167182
uint64_t* number,
168183
FileType* type,
169184
WalFileType* log_type) {
185+
return ParseFileName(fname, number, "", type, log_type);
186+
}
187+
188+
bool ParseFileName(const std::string& fname, uint64_t* number,
189+
const Slice& info_log_name_prefix, FileType* type,
190+
WalFileType* log_type) {
170191
Slice rest(fname);
171192
if (fname.length() > 1 && fname[0] == '/') {
172193
rest.remove_prefix(1);
@@ -180,18 +201,22 @@ bool ParseFileName(const std::string& fname,
180201
} else if (rest == "LOCK") {
181202
*number = 0;
182203
*type = kDBLockFile;
183-
} else if (rest == "LOG" || rest == "LOG.old") {
184-
*number = 0;
185-
*type = kInfoLogFile;
186-
} else if (rest.starts_with("LOG.old.")) {
187-
uint64_t ts_suffix;
188-
// sizeof also counts the trailing '\0'.
189-
rest.remove_prefix(sizeof("LOG.old.") - 1);
190-
if (!ConsumeDecimalNumber(&rest, &ts_suffix)) {
191-
return false;
204+
} else if (info_log_name_prefix.size() > 0 &&
205+
rest.starts_with(info_log_name_prefix)) {
206+
rest.remove_prefix(info_log_name_prefix.size());
207+
if (rest == "" || rest == ".old") {
208+
*number = 0;
209+
*type = kInfoLogFile;
210+
} else if (rest.starts_with(".old.")) {
211+
uint64_t ts_suffix;
212+
// sizeof also counts the trailing '\0'.
213+
rest.remove_prefix(sizeof(".old.") - 1);
214+
if (!ConsumeDecimalNumber(&rest, &ts_suffix)) {
215+
return false;
216+
}
217+
*number = ts_suffix;
218+
*type = kInfoLogFile;
192219
}
193-
*number = ts_suffix;
194-
*type = kInfoLogFile;
195220
} else if (rest.starts_with("MANIFEST-")) {
196221
rest.remove_prefix(strlen("MANIFEST-"));
197222
uint64_t num;

‎db/filename.h

+16-3
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,16 @@ extern std::string LockFileName(const std::string& dbname);
8686
// The result will be prefixed with "dbname".
8787
extern std::string TempFileName(const std::string& dbname, uint64_t number);
8888

89+
// A helper structure for prefix of info log names.
90+
struct InfoLogPrefix {
91+
char buf[260];
92+
Slice prefix;
93+
// Prefix with DB absolute path encoded
94+
explicit InfoLogPrefix(bool has_log_dir, const std::string& db_absolute_path);
95+
// Default Prefix
96+
explicit InfoLogPrefix();
97+
};
98+
8999
// Return the name of the info log file for "dbname".
90100
extern std::string InfoLogFileName(const std::string& dbname,
91101
const std::string& db_path="", const std::string& log_dir="");
@@ -107,10 +117,13 @@ extern std::string IdentityFileName(const std::string& dbname);
107117
// If filename is a rocksdb file, store the type of the file in *type.
108118
// The number encoded in the filename is stored in *number. If the
109119
// filename was successfully parsed, returns true. Else return false.
110-
extern bool ParseFileName(const std::string& filename,
111-
uint64_t* number,
112-
FileType* type,
120+
// info_log_name_prefix is the path of info logs.
121+
extern bool ParseFileName(const std::string& filename, uint64_t* number,
122+
const Slice& info_log_name_prefix, FileType* type,
113123
WalFileType* log_type = nullptr);
124+
// Same as previous function, but skip info log files.
125+
extern bool ParseFileName(const std::string& filename, uint64_t* number,
126+
FileType* type, WalFileType* log_type = nullptr);
114127

115128
// Make the CURRENT file point to the descriptor file with the
116129
// specified number.

‎db/filename_test.cc

+54-18
Original file line numberDiff line numberDiff line change
@@ -23,30 +23,50 @@ TEST(FileNameTest, Parse) {
2323
FileType type;
2424
uint64_t number;
2525

26+
char kDefautInfoLogDir = 1;
27+
char kDifferentInfoLogDir = 2;
28+
char kNoCheckLogDir = 4;
29+
char kAllMode = kDefautInfoLogDir | kDifferentInfoLogDir | kNoCheckLogDir;
30+
2631
// Successful parses
2732
static struct {
2833
const char* fname;
2934
uint64_t number;
3035
FileType type;
36+
char mode;
3137
} cases[] = {
32-
{ "100.log", 100, kLogFile },
33-
{ "0.log", 0, kLogFile },
34-
{ "0.sst", 0, kTableFile },
35-
{ "CURRENT", 0, kCurrentFile },
36-
{ "LOCK", 0, kDBLockFile },
37-
{ "MANIFEST-2", 2, kDescriptorFile },
38-
{ "MANIFEST-7", 7, kDescriptorFile },
39-
{ "METADB-2", 2, kMetaDatabase },
40-
{ "METADB-7", 7, kMetaDatabase },
41-
{ "LOG", 0, kInfoLogFile },
42-
{ "LOG.old", 0, kInfoLogFile },
43-
{ "18446744073709551615.log", 18446744073709551615ull, kLogFile },
44-
};
45-
for (unsigned int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
46-
std::string f = cases[i].fname;
47-
ASSERT_TRUE(ParseFileName(f, &number, &type)) << f;
48-
ASSERT_EQ(cases[i].type, type) << f;
49-
ASSERT_EQ(cases[i].number, number) << f;
38+
{"100.log", 100, kLogFile, kAllMode},
39+
{"0.log", 0, kLogFile, kAllMode},
40+
{"0.sst", 0, kTableFile, kAllMode},
41+
{"CURRENT", 0, kCurrentFile, kAllMode},
42+
{"LOCK", 0, kDBLockFile, kAllMode},
43+
{"MANIFEST-2", 2, kDescriptorFile, kAllMode},
44+
{"MANIFEST-7", 7, kDescriptorFile, kAllMode},
45+
{"METADB-2", 2, kMetaDatabase, kAllMode},
46+
{"METADB-7", 7, kMetaDatabase, kAllMode},
47+
{"LOG", 0, kInfoLogFile, kDefautInfoLogDir},
48+
{"LOG.old", 0, kInfoLogFile, kDefautInfoLogDir},
49+
{"LOG.old.6688", 6688, kInfoLogFile, kDefautInfoLogDir},
50+
{"rocksdb_dir_LOG", 0, kInfoLogFile, kDifferentInfoLogDir},
51+
{"rocksdb_dir_LOG.old", 0, kInfoLogFile, kDifferentInfoLogDir},
52+
{"rocksdb_dir_LOG.old.6688", 6688, kInfoLogFile, kDifferentInfoLogDir},
53+
{"18446744073709551615.log", 18446744073709551615ull, kLogFile,
54+
kAllMode}, };
55+
for (char mode : {kDifferentInfoLogDir, kDefautInfoLogDir, kNoCheckLogDir}) {
56+
for (unsigned int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
57+
InfoLogPrefix info_log_prefix(mode != kDefautInfoLogDir, "/rocksdb/dir");
58+
if (cases[i].mode & mode) {
59+
std::string f = cases[i].fname;
60+
if (mode == kNoCheckLogDir) {
61+
ASSERT_TRUE(ParseFileName(f, &number, &type)) << f;
62+
} else {
63+
ASSERT_TRUE(ParseFileName(f, &number, info_log_prefix.prefix, &type))
64+
<< f;
65+
}
66+
ASSERT_EQ(cases[i].type, type) << f;
67+
ASSERT_EQ(cases[i].number, number) << f;
68+
}
69+
}
5070
}
5171

5272
// Errors
@@ -85,6 +105,22 @@ TEST(FileNameTest, Parse) {
85105
};
86106
}
87107

108+
TEST(FileNameTest, InfoLogFileName) {
109+
std::string dbname = ("/data/rocksdb");
110+
std::string db_absolute_path;
111+
Env::Default()->GetAbsolutePath(dbname, &db_absolute_path);
112+
113+
ASSERT_EQ("/data/rocksdb/LOG", InfoLogFileName(dbname, db_absolute_path, ""));
114+
ASSERT_EQ("/data/rocksdb/LOG.old.666",
115+
OldInfoLogFileName(dbname, 666u, db_absolute_path, ""));
116+
117+
ASSERT_EQ("/data/rocksdb_log/data_rocksdb_LOG",
118+
InfoLogFileName(dbname, db_absolute_path, "/data/rocksdb_log"));
119+
ASSERT_EQ(
120+
"/data/rocksdb_log/data_rocksdb_LOG.old.666",
121+
OldInfoLogFileName(dbname, 666u, db_absolute_path, "/data/rocksdb_log"));
122+
}
123+
88124
TEST(FileNameTest, Construction) {
89125
uint64_t number;
90126
FileType type;

0 commit comments

Comments
 (0)