1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
5 //
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
9
10 #include "db/version_set.h"
11 #include "db/db_impl/db_impl.h"
12 #include "db/log_writer.h"
13 #include "env/mock_env.h"
14 #include "logging/logging.h"
15 #include "table/mock_table.h"
16 #include "test_util/testharness.h"
17 #include "test_util/testutil.h"
18 #include "util/string_util.h"
19
20 namespace ROCKSDB_NAMESPACE {
21
22 class GenerateLevelFilesBriefTest : public testing::Test {
23 public:
24 std::vector<FileMetaData*> files_;
25 LevelFilesBrief file_level_;
26 Arena arena_;
27
GenerateLevelFilesBriefTest()28 GenerateLevelFilesBriefTest() { }
29
~GenerateLevelFilesBriefTest()30 ~GenerateLevelFilesBriefTest() override {
31 for (size_t i = 0; i < files_.size(); i++) {
32 delete files_[i];
33 }
34 }
35
Add(const char * smallest,const char * largest,SequenceNumber smallest_seq=100,SequenceNumber largest_seq=100)36 void Add(const char* smallest, const char* largest,
37 SequenceNumber smallest_seq = 100,
38 SequenceNumber largest_seq = 100) {
39 FileMetaData* f = new FileMetaData(
40 files_.size() + 1, 0, 0,
41 InternalKey(smallest, smallest_seq, kTypeValue),
42 InternalKey(largest, largest_seq, kTypeValue), smallest_seq,
43 largest_seq, /* marked_for_compact */ false, kInvalidBlobFileNumber,
44 kUnknownOldestAncesterTime, kUnknownFileCreationTime,
45 kUnknownFileChecksum, kUnknownFileChecksumFuncName);
46 files_.push_back(f);
47 }
48
Compare()49 int Compare() {
50 int diff = 0;
51 for (size_t i = 0; i < files_.size(); i++) {
52 if (file_level_.files[i].fd.GetNumber() != files_[i]->fd.GetNumber()) {
53 diff++;
54 }
55 }
56 return diff;
57 }
58 };
59
TEST_F(GenerateLevelFilesBriefTest,Empty)60 TEST_F(GenerateLevelFilesBriefTest, Empty) {
61 DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
62 ASSERT_EQ(0u, file_level_.num_files);
63 ASSERT_EQ(0, Compare());
64 }
65
TEST_F(GenerateLevelFilesBriefTest,Single)66 TEST_F(GenerateLevelFilesBriefTest, Single) {
67 Add("p", "q");
68 DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
69 ASSERT_EQ(1u, file_level_.num_files);
70 ASSERT_EQ(0, Compare());
71 }
72
TEST_F(GenerateLevelFilesBriefTest,Multiple)73 TEST_F(GenerateLevelFilesBriefTest, Multiple) {
74 Add("150", "200");
75 Add("200", "250");
76 Add("300", "350");
77 Add("400", "450");
78 DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
79 ASSERT_EQ(4u, file_level_.num_files);
80 ASSERT_EQ(0, Compare());
81 }
82
83 class CountingLogger : public Logger {
84 public:
CountingLogger()85 CountingLogger() : log_count(0) {}
86 using Logger::Logv;
Logv(const char *,va_list)87 void Logv(const char* /*format*/, va_list /*ap*/) override { log_count++; }
88 int log_count;
89 };
90
GetOptionsWithNumLevels(int num_levels,std::shared_ptr<CountingLogger> logger)91 Options GetOptionsWithNumLevels(int num_levels,
92 std::shared_ptr<CountingLogger> logger) {
93 Options opt;
94 opt.num_levels = num_levels;
95 opt.info_log = logger;
96 return opt;
97 }
98
99 class VersionStorageInfoTest : public testing::Test {
100 public:
101 const Comparator* ucmp_;
102 InternalKeyComparator icmp_;
103 std::shared_ptr<CountingLogger> logger_;
104 Options options_;
105 ImmutableCFOptions ioptions_;
106 MutableCFOptions mutable_cf_options_;
107 VersionStorageInfo vstorage_;
108
GetInternalKey(const char * ukey,SequenceNumber smallest_seq=100)109 InternalKey GetInternalKey(const char* ukey,
110 SequenceNumber smallest_seq = 100) {
111 return InternalKey(ukey, smallest_seq, kTypeValue);
112 }
113
VersionStorageInfoTest()114 VersionStorageInfoTest()
115 : ucmp_(BytewiseComparator()),
116 icmp_(ucmp_),
117 logger_(new CountingLogger()),
118 options_(GetOptionsWithNumLevels(6, logger_)),
119 ioptions_(options_),
120 mutable_cf_options_(options_),
121 vstorage_(&icmp_, ucmp_, 6, kCompactionStyleLevel, nullptr, false) {}
122
~VersionStorageInfoTest()123 ~VersionStorageInfoTest() override {
124 for (int i = 0; i < vstorage_.num_levels(); i++) {
125 for (auto* f : vstorage_.LevelFiles(i)) {
126 if (--f->refs == 0) {
127 delete f;
128 }
129 }
130 }
131 }
132
Add(int level,uint32_t file_number,const char * smallest,const char * largest,uint64_t file_size=0)133 void Add(int level, uint32_t file_number, const char* smallest,
134 const char* largest, uint64_t file_size = 0) {
135 assert(level < vstorage_.num_levels());
136 FileMetaData* f = new FileMetaData(
137 file_number, 0, file_size, GetInternalKey(smallest, 0),
138 GetInternalKey(largest, 0), /* smallest_seq */ 0, /* largest_seq */ 0,
139 /* marked_for_compact */ false, kInvalidBlobFileNumber,
140 kUnknownOldestAncesterTime, kUnknownFileCreationTime,
141 kUnknownFileChecksum, kUnknownFileChecksumFuncName);
142 f->compensated_file_size = file_size;
143 vstorage_.AddFile(level, f);
144 }
145
Add(int level,uint32_t file_number,const InternalKey & smallest,const InternalKey & largest,uint64_t file_size=0)146 void Add(int level, uint32_t file_number, const InternalKey& smallest,
147 const InternalKey& largest, uint64_t file_size = 0) {
148 assert(level < vstorage_.num_levels());
149 FileMetaData* f = new FileMetaData(
150 file_number, 0, file_size, smallest, largest, /* smallest_seq */ 0,
151 /* largest_seq */ 0, /* marked_for_compact */ false,
152 kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
153 kUnknownFileCreationTime, kUnknownFileChecksum,
154 kUnknownFileChecksumFuncName);
155 f->compensated_file_size = file_size;
156 vstorage_.AddFile(level, f);
157 }
158
GetOverlappingFiles(int level,const InternalKey & begin,const InternalKey & end)159 std::string GetOverlappingFiles(int level, const InternalKey& begin,
160 const InternalKey& end) {
161 std::vector<FileMetaData*> inputs;
162 vstorage_.GetOverlappingInputs(level, &begin, &end, &inputs);
163
164 std::string result;
165 for (size_t i = 0; i < inputs.size(); ++i) {
166 if (i > 0) {
167 result += ",";
168 }
169 AppendNumberTo(&result, inputs[i]->fd.GetNumber());
170 }
171 return result;
172 }
173 };
174
TEST_F(VersionStorageInfoTest,MaxBytesForLevelStatic)175 TEST_F(VersionStorageInfoTest, MaxBytesForLevelStatic) {
176 ioptions_.level_compaction_dynamic_level_bytes = false;
177 mutable_cf_options_.max_bytes_for_level_base = 10;
178 mutable_cf_options_.max_bytes_for_level_multiplier = 5;
179 Add(4, 100U, "1", "2");
180 Add(5, 101U, "1", "2");
181
182 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
183 ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 10U);
184 ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 50U);
185 ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 250U);
186 ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1250U);
187
188 ASSERT_EQ(0, logger_->log_count);
189 }
190
TEST_F(VersionStorageInfoTest,MaxBytesForLevelDynamic)191 TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamic) {
192 ioptions_.level_compaction_dynamic_level_bytes = true;
193 mutable_cf_options_.max_bytes_for_level_base = 1000;
194 mutable_cf_options_.max_bytes_for_level_multiplier = 5;
195 Add(5, 1U, "1", "2", 500U);
196
197 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
198 ASSERT_EQ(0, logger_->log_count);
199 ASSERT_EQ(vstorage_.base_level(), 5);
200
201 Add(5, 2U, "3", "4", 550U);
202 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
203 ASSERT_EQ(0, logger_->log_count);
204 ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1000U);
205 ASSERT_EQ(vstorage_.base_level(), 4);
206
207 Add(4, 3U, "3", "4", 550U);
208 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
209 ASSERT_EQ(0, logger_->log_count);
210 ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1000U);
211 ASSERT_EQ(vstorage_.base_level(), 4);
212
213 Add(3, 4U, "3", "4", 250U);
214 Add(3, 5U, "5", "7", 300U);
215 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
216 ASSERT_EQ(1, logger_->log_count);
217 ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 1005U);
218 ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 1000U);
219 ASSERT_EQ(vstorage_.base_level(), 3);
220
221 Add(1, 6U, "3", "4", 5U);
222 Add(1, 7U, "8", "9", 5U);
223 logger_->log_count = 0;
224 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
225 ASSERT_EQ(1, logger_->log_count);
226 ASSERT_GT(vstorage_.MaxBytesForLevel(4), 1005U);
227 ASSERT_GT(vstorage_.MaxBytesForLevel(3), 1005U);
228 ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 1005U);
229 ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 1000U);
230 ASSERT_EQ(vstorage_.base_level(), 1);
231 }
232
TEST_F(VersionStorageInfoTest,MaxBytesForLevelDynamicLotsOfData)233 TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLotsOfData) {
234 ioptions_.level_compaction_dynamic_level_bytes = true;
235 mutable_cf_options_.max_bytes_for_level_base = 100;
236 mutable_cf_options_.max_bytes_for_level_multiplier = 2;
237 Add(0, 1U, "1", "2", 50U);
238 Add(1, 2U, "1", "2", 50U);
239 Add(2, 3U, "1", "2", 500U);
240 Add(3, 4U, "1", "2", 500U);
241 Add(4, 5U, "1", "2", 1700U);
242 Add(5, 6U, "1", "2", 500U);
243
244 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
245 ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 800U);
246 ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 400U);
247 ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 200U);
248 ASSERT_EQ(vstorage_.MaxBytesForLevel(1), 100U);
249 ASSERT_EQ(vstorage_.base_level(), 1);
250 ASSERT_EQ(0, logger_->log_count);
251 }
252
TEST_F(VersionStorageInfoTest,MaxBytesForLevelDynamicLargeLevel)253 TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicLargeLevel) {
254 uint64_t kOneGB = 1000U * 1000U * 1000U;
255 ioptions_.level_compaction_dynamic_level_bytes = true;
256 mutable_cf_options_.max_bytes_for_level_base = 10U * kOneGB;
257 mutable_cf_options_.max_bytes_for_level_multiplier = 10;
258 Add(0, 1U, "1", "2", 50U);
259 Add(3, 4U, "1", "2", 32U * kOneGB);
260 Add(4, 5U, "1", "2", 500U * kOneGB);
261 Add(5, 6U, "1", "2", 3000U * kOneGB);
262
263 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
264 ASSERT_EQ(vstorage_.MaxBytesForLevel(5), 3000U * kOneGB);
265 ASSERT_EQ(vstorage_.MaxBytesForLevel(4), 300U * kOneGB);
266 ASSERT_EQ(vstorage_.MaxBytesForLevel(3), 30U * kOneGB);
267 ASSERT_EQ(vstorage_.MaxBytesForLevel(2), 10U * kOneGB);
268 ASSERT_EQ(vstorage_.base_level(), 2);
269 ASSERT_EQ(0, logger_->log_count);
270 }
271
TEST_F(VersionStorageInfoTest,MaxBytesForLevelDynamicWithLargeL0_1)272 TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicWithLargeL0_1) {
273 ioptions_.level_compaction_dynamic_level_bytes = true;
274 mutable_cf_options_.max_bytes_for_level_base = 40000;
275 mutable_cf_options_.max_bytes_for_level_multiplier = 5;
276 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
277
278 Add(0, 1U, "1", "2", 10000U);
279 Add(0, 2U, "1", "2", 10000U);
280 Add(0, 3U, "1", "2", 10000U);
281
282 Add(5, 4U, "1", "2", 1286250U);
283 Add(4, 5U, "1", "2", 200000U);
284 Add(3, 6U, "1", "2", 40000U);
285 Add(2, 7U, "1", "2", 8000U);
286
287 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
288 ASSERT_EQ(0, logger_->log_count);
289 ASSERT_EQ(2, vstorage_.base_level());
290 // level multiplier should be 3.5
291 ASSERT_EQ(vstorage_.level_multiplier(), 5.0);
292 // Level size should be around 30,000, 105,000, 367,500
293 ASSERT_EQ(40000U, vstorage_.MaxBytesForLevel(2));
294 ASSERT_EQ(51450U, vstorage_.MaxBytesForLevel(3));
295 ASSERT_EQ(257250U, vstorage_.MaxBytesForLevel(4));
296 }
297
TEST_F(VersionStorageInfoTest,MaxBytesForLevelDynamicWithLargeL0_2)298 TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicWithLargeL0_2) {
299 ioptions_.level_compaction_dynamic_level_bytes = true;
300 mutable_cf_options_.max_bytes_for_level_base = 10000;
301 mutable_cf_options_.max_bytes_for_level_multiplier = 5;
302 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
303
304 Add(0, 11U, "1", "2", 10000U);
305 Add(0, 12U, "1", "2", 10000U);
306 Add(0, 13U, "1", "2", 10000U);
307
308 Add(5, 4U, "1", "2", 1286250U);
309 Add(4, 5U, "1", "2", 200000U);
310 Add(3, 6U, "1", "2", 40000U);
311 Add(2, 7U, "1", "2", 8000U);
312
313 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
314 ASSERT_EQ(0, logger_->log_count);
315 ASSERT_EQ(2, vstorage_.base_level());
316 // level multiplier should be 3.5
317 ASSERT_LT(vstorage_.level_multiplier(), 3.6);
318 ASSERT_GT(vstorage_.level_multiplier(), 3.4);
319 // Level size should be around 30,000, 105,000, 367,500
320 ASSERT_EQ(30000U, vstorage_.MaxBytesForLevel(2));
321 ASSERT_LT(vstorage_.MaxBytesForLevel(3), 110000U);
322 ASSERT_GT(vstorage_.MaxBytesForLevel(3), 100000U);
323 ASSERT_LT(vstorage_.MaxBytesForLevel(4), 370000U);
324 ASSERT_GT(vstorage_.MaxBytesForLevel(4), 360000U);
325 }
326
TEST_F(VersionStorageInfoTest,MaxBytesForLevelDynamicWithLargeL0_3)327 TEST_F(VersionStorageInfoTest, MaxBytesForLevelDynamicWithLargeL0_3) {
328 ioptions_.level_compaction_dynamic_level_bytes = true;
329 mutable_cf_options_.max_bytes_for_level_base = 10000;
330 mutable_cf_options_.max_bytes_for_level_multiplier = 5;
331 mutable_cf_options_.level0_file_num_compaction_trigger = 2;
332
333 Add(0, 11U, "1", "2", 5000U);
334 Add(0, 12U, "1", "2", 5000U);
335 Add(0, 13U, "1", "2", 5000U);
336 Add(0, 14U, "1", "2", 5000U);
337 Add(0, 15U, "1", "2", 5000U);
338 Add(0, 16U, "1", "2", 5000U);
339
340 Add(5, 4U, "1", "2", 1286250U);
341 Add(4, 5U, "1", "2", 200000U);
342 Add(3, 6U, "1", "2", 40000U);
343 Add(2, 7U, "1", "2", 8000U);
344
345 vstorage_.CalculateBaseBytes(ioptions_, mutable_cf_options_);
346 ASSERT_EQ(0, logger_->log_count);
347 ASSERT_EQ(2, vstorage_.base_level());
348 // level multiplier should be 3.5
349 ASSERT_LT(vstorage_.level_multiplier(), 3.6);
350 ASSERT_GT(vstorage_.level_multiplier(), 3.4);
351 // Level size should be around 30,000, 105,000, 367,500
352 ASSERT_EQ(30000U, vstorage_.MaxBytesForLevel(2));
353 ASSERT_LT(vstorage_.MaxBytesForLevel(3), 110000U);
354 ASSERT_GT(vstorage_.MaxBytesForLevel(3), 100000U);
355 ASSERT_LT(vstorage_.MaxBytesForLevel(4), 370000U);
356 ASSERT_GT(vstorage_.MaxBytesForLevel(4), 360000U);
357 }
358
TEST_F(VersionStorageInfoTest,EstimateLiveDataSize)359 TEST_F(VersionStorageInfoTest, EstimateLiveDataSize) {
360 // Test whether the overlaps are detected as expected
361 Add(1, 1U, "4", "7", 1U); // Perfect overlap with last level
362 Add(2, 2U, "3", "5", 1U); // Partial overlap with last level
363 Add(2, 3U, "6", "8", 1U); // Partial overlap with last level
364 Add(3, 4U, "1", "9", 1U); // Contains range of last level
365 Add(4, 5U, "4", "5", 1U); // Inside range of last level
366 Add(4, 5U, "6", "7", 1U); // Inside range of last level
367 Add(5, 6U, "4", "7", 10U);
368 ASSERT_EQ(10U, vstorage_.EstimateLiveDataSize());
369 }
370
TEST_F(VersionStorageInfoTest,EstimateLiveDataSize2)371 TEST_F(VersionStorageInfoTest, EstimateLiveDataSize2) {
372 Add(0, 1U, "9", "9", 1U); // Level 0 is not ordered
373 Add(0, 1U, "5", "6", 1U); // Ignored because of [5,6] in l1
374 Add(1, 1U, "1", "2", 1U); // Ignored because of [2,3] in l2
375 Add(1, 2U, "3", "4", 1U); // Ignored because of [2,3] in l2
376 Add(1, 3U, "5", "6", 1U);
377 Add(2, 4U, "2", "3", 1U);
378 Add(3, 5U, "7", "8", 1U);
379 ASSERT_EQ(4U, vstorage_.EstimateLiveDataSize());
380 }
381
TEST_F(VersionStorageInfoTest,GetOverlappingInputs)382 TEST_F(VersionStorageInfoTest, GetOverlappingInputs) {
383 // Two files that overlap at the range deletion tombstone sentinel.
384 Add(1, 1U, {"a", 0, kTypeValue}, {"b", kMaxSequenceNumber, kTypeRangeDeletion}, 1);
385 Add(1, 2U, {"b", 0, kTypeValue}, {"c", 0, kTypeValue}, 1);
386 // Two files that overlap at the same user key.
387 Add(1, 3U, {"d", 0, kTypeValue}, {"e", kMaxSequenceNumber, kTypeValue}, 1);
388 Add(1, 4U, {"e", 0, kTypeValue}, {"f", 0, kTypeValue}, 1);
389 // Two files that do not overlap.
390 Add(1, 5U, {"g", 0, kTypeValue}, {"h", 0, kTypeValue}, 1);
391 Add(1, 6U, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}, 1);
392 vstorage_.UpdateNumNonEmptyLevels();
393 vstorage_.GenerateLevelFilesBrief();
394
395 ASSERT_EQ("1,2", GetOverlappingFiles(
396 1, {"a", 0, kTypeValue}, {"b", 0, kTypeValue}));
397 ASSERT_EQ("1", GetOverlappingFiles(
398 1, {"a", 0, kTypeValue}, {"b", kMaxSequenceNumber, kTypeRangeDeletion}));
399 ASSERT_EQ("2", GetOverlappingFiles(
400 1, {"b", kMaxSequenceNumber, kTypeValue}, {"c", 0, kTypeValue}));
401 ASSERT_EQ("3,4", GetOverlappingFiles(
402 1, {"d", 0, kTypeValue}, {"e", 0, kTypeValue}));
403 ASSERT_EQ("3", GetOverlappingFiles(
404 1, {"d", 0, kTypeValue}, {"e", kMaxSequenceNumber, kTypeRangeDeletion}));
405 ASSERT_EQ("3,4", GetOverlappingFiles(
406 1, {"e", kMaxSequenceNumber, kTypeValue}, {"f", 0, kTypeValue}));
407 ASSERT_EQ("3,4", GetOverlappingFiles(
408 1, {"e", 0, kTypeValue}, {"f", 0, kTypeValue}));
409 ASSERT_EQ("5", GetOverlappingFiles(
410 1, {"g", 0, kTypeValue}, {"h", 0, kTypeValue}));
411 ASSERT_EQ("6", GetOverlappingFiles(
412 1, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}));
413 }
414
415
416 class FindLevelFileTest : public testing::Test {
417 public:
418 LevelFilesBrief file_level_;
419 bool disjoint_sorted_files_;
420 Arena arena_;
421
FindLevelFileTest()422 FindLevelFileTest() : disjoint_sorted_files_(true) { }
423
~FindLevelFileTest()424 ~FindLevelFileTest() override {}
425
LevelFileInit(size_t num=0)426 void LevelFileInit(size_t num = 0) {
427 char* mem = arena_.AllocateAligned(num * sizeof(FdWithKeyRange));
428 file_level_.files = new (mem)FdWithKeyRange[num];
429 file_level_.num_files = 0;
430 }
431
Add(const char * smallest,const char * largest,SequenceNumber smallest_seq=100,SequenceNumber largest_seq=100)432 void Add(const char* smallest, const char* largest,
433 SequenceNumber smallest_seq = 100,
434 SequenceNumber largest_seq = 100) {
435 InternalKey smallest_key = InternalKey(smallest, smallest_seq, kTypeValue);
436 InternalKey largest_key = InternalKey(largest, largest_seq, kTypeValue);
437
438 Slice smallest_slice = smallest_key.Encode();
439 Slice largest_slice = largest_key.Encode();
440
441 char* mem = arena_.AllocateAligned(
442 smallest_slice.size() + largest_slice.size());
443 memcpy(mem, smallest_slice.data(), smallest_slice.size());
444 memcpy(mem + smallest_slice.size(), largest_slice.data(),
445 largest_slice.size());
446
447 // add to file_level_
448 size_t num = file_level_.num_files;
449 auto& file = file_level_.files[num];
450 file.fd = FileDescriptor(num + 1, 0, 0);
451 file.smallest_key = Slice(mem, smallest_slice.size());
452 file.largest_key = Slice(mem + smallest_slice.size(),
453 largest_slice.size());
454 file_level_.num_files++;
455 }
456
Find(const char * key)457 int Find(const char* key) {
458 InternalKey target(key, 100, kTypeValue);
459 InternalKeyComparator cmp(BytewiseComparator());
460 return FindFile(cmp, file_level_, target.Encode());
461 }
462
Overlaps(const char * smallest,const char * largest)463 bool Overlaps(const char* smallest, const char* largest) {
464 InternalKeyComparator cmp(BytewiseComparator());
465 Slice s(smallest != nullptr ? smallest : "");
466 Slice l(largest != nullptr ? largest : "");
467 return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, file_level_,
468 (smallest != nullptr ? &s : nullptr),
469 (largest != nullptr ? &l : nullptr));
470 }
471 };
472
TEST_F(FindLevelFileTest,LevelEmpty)473 TEST_F(FindLevelFileTest, LevelEmpty) {
474 LevelFileInit(0);
475
476 ASSERT_EQ(0, Find("foo"));
477 ASSERT_TRUE(! Overlaps("a", "z"));
478 ASSERT_TRUE(! Overlaps(nullptr, "z"));
479 ASSERT_TRUE(! Overlaps("a", nullptr));
480 ASSERT_TRUE(! Overlaps(nullptr, nullptr));
481 }
482
TEST_F(FindLevelFileTest,LevelSingle)483 TEST_F(FindLevelFileTest, LevelSingle) {
484 LevelFileInit(1);
485
486 Add("p", "q");
487 ASSERT_EQ(0, Find("a"));
488 ASSERT_EQ(0, Find("p"));
489 ASSERT_EQ(0, Find("p1"));
490 ASSERT_EQ(0, Find("q"));
491 ASSERT_EQ(1, Find("q1"));
492 ASSERT_EQ(1, Find("z"));
493
494 ASSERT_TRUE(! Overlaps("a", "b"));
495 ASSERT_TRUE(! Overlaps("z1", "z2"));
496 ASSERT_TRUE(Overlaps("a", "p"));
497 ASSERT_TRUE(Overlaps("a", "q"));
498 ASSERT_TRUE(Overlaps("a", "z"));
499 ASSERT_TRUE(Overlaps("p", "p1"));
500 ASSERT_TRUE(Overlaps("p", "q"));
501 ASSERT_TRUE(Overlaps("p", "z"));
502 ASSERT_TRUE(Overlaps("p1", "p2"));
503 ASSERT_TRUE(Overlaps("p1", "z"));
504 ASSERT_TRUE(Overlaps("q", "q"));
505 ASSERT_TRUE(Overlaps("q", "q1"));
506
507 ASSERT_TRUE(! Overlaps(nullptr, "j"));
508 ASSERT_TRUE(! Overlaps("r", nullptr));
509 ASSERT_TRUE(Overlaps(nullptr, "p"));
510 ASSERT_TRUE(Overlaps(nullptr, "p1"));
511 ASSERT_TRUE(Overlaps("q", nullptr));
512 ASSERT_TRUE(Overlaps(nullptr, nullptr));
513 }
514
TEST_F(FindLevelFileTest,LevelMultiple)515 TEST_F(FindLevelFileTest, LevelMultiple) {
516 LevelFileInit(4);
517
518 Add("150", "200");
519 Add("200", "250");
520 Add("300", "350");
521 Add("400", "450");
522 ASSERT_EQ(0, Find("100"));
523 ASSERT_EQ(0, Find("150"));
524 ASSERT_EQ(0, Find("151"));
525 ASSERT_EQ(0, Find("199"));
526 ASSERT_EQ(0, Find("200"));
527 ASSERT_EQ(1, Find("201"));
528 ASSERT_EQ(1, Find("249"));
529 ASSERT_EQ(1, Find("250"));
530 ASSERT_EQ(2, Find("251"));
531 ASSERT_EQ(2, Find("299"));
532 ASSERT_EQ(2, Find("300"));
533 ASSERT_EQ(2, Find("349"));
534 ASSERT_EQ(2, Find("350"));
535 ASSERT_EQ(3, Find("351"));
536 ASSERT_EQ(3, Find("400"));
537 ASSERT_EQ(3, Find("450"));
538 ASSERT_EQ(4, Find("451"));
539
540 ASSERT_TRUE(! Overlaps("100", "149"));
541 ASSERT_TRUE(! Overlaps("251", "299"));
542 ASSERT_TRUE(! Overlaps("451", "500"));
543 ASSERT_TRUE(! Overlaps("351", "399"));
544
545 ASSERT_TRUE(Overlaps("100", "150"));
546 ASSERT_TRUE(Overlaps("100", "200"));
547 ASSERT_TRUE(Overlaps("100", "300"));
548 ASSERT_TRUE(Overlaps("100", "400"));
549 ASSERT_TRUE(Overlaps("100", "500"));
550 ASSERT_TRUE(Overlaps("375", "400"));
551 ASSERT_TRUE(Overlaps("450", "450"));
552 ASSERT_TRUE(Overlaps("450", "500"));
553 }
554
TEST_F(FindLevelFileTest,LevelMultipleNullBoundaries)555 TEST_F(FindLevelFileTest, LevelMultipleNullBoundaries) {
556 LevelFileInit(4);
557
558 Add("150", "200");
559 Add("200", "250");
560 Add("300", "350");
561 Add("400", "450");
562 ASSERT_TRUE(! Overlaps(nullptr, "149"));
563 ASSERT_TRUE(! Overlaps("451", nullptr));
564 ASSERT_TRUE(Overlaps(nullptr, nullptr));
565 ASSERT_TRUE(Overlaps(nullptr, "150"));
566 ASSERT_TRUE(Overlaps(nullptr, "199"));
567 ASSERT_TRUE(Overlaps(nullptr, "200"));
568 ASSERT_TRUE(Overlaps(nullptr, "201"));
569 ASSERT_TRUE(Overlaps(nullptr, "400"));
570 ASSERT_TRUE(Overlaps(nullptr, "800"));
571 ASSERT_TRUE(Overlaps("100", nullptr));
572 ASSERT_TRUE(Overlaps("200", nullptr));
573 ASSERT_TRUE(Overlaps("449", nullptr));
574 ASSERT_TRUE(Overlaps("450", nullptr));
575 }
576
TEST_F(FindLevelFileTest,LevelOverlapSequenceChecks)577 TEST_F(FindLevelFileTest, LevelOverlapSequenceChecks) {
578 LevelFileInit(1);
579
580 Add("200", "200", 5000, 3000);
581 ASSERT_TRUE(! Overlaps("199", "199"));
582 ASSERT_TRUE(! Overlaps("201", "300"));
583 ASSERT_TRUE(Overlaps("200", "200"));
584 ASSERT_TRUE(Overlaps("190", "200"));
585 ASSERT_TRUE(Overlaps("200", "210"));
586 }
587
TEST_F(FindLevelFileTest,LevelOverlappingFiles)588 TEST_F(FindLevelFileTest, LevelOverlappingFiles) {
589 LevelFileInit(2);
590
591 Add("150", "600");
592 Add("400", "500");
593 disjoint_sorted_files_ = false;
594 ASSERT_TRUE(! Overlaps("100", "149"));
595 ASSERT_TRUE(! Overlaps("601", "700"));
596 ASSERT_TRUE(Overlaps("100", "150"));
597 ASSERT_TRUE(Overlaps("100", "200"));
598 ASSERT_TRUE(Overlaps("100", "300"));
599 ASSERT_TRUE(Overlaps("100", "400"));
600 ASSERT_TRUE(Overlaps("100", "500"));
601 ASSERT_TRUE(Overlaps("375", "400"));
602 ASSERT_TRUE(Overlaps("450", "450"));
603 ASSERT_TRUE(Overlaps("450", "500"));
604 ASSERT_TRUE(Overlaps("450", "700"));
605 ASSERT_TRUE(Overlaps("600", "700"));
606 }
607
608 class VersionSetTestBase {
609 public:
610 const static std::string kColumnFamilyName1;
611 const static std::string kColumnFamilyName2;
612 const static std::string kColumnFamilyName3;
613 int num_initial_edits_;
614
VersionSetTestBase(const std::string & name)615 explicit VersionSetTestBase(const std::string& name)
616 : mem_env_(nullptr),
617 env_(nullptr),
618 env_guard_(),
619 fs_(),
620 dbname_(test::PerThreadDBPath(name)),
621 options_(),
622 db_options_(options_),
623 cf_options_(options_),
624 immutable_cf_options_(db_options_, cf_options_),
625 mutable_cf_options_(cf_options_),
626 table_cache_(NewLRUCache(50000, 16)),
627 write_buffer_manager_(db_options_.db_write_buffer_size),
628 shutting_down_(false),
629 mock_table_factory_(std::make_shared<mock::MockTableFactory>()) {
630 const char* test_env_uri = getenv("TEST_ENV_URI");
631 Env* base_env = nullptr;
632 if (test_env_uri) {
633 Status s = Env::LoadEnv(test_env_uri, &base_env, &env_guard_);
634 EXPECT_OK(s);
635 EXPECT_NE(Env::Default(), base_env);
636 } else {
637 base_env = Env::Default();
638 }
639 EXPECT_NE(nullptr, base_env);
640 if (getenv("MEM_ENV")) {
641 mem_env_ = new MockEnv(base_env);
642 }
643 env_ = mem_env_ ? mem_env_ : base_env;
644
645 fs_ = std::make_shared<LegacyFileSystemWrapper>(env_);
646 EXPECT_OK(env_->CreateDirIfMissing(dbname_));
647
648 db_options_.env = env_;
649 db_options_.fs = fs_;
650 versions_.reset(new VersionSet(dbname_, &db_options_, env_options_,
651 table_cache_.get(), &write_buffer_manager_,
652 &write_controller_,
653 /*block_cache_tracer=*/nullptr));
654 reactive_versions_ = std::make_shared<ReactiveVersionSet>(
655 dbname_, &db_options_, env_options_, table_cache_.get(),
656 &write_buffer_manager_, &write_controller_);
657 db_options_.db_paths.emplace_back(dbname_,
658 std::numeric_limits<uint64_t>::max());
659 }
660
~VersionSetTestBase()661 virtual ~VersionSetTestBase() {
662 if (getenv("KEEP_DB")) {
663 fprintf(stdout, "DB is still at %s\n", dbname_.c_str());
664 } else {
665 Options options;
666 options.env = env_;
667 EXPECT_OK(DestroyDB(dbname_, options));
668 }
669 if (mem_env_) {
670 delete mem_env_;
671 mem_env_ = nullptr;
672 }
673 }
674
675 protected:
PrepareManifest(std::vector<ColumnFamilyDescriptor> * column_families,SequenceNumber * last_seqno,std::unique_ptr<log::Writer> * log_writer)676 virtual void PrepareManifest(
677 std::vector<ColumnFamilyDescriptor>* column_families,
678 SequenceNumber* last_seqno, std::unique_ptr<log::Writer>* log_writer) {
679 assert(column_families != nullptr);
680 assert(last_seqno != nullptr);
681 assert(log_writer != nullptr);
682 VersionEdit new_db;
683 if (db_options_.write_dbid_to_manifest) {
684 std::unique_ptr<DBImpl> impl(new DBImpl(DBOptions(), dbname_));
685 std::string db_id;
686 impl->GetDbIdentityFromIdentityFile(&db_id);
687 new_db.SetDBId(db_id);
688 }
689 new_db.SetLogNumber(0);
690 new_db.SetNextFile(2);
691 new_db.SetLastSequence(0);
692
693 const std::vector<std::string> cf_names = {
694 kDefaultColumnFamilyName, kColumnFamilyName1, kColumnFamilyName2,
695 kColumnFamilyName3};
696 const int kInitialNumOfCfs = static_cast<int>(cf_names.size());
697 autovector<VersionEdit> new_cfs;
698 uint64_t last_seq = 1;
699 uint32_t cf_id = 1;
700 for (int i = 1; i != kInitialNumOfCfs; ++i) {
701 VersionEdit new_cf;
702 new_cf.AddColumnFamily(cf_names[i]);
703 new_cf.SetColumnFamily(cf_id++);
704 new_cf.SetLogNumber(0);
705 new_cf.SetNextFile(2);
706 new_cf.SetLastSequence(last_seq++);
707 new_cfs.emplace_back(new_cf);
708 }
709 *last_seqno = last_seq;
710 num_initial_edits_ = static_cast<int>(new_cfs.size() + 1);
711 const std::string manifest = DescriptorFileName(dbname_, 1);
712 std::unique_ptr<WritableFile> file;
713 Status s = env_->NewWritableFile(
714 manifest, &file, env_->OptimizeForManifestWrite(env_options_));
715 ASSERT_OK(s);
716 std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
717 NewLegacyWritableFileWrapper(std::move(file)), manifest, env_options_));
718 {
719 log_writer->reset(new log::Writer(std::move(file_writer), 0, false));
720 std::string record;
721 new_db.EncodeTo(&record);
722 s = (*log_writer)->AddRecord(record);
723 for (const auto& e : new_cfs) {
724 record.clear();
725 e.EncodeTo(&record);
726 s = (*log_writer)->AddRecord(record);
727 ASSERT_OK(s);
728 }
729 }
730 ASSERT_OK(s);
731
732 cf_options_.table_factory = mock_table_factory_;
733 for (const auto& cf_name : cf_names) {
734 column_families->emplace_back(cf_name, cf_options_);
735 }
736 }
737
738 // Create DB with 3 column families.
NewDB()739 void NewDB() {
740 std::vector<ColumnFamilyDescriptor> column_families;
741 SequenceNumber last_seqno;
742 std::unique_ptr<log::Writer> log_writer;
743 SetIdentityFile(env_, dbname_);
744 PrepareManifest(&column_families, &last_seqno, &log_writer);
745 log_writer.reset();
746 // Make "CURRENT" file point to the new manifest file.
747 Status s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr);
748 ASSERT_OK(s);
749
750 EXPECT_OK(versions_->Recover(column_families, false));
751 EXPECT_EQ(column_families.size(),
752 versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
753 }
754
VerifyManifest(std::string * manifest_path) const755 void VerifyManifest(std::string* manifest_path) const {
756 assert(manifest_path != nullptr);
757 uint64_t manifest_file_number = 0;
758 Status s = versions_->GetCurrentManifestPath(
759 dbname_, fs_.get(), manifest_path, &manifest_file_number);
760 ASSERT_OK(s);
761 ASSERT_EQ(1, manifest_file_number);
762 }
763
764 MockEnv* mem_env_;
765 Env* env_;
766 std::shared_ptr<Env> env_guard_;
767 std::shared_ptr<FileSystem> fs_;
768 const std::string dbname_;
769 EnvOptions env_options_;
770 Options options_;
771 ImmutableDBOptions db_options_;
772 ColumnFamilyOptions cf_options_;
773 ImmutableCFOptions immutable_cf_options_;
774 MutableCFOptions mutable_cf_options_;
775 std::shared_ptr<Cache> table_cache_;
776 WriteController write_controller_;
777 WriteBufferManager write_buffer_manager_;
778 std::shared_ptr<VersionSet> versions_;
779 std::shared_ptr<ReactiveVersionSet> reactive_versions_;
780 InstrumentedMutex mutex_;
781 std::atomic<bool> shutting_down_;
782 std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
783 };
784
785 const std::string VersionSetTestBase::kColumnFamilyName1 = "alice";
786 const std::string VersionSetTestBase::kColumnFamilyName2 = "bob";
787 const std::string VersionSetTestBase::kColumnFamilyName3 = "charles";
788
789 class VersionSetTest : public VersionSetTestBase, public testing::Test {
790 public:
VersionSetTest()791 VersionSetTest() : VersionSetTestBase("version_set_test") {}
792 };
793
TEST_F(VersionSetTest,SameColumnFamilyGroupCommit)794 TEST_F(VersionSetTest, SameColumnFamilyGroupCommit) {
795 NewDB();
796 const int kGroupSize = 5;
797 autovector<VersionEdit> edits;
798 for (int i = 0; i != kGroupSize; ++i) {
799 edits.emplace_back(VersionEdit());
800 }
801 autovector<ColumnFamilyData*> cfds;
802 autovector<const MutableCFOptions*> all_mutable_cf_options;
803 autovector<autovector<VersionEdit*>> edit_lists;
804 for (int i = 0; i != kGroupSize; ++i) {
805 cfds.emplace_back(versions_->GetColumnFamilySet()->GetDefault());
806 all_mutable_cf_options.emplace_back(&mutable_cf_options_);
807 autovector<VersionEdit*> edit_list;
808 edit_list.emplace_back(&edits[i]);
809 edit_lists.emplace_back(edit_list);
810 }
811
812 SyncPoint::GetInstance()->DisableProcessing();
813 SyncPoint::GetInstance()->ClearAllCallBacks();
814 int count = 0;
815 SyncPoint::GetInstance()->SetCallBack(
816 "VersionSet::ProcessManifestWrites:SameColumnFamily", [&](void* arg) {
817 uint32_t* cf_id = reinterpret_cast<uint32_t*>(arg);
818 EXPECT_EQ(0u, *cf_id);
819 ++count;
820 });
821 SyncPoint::GetInstance()->EnableProcessing();
822 mutex_.Lock();
823 Status s =
824 versions_->LogAndApply(cfds, all_mutable_cf_options, edit_lists, &mutex_);
825 mutex_.Unlock();
826 EXPECT_OK(s);
827 EXPECT_EQ(kGroupSize - 1, count);
828 }
829
830 class VersionSetAtomicGroupTest : public VersionSetTestBase,
831 public testing::Test {
832 public:
VersionSetAtomicGroupTest()833 VersionSetAtomicGroupTest()
834 : VersionSetTestBase("version_set_atomic_group_test") {}
835
SetUp()836 void SetUp() override {
837 PrepareManifest(&column_families_, &last_seqno_, &log_writer_);
838 SetupTestSyncPoints();
839 }
840
SetupValidAtomicGroup(int atomic_group_size)841 void SetupValidAtomicGroup(int atomic_group_size) {
842 edits_.resize(atomic_group_size);
843 int remaining = atomic_group_size;
844 for (size_t i = 0; i != edits_.size(); ++i) {
845 edits_[i].SetLogNumber(0);
846 edits_[i].SetNextFile(2);
847 edits_[i].MarkAtomicGroup(--remaining);
848 edits_[i].SetLastSequence(last_seqno_++);
849 }
850 ASSERT_OK(SetCurrentFile(fs_.get(), dbname_, 1, nullptr));
851 }
852
SetupIncompleteTrailingAtomicGroup(int atomic_group_size)853 void SetupIncompleteTrailingAtomicGroup(int atomic_group_size) {
854 edits_.resize(atomic_group_size);
855 int remaining = atomic_group_size;
856 for (size_t i = 0; i != edits_.size(); ++i) {
857 edits_[i].SetLogNumber(0);
858 edits_[i].SetNextFile(2);
859 edits_[i].MarkAtomicGroup(--remaining);
860 edits_[i].SetLastSequence(last_seqno_++);
861 }
862 ASSERT_OK(SetCurrentFile(fs_.get(), dbname_, 1, nullptr));
863 }
864
SetupCorruptedAtomicGroup(int atomic_group_size)865 void SetupCorruptedAtomicGroup(int atomic_group_size) {
866 edits_.resize(atomic_group_size);
867 int remaining = atomic_group_size;
868 for (size_t i = 0; i != edits_.size(); ++i) {
869 edits_[i].SetLogNumber(0);
870 edits_[i].SetNextFile(2);
871 if (i != ((size_t)atomic_group_size / 2)) {
872 edits_[i].MarkAtomicGroup(--remaining);
873 }
874 edits_[i].SetLastSequence(last_seqno_++);
875 }
876 ASSERT_OK(SetCurrentFile(fs_.get(), dbname_, 1, nullptr));
877 }
878
SetupIncorrectAtomicGroup(int atomic_group_size)879 void SetupIncorrectAtomicGroup(int atomic_group_size) {
880 edits_.resize(atomic_group_size);
881 int remaining = atomic_group_size;
882 for (size_t i = 0; i != edits_.size(); ++i) {
883 edits_[i].SetLogNumber(0);
884 edits_[i].SetNextFile(2);
885 if (i != 1) {
886 edits_[i].MarkAtomicGroup(--remaining);
887 } else {
888 edits_[i].MarkAtomicGroup(remaining--);
889 }
890 edits_[i].SetLastSequence(last_seqno_++);
891 }
892 ASSERT_OK(SetCurrentFile(fs_.get(), dbname_, 1, nullptr));
893 }
894
SetupTestSyncPoints()895 void SetupTestSyncPoints() {
896 SyncPoint::GetInstance()->DisableProcessing();
897 SyncPoint::GetInstance()->ClearAllCallBacks();
898 SyncPoint::GetInstance()->SetCallBack(
899 "AtomicGroupReadBuffer::AddEdit:FirstInAtomicGroup", [&](void* arg) {
900 VersionEdit* e = reinterpret_cast<VersionEdit*>(arg);
901 EXPECT_EQ(edits_.front().DebugString(),
902 e->DebugString()); // compare based on value
903 first_in_atomic_group_ = true;
904 });
905 SyncPoint::GetInstance()->SetCallBack(
906 "AtomicGroupReadBuffer::AddEdit:LastInAtomicGroup", [&](void* arg) {
907 VersionEdit* e = reinterpret_cast<VersionEdit*>(arg);
908 EXPECT_EQ(edits_.back().DebugString(),
909 e->DebugString()); // compare based on value
910 EXPECT_TRUE(first_in_atomic_group_);
911 last_in_atomic_group_ = true;
912 });
913 SyncPoint::GetInstance()->SetCallBack(
914 "VersionSet::ReadAndRecover:RecoveredEdits", [&](void* arg) {
915 num_recovered_edits_ = *reinterpret_cast<int*>(arg);
916 });
917 SyncPoint::GetInstance()->SetCallBack(
918 "ReactiveVersionSet::ReadAndApply:AppliedEdits",
919 [&](void* arg) { num_applied_edits_ = *reinterpret_cast<int*>(arg); });
920 SyncPoint::GetInstance()->SetCallBack(
921 "AtomicGroupReadBuffer::AddEdit:AtomicGroup",
922 [&](void* /* arg */) { ++num_edits_in_atomic_group_; });
923 SyncPoint::GetInstance()->SetCallBack(
924 "AtomicGroupReadBuffer::AddEdit:AtomicGroupMixedWithNormalEdits",
925 [&](void* arg) {
926 corrupted_edit_ = *reinterpret_cast<VersionEdit*>(arg);
927 });
928 SyncPoint::GetInstance()->SetCallBack(
929 "AtomicGroupReadBuffer::AddEdit:IncorrectAtomicGroupSize",
930 [&](void* arg) {
931 edit_with_incorrect_group_size_ =
932 *reinterpret_cast<VersionEdit*>(arg);
933 });
934 SyncPoint::GetInstance()->EnableProcessing();
935 }
936
AddNewEditsToLog(int num_edits)937 void AddNewEditsToLog(int num_edits) {
938 for (int i = 0; i < num_edits; i++) {
939 std::string record;
940 edits_[i].EncodeTo(&record);
941 ASSERT_OK(log_writer_->AddRecord(record));
942 }
943 }
944
TearDown()945 void TearDown() override {
946 SyncPoint::GetInstance()->DisableProcessing();
947 SyncPoint::GetInstance()->ClearAllCallBacks();
948 log_writer_.reset();
949 }
950
951 protected:
952 std::vector<ColumnFamilyDescriptor> column_families_;
953 SequenceNumber last_seqno_;
954 std::vector<VersionEdit> edits_;
955 bool first_in_atomic_group_ = false;
956 bool last_in_atomic_group_ = false;
957 int num_edits_in_atomic_group_ = 0;
958 int num_recovered_edits_ = 0;
959 int num_applied_edits_ = 0;
960 VersionEdit corrupted_edit_;
961 VersionEdit edit_with_incorrect_group_size_;
962 std::unique_ptr<log::Writer> log_writer_;
963 };
964
TEST_F(VersionSetAtomicGroupTest,HandleValidAtomicGroupWithVersionSetRecover)965 TEST_F(VersionSetAtomicGroupTest, HandleValidAtomicGroupWithVersionSetRecover) {
966 const int kAtomicGroupSize = 3;
967 SetupValidAtomicGroup(kAtomicGroupSize);
968 AddNewEditsToLog(kAtomicGroupSize);
969 EXPECT_OK(versions_->Recover(column_families_, false));
970 EXPECT_EQ(column_families_.size(),
971 versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
972 EXPECT_TRUE(first_in_atomic_group_);
973 EXPECT_TRUE(last_in_atomic_group_);
974 EXPECT_EQ(num_initial_edits_ + kAtomicGroupSize, num_recovered_edits_);
975 EXPECT_EQ(0, num_applied_edits_);
976 }
977
TEST_F(VersionSetAtomicGroupTest,HandleValidAtomicGroupWithReactiveVersionSetRecover)978 TEST_F(VersionSetAtomicGroupTest,
979 HandleValidAtomicGroupWithReactiveVersionSetRecover) {
980 const int kAtomicGroupSize = 3;
981 SetupValidAtomicGroup(kAtomicGroupSize);
982 AddNewEditsToLog(kAtomicGroupSize);
983 std::unique_ptr<log::FragmentBufferedReader> manifest_reader;
984 std::unique_ptr<log::Reader::Reporter> manifest_reporter;
985 std::unique_ptr<Status> manifest_reader_status;
986 EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader,
987 &manifest_reporter,
988 &manifest_reader_status));
989 EXPECT_EQ(column_families_.size(),
990 reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
991 EXPECT_TRUE(first_in_atomic_group_);
992 EXPECT_TRUE(last_in_atomic_group_);
993 // The recover should clean up the replay buffer.
994 EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == 0);
995 EXPECT_TRUE(reactive_versions_->replay_buffer().size() == 0);
996 EXPECT_EQ(num_initial_edits_ + kAtomicGroupSize, num_recovered_edits_);
997 EXPECT_EQ(0, num_applied_edits_);
998 }
999
TEST_F(VersionSetAtomicGroupTest,HandleValidAtomicGroupWithReactiveVersionSetReadAndApply)1000 TEST_F(VersionSetAtomicGroupTest,
1001 HandleValidAtomicGroupWithReactiveVersionSetReadAndApply) {
1002 const int kAtomicGroupSize = 3;
1003 SetupValidAtomicGroup(kAtomicGroupSize);
1004 std::unique_ptr<log::FragmentBufferedReader> manifest_reader;
1005 std::unique_ptr<log::Reader::Reporter> manifest_reporter;
1006 std::unique_ptr<Status> manifest_reader_status;
1007 EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader,
1008 &manifest_reporter,
1009 &manifest_reader_status));
1010 AddNewEditsToLog(kAtomicGroupSize);
1011 InstrumentedMutex mu;
1012 std::unordered_set<ColumnFamilyData*> cfds_changed;
1013 mu.Lock();
1014 EXPECT_OK(
1015 reactive_versions_->ReadAndApply(&mu, &manifest_reader, &cfds_changed));
1016 mu.Unlock();
1017 EXPECT_TRUE(first_in_atomic_group_);
1018 EXPECT_TRUE(last_in_atomic_group_);
1019 // The recover should clean up the replay buffer.
1020 EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == 0);
1021 EXPECT_TRUE(reactive_versions_->replay_buffer().size() == 0);
1022 EXPECT_EQ(num_initial_edits_, num_recovered_edits_);
1023 EXPECT_EQ(kAtomicGroupSize, num_applied_edits_);
1024 }
1025
TEST_F(VersionSetAtomicGroupTest,HandleIncompleteTrailingAtomicGroupWithVersionSetRecover)1026 TEST_F(VersionSetAtomicGroupTest,
1027 HandleIncompleteTrailingAtomicGroupWithVersionSetRecover) {
1028 const int kAtomicGroupSize = 4;
1029 const int kNumberOfPersistedVersionEdits = kAtomicGroupSize - 1;
1030 SetupIncompleteTrailingAtomicGroup(kAtomicGroupSize);
1031 AddNewEditsToLog(kNumberOfPersistedVersionEdits);
1032 EXPECT_OK(versions_->Recover(column_families_, false));
1033 EXPECT_EQ(column_families_.size(),
1034 versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
1035 EXPECT_TRUE(first_in_atomic_group_);
1036 EXPECT_FALSE(last_in_atomic_group_);
1037 EXPECT_EQ(kNumberOfPersistedVersionEdits, num_edits_in_atomic_group_);
1038 EXPECT_EQ(num_initial_edits_, num_recovered_edits_);
1039 EXPECT_EQ(0, num_applied_edits_);
1040 }
1041
TEST_F(VersionSetAtomicGroupTest,HandleIncompleteTrailingAtomicGroupWithReactiveVersionSetRecover)1042 TEST_F(VersionSetAtomicGroupTest,
1043 HandleIncompleteTrailingAtomicGroupWithReactiveVersionSetRecover) {
1044 const int kAtomicGroupSize = 4;
1045 const int kNumberOfPersistedVersionEdits = kAtomicGroupSize - 1;
1046 SetupIncompleteTrailingAtomicGroup(kAtomicGroupSize);
1047 AddNewEditsToLog(kNumberOfPersistedVersionEdits);
1048 std::unique_ptr<log::FragmentBufferedReader> manifest_reader;
1049 std::unique_ptr<log::Reader::Reporter> manifest_reporter;
1050 std::unique_ptr<Status> manifest_reader_status;
1051 EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader,
1052 &manifest_reporter,
1053 &manifest_reader_status));
1054 EXPECT_EQ(column_families_.size(),
1055 reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
1056 EXPECT_TRUE(first_in_atomic_group_);
1057 EXPECT_FALSE(last_in_atomic_group_);
1058 EXPECT_EQ(kNumberOfPersistedVersionEdits, num_edits_in_atomic_group_);
1059 // Reactive version set should store the edits in the replay buffer.
1060 EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() ==
1061 kNumberOfPersistedVersionEdits);
1062 EXPECT_TRUE(reactive_versions_->replay_buffer().size() == kAtomicGroupSize);
1063 // Write the last record. The reactive version set should now apply all
1064 // edits.
1065 std::string last_record;
1066 edits_[kAtomicGroupSize - 1].EncodeTo(&last_record);
1067 EXPECT_OK(log_writer_->AddRecord(last_record));
1068 InstrumentedMutex mu;
1069 std::unordered_set<ColumnFamilyData*> cfds_changed;
1070 mu.Lock();
1071 EXPECT_OK(
1072 reactive_versions_->ReadAndApply(&mu, &manifest_reader, &cfds_changed));
1073 mu.Unlock();
1074 // Reactive version set should be empty now.
1075 EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() == 0);
1076 EXPECT_TRUE(reactive_versions_->replay_buffer().size() == 0);
1077 EXPECT_EQ(num_initial_edits_, num_recovered_edits_);
1078 EXPECT_EQ(kAtomicGroupSize, num_applied_edits_);
1079 }
1080
TEST_F(VersionSetAtomicGroupTest,HandleIncompleteTrailingAtomicGroupWithReactiveVersionSetReadAndApply)1081 TEST_F(VersionSetAtomicGroupTest,
1082 HandleIncompleteTrailingAtomicGroupWithReactiveVersionSetReadAndApply) {
1083 const int kAtomicGroupSize = 4;
1084 const int kNumberOfPersistedVersionEdits = kAtomicGroupSize - 1;
1085 SetupIncompleteTrailingAtomicGroup(kAtomicGroupSize);
1086 std::unique_ptr<log::FragmentBufferedReader> manifest_reader;
1087 std::unique_ptr<log::Reader::Reporter> manifest_reporter;
1088 std::unique_ptr<Status> manifest_reader_status;
1089 // No edits in an atomic group.
1090 EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader,
1091 &manifest_reporter,
1092 &manifest_reader_status));
1093 EXPECT_EQ(column_families_.size(),
1094 reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
1095 // Write a few edits in an atomic group.
1096 AddNewEditsToLog(kNumberOfPersistedVersionEdits);
1097 InstrumentedMutex mu;
1098 std::unordered_set<ColumnFamilyData*> cfds_changed;
1099 mu.Lock();
1100 EXPECT_OK(
1101 reactive_versions_->ReadAndApply(&mu, &manifest_reader, &cfds_changed));
1102 mu.Unlock();
1103 EXPECT_TRUE(first_in_atomic_group_);
1104 EXPECT_FALSE(last_in_atomic_group_);
1105 EXPECT_EQ(kNumberOfPersistedVersionEdits, num_edits_in_atomic_group_);
1106 // Reactive version set should store the edits in the replay buffer.
1107 EXPECT_TRUE(reactive_versions_->TEST_read_edits_in_atomic_group() ==
1108 kNumberOfPersistedVersionEdits);
1109 EXPECT_TRUE(reactive_versions_->replay_buffer().size() == kAtomicGroupSize);
1110 EXPECT_EQ(num_initial_edits_, num_recovered_edits_);
1111 EXPECT_EQ(0, num_applied_edits_);
1112 }
1113
TEST_F(VersionSetAtomicGroupTest,HandleCorruptedAtomicGroupWithVersionSetRecover)1114 TEST_F(VersionSetAtomicGroupTest,
1115 HandleCorruptedAtomicGroupWithVersionSetRecover) {
1116 const int kAtomicGroupSize = 4;
1117 SetupCorruptedAtomicGroup(kAtomicGroupSize);
1118 AddNewEditsToLog(kAtomicGroupSize);
1119 EXPECT_NOK(versions_->Recover(column_families_, false));
1120 EXPECT_EQ(column_families_.size(),
1121 versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
1122 EXPECT_EQ(edits_[kAtomicGroupSize / 2].DebugString(),
1123 corrupted_edit_.DebugString());
1124 }
1125
TEST_F(VersionSetAtomicGroupTest,HandleCorruptedAtomicGroupWithReactiveVersionSetRecover)1126 TEST_F(VersionSetAtomicGroupTest,
1127 HandleCorruptedAtomicGroupWithReactiveVersionSetRecover) {
1128 const int kAtomicGroupSize = 4;
1129 SetupCorruptedAtomicGroup(kAtomicGroupSize);
1130 AddNewEditsToLog(kAtomicGroupSize);
1131 std::unique_ptr<log::FragmentBufferedReader> manifest_reader;
1132 std::unique_ptr<log::Reader::Reporter> manifest_reporter;
1133 std::unique_ptr<Status> manifest_reader_status;
1134 EXPECT_NOK(reactive_versions_->Recover(column_families_, &manifest_reader,
1135 &manifest_reporter,
1136 &manifest_reader_status));
1137 EXPECT_EQ(column_families_.size(),
1138 reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
1139 EXPECT_EQ(edits_[kAtomicGroupSize / 2].DebugString(),
1140 corrupted_edit_.DebugString());
1141 }
1142
TEST_F(VersionSetAtomicGroupTest,HandleCorruptedAtomicGroupWithReactiveVersionSetReadAndApply)1143 TEST_F(VersionSetAtomicGroupTest,
1144 HandleCorruptedAtomicGroupWithReactiveVersionSetReadAndApply) {
1145 const int kAtomicGroupSize = 4;
1146 SetupCorruptedAtomicGroup(kAtomicGroupSize);
1147 InstrumentedMutex mu;
1148 std::unordered_set<ColumnFamilyData*> cfds_changed;
1149 std::unique_ptr<log::FragmentBufferedReader> manifest_reader;
1150 std::unique_ptr<log::Reader::Reporter> manifest_reporter;
1151 std::unique_ptr<Status> manifest_reader_status;
1152 EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader,
1153 &manifest_reporter,
1154 &manifest_reader_status));
1155 // Write the corrupted edits.
1156 AddNewEditsToLog(kAtomicGroupSize);
1157 mu.Lock();
1158 EXPECT_OK(
1159 reactive_versions_->ReadAndApply(&mu, &manifest_reader, &cfds_changed));
1160 mu.Unlock();
1161 EXPECT_EQ(edits_[kAtomicGroupSize / 2].DebugString(),
1162 corrupted_edit_.DebugString());
1163 }
1164
TEST_F(VersionSetAtomicGroupTest,HandleIncorrectAtomicGroupSizeWithVersionSetRecover)1165 TEST_F(VersionSetAtomicGroupTest,
1166 HandleIncorrectAtomicGroupSizeWithVersionSetRecover) {
1167 const int kAtomicGroupSize = 4;
1168 SetupIncorrectAtomicGroup(kAtomicGroupSize);
1169 AddNewEditsToLog(kAtomicGroupSize);
1170 EXPECT_NOK(versions_->Recover(column_families_, false));
1171 EXPECT_EQ(column_families_.size(),
1172 versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
1173 EXPECT_EQ(edits_[1].DebugString(),
1174 edit_with_incorrect_group_size_.DebugString());
1175 }
1176
TEST_F(VersionSetAtomicGroupTest,HandleIncorrectAtomicGroupSizeWithReactiveVersionSetRecover)1177 TEST_F(VersionSetAtomicGroupTest,
1178 HandleIncorrectAtomicGroupSizeWithReactiveVersionSetRecover) {
1179 const int kAtomicGroupSize = 4;
1180 SetupIncorrectAtomicGroup(kAtomicGroupSize);
1181 AddNewEditsToLog(kAtomicGroupSize);
1182 std::unique_ptr<log::FragmentBufferedReader> manifest_reader;
1183 std::unique_ptr<log::Reader::Reporter> manifest_reporter;
1184 std::unique_ptr<Status> manifest_reader_status;
1185 EXPECT_NOK(reactive_versions_->Recover(column_families_, &manifest_reader,
1186 &manifest_reporter,
1187 &manifest_reader_status));
1188 EXPECT_EQ(column_families_.size(),
1189 reactive_versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
1190 EXPECT_EQ(edits_[1].DebugString(),
1191 edit_with_incorrect_group_size_.DebugString());
1192 }
1193
TEST_F(VersionSetAtomicGroupTest,HandleIncorrectAtomicGroupSizeWithReactiveVersionSetReadAndApply)1194 TEST_F(VersionSetAtomicGroupTest,
1195 HandleIncorrectAtomicGroupSizeWithReactiveVersionSetReadAndApply) {
1196 const int kAtomicGroupSize = 4;
1197 SetupIncorrectAtomicGroup(kAtomicGroupSize);
1198 InstrumentedMutex mu;
1199 std::unordered_set<ColumnFamilyData*> cfds_changed;
1200 std::unique_ptr<log::FragmentBufferedReader> manifest_reader;
1201 std::unique_ptr<log::Reader::Reporter> manifest_reporter;
1202 std::unique_ptr<Status> manifest_reader_status;
1203 EXPECT_OK(reactive_versions_->Recover(column_families_, &manifest_reader,
1204 &manifest_reporter,
1205 &manifest_reader_status));
1206 AddNewEditsToLog(kAtomicGroupSize);
1207 mu.Lock();
1208 EXPECT_OK(
1209 reactive_versions_->ReadAndApply(&mu, &manifest_reader, &cfds_changed));
1210 mu.Unlock();
1211 EXPECT_EQ(edits_[1].DebugString(),
1212 edit_with_incorrect_group_size_.DebugString());
1213 }
1214
1215 class VersionSetTestDropOneCF : public VersionSetTestBase,
1216 public testing::TestWithParam<std::string> {
1217 public:
VersionSetTestDropOneCF()1218 VersionSetTestDropOneCF()
1219 : VersionSetTestBase("version_set_test_drop_one_cf") {}
1220 };
1221
1222 // This test simulates the following execution sequence
1223 // Time thread1 bg_flush_thr
1224 // | Prepare version edits (e1,e2,e3) for atomic
1225 // | flush cf1, cf2, cf3
1226 // | Enqueue e to drop cfi
1227 // | to manifest_writers_
1228 // | Enqueue (e1,e2,e3) to manifest_writers_
1229 // |
1230 // | Apply e,
1231 // | cfi.IsDropped() is true
1232 // | Apply (e1,e2,e3),
1233 // | since cfi.IsDropped() == true, we need to
1234 // | drop ei and write the rest to MANIFEST.
1235 // V
1236 //
1237 // Repeat the test for i = 1, 2, 3 to simulate dropping the first, middle and
1238 // last column family in an atomic group.
TEST_P(VersionSetTestDropOneCF,HandleDroppedColumnFamilyInAtomicGroup)1239 TEST_P(VersionSetTestDropOneCF, HandleDroppedColumnFamilyInAtomicGroup) {
1240 std::vector<ColumnFamilyDescriptor> column_families;
1241 SequenceNumber last_seqno;
1242 std::unique_ptr<log::Writer> log_writer;
1243 PrepareManifest(&column_families, &last_seqno, &log_writer);
1244 Status s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr);
1245 ASSERT_OK(s);
1246
1247 EXPECT_OK(versions_->Recover(column_families, false /* read_only */));
1248 EXPECT_EQ(column_families.size(),
1249 versions_->GetColumnFamilySet()->NumberOfColumnFamilies());
1250
1251 const int kAtomicGroupSize = 3;
1252 const std::vector<std::string> non_default_cf_names = {
1253 kColumnFamilyName1, kColumnFamilyName2, kColumnFamilyName3};
1254
1255 // Drop one column family
1256 VersionEdit drop_cf_edit;
1257 drop_cf_edit.DropColumnFamily();
1258 const std::string cf_to_drop_name(GetParam());
1259 auto cfd_to_drop =
1260 versions_->GetColumnFamilySet()->GetColumnFamily(cf_to_drop_name);
1261 ASSERT_NE(nullptr, cfd_to_drop);
1262 // Increase its refcount because cfd_to_drop is used later, and we need to
1263 // prevent it from being deleted.
1264 cfd_to_drop->Ref();
1265 drop_cf_edit.SetColumnFamily(cfd_to_drop->GetID());
1266 mutex_.Lock();
1267 s = versions_->LogAndApply(cfd_to_drop,
1268 *cfd_to_drop->GetLatestMutableCFOptions(),
1269 &drop_cf_edit, &mutex_);
1270 mutex_.Unlock();
1271 ASSERT_OK(s);
1272
1273 std::vector<VersionEdit> edits(kAtomicGroupSize);
1274 uint32_t remaining = kAtomicGroupSize;
1275 size_t i = 0;
1276 autovector<ColumnFamilyData*> cfds;
1277 autovector<const MutableCFOptions*> mutable_cf_options_list;
1278 autovector<autovector<VersionEdit*>> edit_lists;
1279 for (const auto& cf_name : non_default_cf_names) {
1280 auto cfd = (cf_name != cf_to_drop_name)
1281 ? versions_->GetColumnFamilySet()->GetColumnFamily(cf_name)
1282 : cfd_to_drop;
1283 ASSERT_NE(nullptr, cfd);
1284 cfds.push_back(cfd);
1285 mutable_cf_options_list.emplace_back(cfd->GetLatestMutableCFOptions());
1286 edits[i].SetColumnFamily(cfd->GetID());
1287 edits[i].SetLogNumber(0);
1288 edits[i].SetNextFile(2);
1289 edits[i].MarkAtomicGroup(--remaining);
1290 edits[i].SetLastSequence(last_seqno++);
1291 autovector<VersionEdit*> tmp_edits;
1292 tmp_edits.push_back(&edits[i]);
1293 edit_lists.emplace_back(tmp_edits);
1294 ++i;
1295 }
1296 int called = 0;
1297 SyncPoint::GetInstance()->DisableProcessing();
1298 SyncPoint::GetInstance()->ClearAllCallBacks();
1299 SyncPoint::GetInstance()->SetCallBack(
1300 "VersionSet::ProcessManifestWrites:CheckOneAtomicGroup", [&](void* arg) {
1301 std::vector<VersionEdit*>* tmp_edits =
1302 reinterpret_cast<std::vector<VersionEdit*>*>(arg);
1303 EXPECT_EQ(kAtomicGroupSize - 1, tmp_edits->size());
1304 for (const auto e : *tmp_edits) {
1305 bool found = false;
1306 for (const auto& e2 : edits) {
1307 if (&e2 == e) {
1308 found = true;
1309 break;
1310 }
1311 }
1312 ASSERT_TRUE(found);
1313 }
1314 ++called;
1315 });
1316 SyncPoint::GetInstance()->EnableProcessing();
1317 mutex_.Lock();
1318 s = versions_->LogAndApply(cfds, mutable_cf_options_list, edit_lists,
1319 &mutex_);
1320 mutex_.Unlock();
1321 ASSERT_OK(s);
1322 ASSERT_EQ(1, called);
1323 if (cfd_to_drop->Unref()) {
1324 delete cfd_to_drop;
1325 cfd_to_drop = nullptr;
1326 }
1327 }
1328
1329 INSTANTIATE_TEST_CASE_P(
1330 AtomicGroup, VersionSetTestDropOneCF,
1331 testing::Values(VersionSetTestBase::kColumnFamilyName1,
1332 VersionSetTestBase::kColumnFamilyName2,
1333 VersionSetTestBase::kColumnFamilyName3));
1334
1335 class EmptyDefaultCfNewManifest : public VersionSetTestBase,
1336 public testing::Test {
1337 public:
EmptyDefaultCfNewManifest()1338 EmptyDefaultCfNewManifest() : VersionSetTestBase("version_set_new_db_test") {}
1339 // Emulate DBImpl::NewDB()
PrepareManifest(std::vector<ColumnFamilyDescriptor> *,SequenceNumber *,std::unique_ptr<log::Writer> * log_writer)1340 void PrepareManifest(std::vector<ColumnFamilyDescriptor>* /*column_families*/,
1341 SequenceNumber* /*last_seqno*/,
1342 std::unique_ptr<log::Writer>* log_writer) override {
1343 assert(log_writer != nullptr);
1344 VersionEdit new_db;
1345 new_db.SetLogNumber(0);
1346 std::unique_ptr<WritableFile> file;
1347 const std::string manifest_path = DescriptorFileName(dbname_, 1);
1348 Status s = env_->NewWritableFile(
1349 manifest_path, &file, env_->OptimizeForManifestWrite(env_options_));
1350 ASSERT_OK(s);
1351 std::unique_ptr<WritableFileWriter> file_writer(
1352 new WritableFileWriter(NewLegacyWritableFileWrapper(std::move(file)),
1353 manifest_path, env_options_));
1354 log_writer->reset(new log::Writer(std::move(file_writer), 0, true));
1355 std::string record;
1356 ASSERT_TRUE(new_db.EncodeTo(&record));
1357 s = (*log_writer)->AddRecord(record);
1358 ASSERT_OK(s);
1359 // Create new column family
1360 VersionEdit new_cf;
1361 new_cf.AddColumnFamily(VersionSetTestBase::kColumnFamilyName1);
1362 new_cf.SetColumnFamily(1);
1363 new_cf.SetLastSequence(2);
1364 new_cf.SetNextFile(2);
1365 record.clear();
1366 ASSERT_TRUE(new_cf.EncodeTo(&record));
1367 s = (*log_writer)->AddRecord(record);
1368 ASSERT_OK(s);
1369 }
1370
1371 protected:
1372 bool write_dbid_to_manifest_ = false;
1373 std::unique_ptr<log::Writer> log_writer_;
1374 };
1375
1376 // Create db, create column family. Cf creation will switch to a new MANIFEST.
1377 // Then reopen db, trying to recover.
TEST_F(EmptyDefaultCfNewManifest,Recover)1378 TEST_F(EmptyDefaultCfNewManifest, Recover) {
1379 PrepareManifest(nullptr, nullptr, &log_writer_);
1380 log_writer_.reset();
1381 Status s =
1382 SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr);
1383 ASSERT_OK(s);
1384 std::string manifest_path;
1385 VerifyManifest(&manifest_path);
1386 std::vector<ColumnFamilyDescriptor> column_families;
1387 column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
1388 column_families.emplace_back(VersionSetTestBase::kColumnFamilyName1,
1389 cf_options_);
1390 std::string db_id;
1391 bool has_missing_table_file = false;
1392 s = versions_->TryRecoverFromOneManifest(
1393 manifest_path, column_families, false, &db_id, &has_missing_table_file);
1394 ASSERT_OK(s);
1395 ASSERT_FALSE(has_missing_table_file);
1396 }
1397
1398 class VersionSetTestEmptyDb
1399 : public VersionSetTestBase,
1400 public testing::TestWithParam<
1401 std::tuple<bool, bool, std::vector<std::string>>> {
1402 public:
1403 static const std::string kUnknownColumnFamilyName;
VersionSetTestEmptyDb()1404 VersionSetTestEmptyDb() : VersionSetTestBase("version_set_test_empty_db") {}
1405
1406 protected:
PrepareManifest(std::vector<ColumnFamilyDescriptor> *,SequenceNumber *,std::unique_ptr<log::Writer> * log_writer)1407 void PrepareManifest(std::vector<ColumnFamilyDescriptor>* /*column_families*/,
1408 SequenceNumber* /*last_seqno*/,
1409 std::unique_ptr<log::Writer>* log_writer) override {
1410 assert(nullptr != log_writer);
1411 VersionEdit new_db;
1412 if (db_options_.write_dbid_to_manifest) {
1413 std::unique_ptr<DBImpl> impl(new DBImpl(DBOptions(), dbname_));
1414 std::string db_id;
1415 impl->GetDbIdentityFromIdentityFile(&db_id);
1416 new_db.SetDBId(db_id);
1417 }
1418 const std::string manifest_path = DescriptorFileName(dbname_, 1);
1419 std::unique_ptr<WritableFile> file;
1420 Status s = env_->NewWritableFile(
1421 manifest_path, &file, env_->OptimizeForManifestWrite(env_options_));
1422 ASSERT_OK(s);
1423 std::unique_ptr<WritableFileWriter> file_writer(
1424 new WritableFileWriter(NewLegacyWritableFileWrapper(std::move(file)),
1425 manifest_path, env_options_));
1426 {
1427 log_writer->reset(new log::Writer(std::move(file_writer), 0, false));
1428 std::string record;
1429 new_db.EncodeTo(&record);
1430 s = (*log_writer)->AddRecord(record);
1431 ASSERT_OK(s);
1432 }
1433 }
1434
1435 std::unique_ptr<log::Writer> log_writer_;
1436 };
1437
1438 const std::string VersionSetTestEmptyDb::kUnknownColumnFamilyName = "unknown";
1439
TEST_P(VersionSetTestEmptyDb,OpenFromIncompleteManifest0)1440 TEST_P(VersionSetTestEmptyDb, OpenFromIncompleteManifest0) {
1441 db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
1442 PrepareManifest(nullptr, nullptr, &log_writer_);
1443 log_writer_.reset();
1444 Status s =
1445 SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr);
1446 ASSERT_OK(s);
1447
1448 std::string manifest_path;
1449 VerifyManifest(&manifest_path);
1450
1451 bool read_only = std::get<1>(GetParam());
1452 const std::vector<std::string> cf_names = std::get<2>(GetParam());
1453
1454 std::vector<ColumnFamilyDescriptor> column_families;
1455 for (const auto& cf_name : cf_names) {
1456 column_families.emplace_back(cf_name, cf_options_);
1457 }
1458
1459 std::string db_id;
1460 bool has_missing_table_file = false;
1461 s = versions_->TryRecoverFromOneManifest(manifest_path, column_families,
1462 read_only, &db_id,
1463 &has_missing_table_file);
1464 auto iter =
1465 std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName);
1466 if (iter == cf_names.end()) {
1467 ASSERT_TRUE(s.IsInvalidArgument());
1468 } else {
1469 ASSERT_TRUE(s.IsCorruption());
1470 }
1471 }
1472
TEST_P(VersionSetTestEmptyDb,OpenFromIncompleteManifest1)1473 TEST_P(VersionSetTestEmptyDb, OpenFromIncompleteManifest1) {
1474 db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
1475 PrepareManifest(nullptr, nullptr, &log_writer_);
1476 // Only a subset of column families in the MANIFEST.
1477 VersionEdit new_cf1;
1478 new_cf1.AddColumnFamily(VersionSetTestBase::kColumnFamilyName1);
1479 new_cf1.SetColumnFamily(1);
1480 Status s;
1481 {
1482 std::string record;
1483 new_cf1.EncodeTo(&record);
1484 s = log_writer_->AddRecord(record);
1485 ASSERT_OK(s);
1486 }
1487 log_writer_.reset();
1488 s = SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr);
1489 ASSERT_OK(s);
1490
1491 std::string manifest_path;
1492 VerifyManifest(&manifest_path);
1493
1494 bool read_only = std::get<1>(GetParam());
1495 const std::vector<std::string>& cf_names = std::get<2>(GetParam());
1496 std::vector<ColumnFamilyDescriptor> column_families;
1497 for (const auto& cf_name : cf_names) {
1498 column_families.emplace_back(cf_name, cf_options_);
1499 }
1500 std::string db_id;
1501 bool has_missing_table_file = false;
1502 s = versions_->TryRecoverFromOneManifest(manifest_path, column_families,
1503 read_only, &db_id,
1504 &has_missing_table_file);
1505 auto iter =
1506 std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName);
1507 if (iter == cf_names.end()) {
1508 ASSERT_TRUE(s.IsInvalidArgument());
1509 } else {
1510 ASSERT_TRUE(s.IsCorruption());
1511 }
1512 }
1513
TEST_P(VersionSetTestEmptyDb,OpenFromInCompleteManifest2)1514 TEST_P(VersionSetTestEmptyDb, OpenFromInCompleteManifest2) {
1515 db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
1516 PrepareManifest(nullptr, nullptr, &log_writer_);
1517 // Write all column families but no log_number, next_file_number and
1518 // last_sequence.
1519 const std::vector<std::string> all_cf_names = {
1520 kDefaultColumnFamilyName, kColumnFamilyName1, kColumnFamilyName2,
1521 kColumnFamilyName3};
1522 uint32_t cf_id = 1;
1523 Status s;
1524 for (size_t i = 1; i != all_cf_names.size(); ++i) {
1525 VersionEdit new_cf;
1526 new_cf.AddColumnFamily(all_cf_names[i]);
1527 new_cf.SetColumnFamily(cf_id++);
1528 std::string record;
1529 ASSERT_TRUE(new_cf.EncodeTo(&record));
1530 s = log_writer_->AddRecord(record);
1531 ASSERT_OK(s);
1532 }
1533 log_writer_.reset();
1534 s = SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr);
1535 ASSERT_OK(s);
1536
1537 std::string manifest_path;
1538 VerifyManifest(&manifest_path);
1539
1540 bool read_only = std::get<1>(GetParam());
1541 const std::vector<std::string>& cf_names = std::get<2>(GetParam());
1542 std::vector<ColumnFamilyDescriptor> column_families;
1543 for (const auto& cf_name : cf_names) {
1544 column_families.emplace_back(cf_name, cf_options_);
1545 }
1546 std::string db_id;
1547 bool has_missing_table_file = false;
1548 s = versions_->TryRecoverFromOneManifest(manifest_path, column_families,
1549 read_only, &db_id,
1550 &has_missing_table_file);
1551 auto iter =
1552 std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName);
1553 if (iter == cf_names.end()) {
1554 ASSERT_TRUE(s.IsInvalidArgument());
1555 } else {
1556 ASSERT_TRUE(s.IsCorruption());
1557 }
1558 }
1559
TEST_P(VersionSetTestEmptyDb,OpenManifestWithUnknownCF)1560 TEST_P(VersionSetTestEmptyDb, OpenManifestWithUnknownCF) {
1561 db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
1562 PrepareManifest(nullptr, nullptr, &log_writer_);
1563 // Write all column families but no log_number, next_file_number and
1564 // last_sequence.
1565 const std::vector<std::string> all_cf_names = {
1566 kDefaultColumnFamilyName, kColumnFamilyName1, kColumnFamilyName2,
1567 kColumnFamilyName3};
1568 uint32_t cf_id = 1;
1569 Status s;
1570 for (size_t i = 1; i != all_cf_names.size(); ++i) {
1571 VersionEdit new_cf;
1572 new_cf.AddColumnFamily(all_cf_names[i]);
1573 new_cf.SetColumnFamily(cf_id++);
1574 std::string record;
1575 ASSERT_TRUE(new_cf.EncodeTo(&record));
1576 s = log_writer_->AddRecord(record);
1577 ASSERT_OK(s);
1578 }
1579 {
1580 VersionEdit tmp_edit;
1581 tmp_edit.SetColumnFamily(4);
1582 tmp_edit.SetLogNumber(0);
1583 tmp_edit.SetNextFile(2);
1584 tmp_edit.SetLastSequence(0);
1585 std::string record;
1586 ASSERT_TRUE(tmp_edit.EncodeTo(&record));
1587 s = log_writer_->AddRecord(record);
1588 ASSERT_OK(s);
1589 }
1590 log_writer_.reset();
1591 s = SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr);
1592 ASSERT_OK(s);
1593
1594 std::string manifest_path;
1595 VerifyManifest(&manifest_path);
1596
1597 bool read_only = std::get<1>(GetParam());
1598 const std::vector<std::string>& cf_names = std::get<2>(GetParam());
1599 std::vector<ColumnFamilyDescriptor> column_families;
1600 for (const auto& cf_name : cf_names) {
1601 column_families.emplace_back(cf_name, cf_options_);
1602 }
1603 std::string db_id;
1604 bool has_missing_table_file = false;
1605 s = versions_->TryRecoverFromOneManifest(manifest_path, column_families,
1606 read_only, &db_id,
1607 &has_missing_table_file);
1608 auto iter =
1609 std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName);
1610 if (iter == cf_names.end()) {
1611 ASSERT_TRUE(s.IsInvalidArgument());
1612 } else {
1613 ASSERT_TRUE(s.IsCorruption());
1614 }
1615 }
1616
TEST_P(VersionSetTestEmptyDb,OpenCompleteManifest)1617 TEST_P(VersionSetTestEmptyDb, OpenCompleteManifest) {
1618 db_options_.write_dbid_to_manifest = std::get<0>(GetParam());
1619 PrepareManifest(nullptr, nullptr, &log_writer_);
1620 // Write all column families but no log_number, next_file_number and
1621 // last_sequence.
1622 const std::vector<std::string> all_cf_names = {
1623 kDefaultColumnFamilyName, kColumnFamilyName1, kColumnFamilyName2,
1624 kColumnFamilyName3};
1625 uint32_t cf_id = 1;
1626 Status s;
1627 for (size_t i = 1; i != all_cf_names.size(); ++i) {
1628 VersionEdit new_cf;
1629 new_cf.AddColumnFamily(all_cf_names[i]);
1630 new_cf.SetColumnFamily(cf_id++);
1631 std::string record;
1632 ASSERT_TRUE(new_cf.EncodeTo(&record));
1633 s = log_writer_->AddRecord(record);
1634 ASSERT_OK(s);
1635 }
1636 {
1637 VersionEdit tmp_edit;
1638 tmp_edit.SetLogNumber(0);
1639 tmp_edit.SetNextFile(2);
1640 tmp_edit.SetLastSequence(0);
1641 std::string record;
1642 ASSERT_TRUE(tmp_edit.EncodeTo(&record));
1643 s = log_writer_->AddRecord(record);
1644 ASSERT_OK(s);
1645 }
1646 log_writer_.reset();
1647 s = SetCurrentFile(fs_.get(), dbname_, 1, /*directory_to_fsync=*/nullptr);
1648 ASSERT_OK(s);
1649
1650 std::string manifest_path;
1651 VerifyManifest(&manifest_path);
1652
1653 bool read_only = std::get<1>(GetParam());
1654 const std::vector<std::string>& cf_names = std::get<2>(GetParam());
1655 std::vector<ColumnFamilyDescriptor> column_families;
1656 for (const auto& cf_name : cf_names) {
1657 column_families.emplace_back(cf_name, cf_options_);
1658 }
1659 std::string db_id;
1660 bool has_missing_table_file = false;
1661 s = versions_->TryRecoverFromOneManifest(manifest_path, column_families,
1662 read_only, &db_id,
1663 &has_missing_table_file);
1664 auto iter =
1665 std::find(cf_names.begin(), cf_names.end(), kDefaultColumnFamilyName);
1666 if (iter == cf_names.end()) {
1667 ASSERT_TRUE(s.IsInvalidArgument());
1668 } else if (read_only) {
1669 ASSERT_OK(s);
1670 ASSERT_FALSE(has_missing_table_file);
1671 } else if (cf_names.size() == all_cf_names.size()) {
1672 ASSERT_OK(s);
1673 ASSERT_FALSE(has_missing_table_file);
1674 } else if (cf_names.size() < all_cf_names.size()) {
1675 ASSERT_TRUE(s.IsInvalidArgument());
1676 } else {
1677 ASSERT_OK(s);
1678 ASSERT_FALSE(has_missing_table_file);
1679 ColumnFamilyData* cfd = versions_->GetColumnFamilySet()->GetColumnFamily(
1680 kUnknownColumnFamilyName);
1681 ASSERT_EQ(nullptr, cfd);
1682 }
1683 }
1684
1685 INSTANTIATE_TEST_CASE_P(
1686 BestEffortRecovery, VersionSetTestEmptyDb,
1687 testing::Combine(
1688 /*write_dbid_to_manifest=*/testing::Bool(),
1689 /*read_only=*/testing::Bool(),
1690 /*cf_names=*/
1691 testing::Values(
1692 std::vector<std::string>(),
1693 std::vector<std::string>({kDefaultColumnFamilyName}),
1694 std::vector<std::string>({VersionSetTestBase::kColumnFamilyName1,
1695 VersionSetTestBase::kColumnFamilyName2,
1696 VersionSetTestBase::kColumnFamilyName3}),
1697 std::vector<std::string>({kDefaultColumnFamilyName,
1698 VersionSetTestBase::kColumnFamilyName1}),
1699 std::vector<std::string>({kDefaultColumnFamilyName,
1700 VersionSetTestBase::kColumnFamilyName1,
1701 VersionSetTestBase::kColumnFamilyName2,
1702 VersionSetTestBase::kColumnFamilyName3}),
1703 std::vector<std::string>(
1704 {kDefaultColumnFamilyName,
1705 VersionSetTestBase::kColumnFamilyName1,
1706 VersionSetTestBase::kColumnFamilyName2,
1707 VersionSetTestBase::kColumnFamilyName3,
1708 VersionSetTestEmptyDb::kUnknownColumnFamilyName}))));
1709
1710 class VersionSetTestMissingFiles : public VersionSetTestBase,
1711 public testing::Test {
1712 public:
VersionSetTestMissingFiles()1713 VersionSetTestMissingFiles()
1714 : VersionSetTestBase("version_set_test_missing_files"),
1715 block_based_table_options_(),
1716 table_factory_(std::make_shared<BlockBasedTableFactory>(
1717 block_based_table_options_)),
1718 internal_comparator_(
1719 std::make_shared<InternalKeyComparator>(options_.comparator)) {}
1720
1721 protected:
PrepareManifest(std::vector<ColumnFamilyDescriptor> * column_families,SequenceNumber * last_seqno,std::unique_ptr<log::Writer> * log_writer)1722 void PrepareManifest(std::vector<ColumnFamilyDescriptor>* column_families,
1723 SequenceNumber* last_seqno,
1724 std::unique_ptr<log::Writer>* log_writer) override {
1725 assert(column_families != nullptr);
1726 assert(last_seqno != nullptr);
1727 assert(log_writer != nullptr);
1728 const std::string manifest = DescriptorFileName(dbname_, 1);
1729 std::unique_ptr<WritableFile> file;
1730 Status s = env_->NewWritableFile(
1731 manifest, &file, env_->OptimizeForManifestWrite(env_options_));
1732 ASSERT_OK(s);
1733 std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
1734 NewLegacyWritableFileWrapper(std::move(file)), manifest, env_options_));
1735 log_writer->reset(new log::Writer(std::move(file_writer), 0, false));
1736 VersionEdit new_db;
1737 if (db_options_.write_dbid_to_manifest) {
1738 std::unique_ptr<DBImpl> impl(new DBImpl(DBOptions(), dbname_));
1739 std::string db_id;
1740 impl->GetDbIdentityFromIdentityFile(&db_id);
1741 new_db.SetDBId(db_id);
1742 }
1743 {
1744 std::string record;
1745 ASSERT_TRUE(new_db.EncodeTo(&record));
1746 s = (*log_writer)->AddRecord(record);
1747 ASSERT_OK(s);
1748 }
1749 const std::vector<std::string> cf_names = {
1750 kDefaultColumnFamilyName, kColumnFamilyName1, kColumnFamilyName2,
1751 kColumnFamilyName3};
1752 uint32_t cf_id = 1; // default cf id is 0
1753 cf_options_.table_factory = table_factory_;
1754 for (const auto& cf_name : cf_names) {
1755 column_families->emplace_back(cf_name, cf_options_);
1756 if (cf_name == kDefaultColumnFamilyName) {
1757 continue;
1758 }
1759 VersionEdit new_cf;
1760 new_cf.AddColumnFamily(cf_name);
1761 new_cf.SetColumnFamily(cf_id);
1762 std::string record;
1763 ASSERT_TRUE(new_cf.EncodeTo(&record));
1764 s = (*log_writer)->AddRecord(record);
1765 ASSERT_OK(s);
1766
1767 VersionEdit cf_files;
1768 cf_files.SetColumnFamily(cf_id);
1769 cf_files.SetLogNumber(0);
1770 record.clear();
1771 ASSERT_TRUE(cf_files.EncodeTo(&record));
1772 s = (*log_writer)->AddRecord(record);
1773 ASSERT_OK(s);
1774 ++cf_id;
1775 }
1776 SequenceNumber seq = 2;
1777 {
1778 VersionEdit edit;
1779 edit.SetNextFile(7);
1780 edit.SetLastSequence(seq);
1781 std::string record;
1782 ASSERT_TRUE(edit.EncodeTo(&record));
1783 s = (*log_writer)->AddRecord(record);
1784 ASSERT_OK(s);
1785 }
1786 *last_seqno = seq + 1;
1787 }
1788
1789 struct SstInfo {
1790 uint64_t file_number;
1791 std::string column_family;
1792 std::string key; // the only key
1793 int level = 0;
SstInfoROCKSDB_NAMESPACE::VersionSetTestMissingFiles::SstInfo1794 SstInfo(uint64_t file_num, const std::string& cf_name,
1795 const std::string& _key)
1796 : SstInfo(file_num, cf_name, _key, 0) {}
SstInfoROCKSDB_NAMESPACE::VersionSetTestMissingFiles::SstInfo1797 SstInfo(uint64_t file_num, const std::string& cf_name,
1798 const std::string& _key, int lvl)
1799 : file_number(file_num),
1800 column_family(cf_name),
1801 key(_key),
1802 level(lvl) {}
1803 };
1804
1805 // Create dummy sst, return their metadata. Note that only file name and size
1806 // are used.
CreateDummyTableFiles(const std::vector<SstInfo> & file_infos,std::vector<FileMetaData> * file_metas)1807 void CreateDummyTableFiles(const std::vector<SstInfo>& file_infos,
1808 std::vector<FileMetaData>* file_metas) {
1809 assert(file_metas != nullptr);
1810 for (const auto& info : file_infos) {
1811 uint64_t file_num = info.file_number;
1812 std::string fname = MakeTableFileName(dbname_, file_num);
1813 std::unique_ptr<FSWritableFile> file;
1814 Status s = fs_->NewWritableFile(fname, FileOptions(), &file, nullptr);
1815 ASSERT_OK(s);
1816 std::unique_ptr<WritableFileWriter> fwriter(
1817 new WritableFileWriter(std::move(file), fname, FileOptions(), env_));
1818 std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
1819 int_tbl_prop_collector_factories;
1820
1821 std::unique_ptr<TableBuilder> builder(table_factory_->NewTableBuilder(
1822 TableBuilderOptions(
1823 immutable_cf_options_, mutable_cf_options_, *internal_comparator_,
1824 &int_tbl_prop_collector_factories, kNoCompression,
1825 /*_sample_for_compression=*/0, CompressionOptions(),
1826 /*_skip_filters=*/false, info.column_family, info.level),
1827 TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
1828 fwriter.get()));
1829 InternalKey ikey(info.key, 0, ValueType::kTypeValue);
1830 builder->Add(ikey.Encode(), "value");
1831 ASSERT_OK(builder->Finish());
1832 fwriter->Flush();
1833 uint64_t file_size = 0;
1834 s = fs_->GetFileSize(fname, IOOptions(), &file_size, nullptr);
1835 ASSERT_OK(s);
1836 ASSERT_NE(0, file_size);
1837 FileMetaData meta;
1838 meta = FileMetaData(file_num, /*file_path_id=*/0, file_size, ikey, ikey,
1839 0, 0, false, 0, 0, 0, kUnknownFileChecksum,
1840 kUnknownFileChecksumFuncName);
1841 file_metas->emplace_back(meta);
1842 }
1843 }
1844
1845 // This method updates last_sequence_.
WriteFileAdditionAndDeletionToManifest(uint32_t cf,const std::vector<std::pair<int,FileMetaData>> & added_files,const std::vector<std::pair<int,uint64_t>> & deleted_files)1846 void WriteFileAdditionAndDeletionToManifest(
1847 uint32_t cf, const std::vector<std::pair<int, FileMetaData>>& added_files,
1848 const std::vector<std::pair<int, uint64_t>>& deleted_files) {
1849 VersionEdit edit;
1850 edit.SetColumnFamily(cf);
1851 for (const auto& elem : added_files) {
1852 int level = elem.first;
1853 edit.AddFile(level, elem.second);
1854 }
1855 for (const auto& elem : deleted_files) {
1856 int level = elem.first;
1857 edit.DeleteFile(level, elem.second);
1858 }
1859 edit.SetLastSequence(last_seqno_);
1860 ++last_seqno_;
1861 assert(log_writer_.get() != nullptr);
1862 std::string record;
1863 ASSERT_TRUE(edit.EncodeTo(&record));
1864 Status s = log_writer_->AddRecord(record);
1865 ASSERT_OK(s);
1866 }
1867
1868 BlockBasedTableOptions block_based_table_options_;
1869 std::shared_ptr<TableFactory> table_factory_;
1870 std::shared_ptr<InternalKeyComparator> internal_comparator_;
1871 std::vector<ColumnFamilyDescriptor> column_families_;
1872 SequenceNumber last_seqno_;
1873 std::unique_ptr<log::Writer> log_writer_;
1874 };
1875
TEST_F(VersionSetTestMissingFiles,ManifestFarBehindSst)1876 TEST_F(VersionSetTestMissingFiles, ManifestFarBehindSst) {
1877 std::vector<SstInfo> existing_files = {
1878 SstInfo(100, kDefaultColumnFamilyName, "a"),
1879 SstInfo(102, kDefaultColumnFamilyName, "b"),
1880 SstInfo(103, kDefaultColumnFamilyName, "c"),
1881 SstInfo(107, kDefaultColumnFamilyName, "d"),
1882 SstInfo(110, kDefaultColumnFamilyName, "e")};
1883 std::vector<FileMetaData> file_metas;
1884 CreateDummyTableFiles(existing_files, &file_metas);
1885
1886 PrepareManifest(&column_families_, &last_seqno_, &log_writer_);
1887 std::vector<std::pair<int, FileMetaData>> added_files;
1888 for (uint64_t file_num = 10; file_num < 15; ++file_num) {
1889 std::string smallest_ukey = "a";
1890 std::string largest_ukey = "b";
1891 InternalKey smallest_ikey(smallest_ukey, 1, ValueType::kTypeValue);
1892 InternalKey largest_ikey(largest_ukey, 1, ValueType::kTypeValue);
1893 FileMetaData meta =
1894 FileMetaData(file_num, /*file_path_id=*/0, /*file_size=*/12,
1895 smallest_ikey, largest_ikey, 0, 0, false, 0, 0, 0,
1896 kUnknownFileChecksum, kUnknownFileChecksumFuncName);
1897 added_files.emplace_back(0, meta);
1898 }
1899 WriteFileAdditionAndDeletionToManifest(
1900 /*cf=*/0, added_files, std::vector<std::pair<int, uint64_t>>());
1901 std::vector<std::pair<int, uint64_t>> deleted_files;
1902 deleted_files.emplace_back(0, 10);
1903 WriteFileAdditionAndDeletionToManifest(
1904 /*cf=*/0, std::vector<std::pair<int, FileMetaData>>(), deleted_files);
1905 log_writer_.reset();
1906 Status s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr);
1907 ASSERT_OK(s);
1908 std::string manifest_path;
1909 VerifyManifest(&manifest_path);
1910 std::string db_id;
1911 bool has_missing_table_file = false;
1912 s = versions_->TryRecoverFromOneManifest(manifest_path, column_families_,
1913 /*read_only=*/false, &db_id,
1914 &has_missing_table_file);
1915 ASSERT_OK(s);
1916 ASSERT_TRUE(has_missing_table_file);
1917 for (ColumnFamilyData* cfd : *(versions_->GetColumnFamilySet())) {
1918 VersionStorageInfo* vstorage = cfd->current()->storage_info();
1919 const std::vector<FileMetaData*>& files = vstorage->LevelFiles(0);
1920 ASSERT_TRUE(files.empty());
1921 }
1922 }
1923
TEST_F(VersionSetTestMissingFiles,ManifestAheadofSst)1924 TEST_F(VersionSetTestMissingFiles, ManifestAheadofSst) {
1925 std::vector<SstInfo> existing_files = {
1926 SstInfo(100, kDefaultColumnFamilyName, "a"),
1927 SstInfo(102, kDefaultColumnFamilyName, "b"),
1928 SstInfo(103, kDefaultColumnFamilyName, "c"),
1929 SstInfo(107, kDefaultColumnFamilyName, "d"),
1930 SstInfo(110, kDefaultColumnFamilyName, "e")};
1931 std::vector<FileMetaData> file_metas;
1932 CreateDummyTableFiles(existing_files, &file_metas);
1933
1934 PrepareManifest(&column_families_, &last_seqno_, &log_writer_);
1935 std::vector<std::pair<int, FileMetaData>> added_files;
1936 for (size_t i = 3; i != 5; ++i) {
1937 added_files.emplace_back(0, file_metas[i]);
1938 }
1939 WriteFileAdditionAndDeletionToManifest(
1940 /*cf=*/0, added_files, std::vector<std::pair<int, uint64_t>>());
1941
1942 added_files.clear();
1943 for (uint64_t file_num = 120; file_num < 130; ++file_num) {
1944 std::string smallest_ukey = "a";
1945 std::string largest_ukey = "b";
1946 InternalKey smallest_ikey(smallest_ukey, 1, ValueType::kTypeValue);
1947 InternalKey largest_ikey(largest_ukey, 1, ValueType::kTypeValue);
1948 FileMetaData meta =
1949 FileMetaData(file_num, /*file_path_id=*/0, /*file_size=*/12,
1950 smallest_ikey, largest_ikey, 0, 0, false, 0, 0, 0,
1951 kUnknownFileChecksum, kUnknownFileChecksumFuncName);
1952 added_files.emplace_back(0, meta);
1953 }
1954 WriteFileAdditionAndDeletionToManifest(
1955 /*cf=*/0, added_files, std::vector<std::pair<int, uint64_t>>());
1956 log_writer_.reset();
1957 Status s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr);
1958 ASSERT_OK(s);
1959 std::string manifest_path;
1960 VerifyManifest(&manifest_path);
1961 std::string db_id;
1962 bool has_missing_table_file = false;
1963 s = versions_->TryRecoverFromOneManifest(manifest_path, column_families_,
1964 /*read_only=*/false, &db_id,
1965 &has_missing_table_file);
1966 ASSERT_OK(s);
1967 ASSERT_TRUE(has_missing_table_file);
1968 for (ColumnFamilyData* cfd : *(versions_->GetColumnFamilySet())) {
1969 VersionStorageInfo* vstorage = cfd->current()->storage_info();
1970 const std::vector<FileMetaData*>& files = vstorage->LevelFiles(0);
1971 if (cfd->GetName() == kDefaultColumnFamilyName) {
1972 ASSERT_EQ(2, files.size());
1973 for (const auto* fmeta : files) {
1974 if (fmeta->fd.GetNumber() != 107 && fmeta->fd.GetNumber() != 110) {
1975 ASSERT_FALSE(true);
1976 }
1977 }
1978 } else {
1979 ASSERT_TRUE(files.empty());
1980 }
1981 }
1982 }
1983
TEST_F(VersionSetTestMissingFiles,NoFileMissing)1984 TEST_F(VersionSetTestMissingFiles, NoFileMissing) {
1985 std::vector<SstInfo> existing_files = {
1986 SstInfo(100, kDefaultColumnFamilyName, "a"),
1987 SstInfo(102, kDefaultColumnFamilyName, "b"),
1988 SstInfo(103, kDefaultColumnFamilyName, "c"),
1989 SstInfo(107, kDefaultColumnFamilyName, "d"),
1990 SstInfo(110, kDefaultColumnFamilyName, "e")};
1991 std::vector<FileMetaData> file_metas;
1992 CreateDummyTableFiles(existing_files, &file_metas);
1993
1994 PrepareManifest(&column_families_, &last_seqno_, &log_writer_);
1995 std::vector<std::pair<int, FileMetaData>> added_files;
1996 for (const auto& meta : file_metas) {
1997 added_files.emplace_back(0, meta);
1998 }
1999 WriteFileAdditionAndDeletionToManifest(
2000 /*cf=*/0, added_files, std::vector<std::pair<int, uint64_t>>());
2001 std::vector<std::pair<int, uint64_t>> deleted_files;
2002 deleted_files.emplace_back(/*level=*/0, 100);
2003 WriteFileAdditionAndDeletionToManifest(
2004 /*cf=*/0, std::vector<std::pair<int, FileMetaData>>(), deleted_files);
2005 log_writer_.reset();
2006 Status s = SetCurrentFile(fs_.get(), dbname_, 1, nullptr);
2007 ASSERT_OK(s);
2008 std::string manifest_path;
2009 VerifyManifest(&manifest_path);
2010 std::string db_id;
2011 bool has_missing_table_file = false;
2012 s = versions_->TryRecoverFromOneManifest(manifest_path, column_families_,
2013 /*read_only=*/false, &db_id,
2014 &has_missing_table_file);
2015 ASSERT_OK(s);
2016 ASSERT_FALSE(has_missing_table_file);
2017 for (ColumnFamilyData* cfd : *(versions_->GetColumnFamilySet())) {
2018 VersionStorageInfo* vstorage = cfd->current()->storage_info();
2019 const std::vector<FileMetaData*>& files = vstorage->LevelFiles(0);
2020 if (cfd->GetName() == kDefaultColumnFamilyName) {
2021 ASSERT_EQ(existing_files.size() - deleted_files.size(), files.size());
2022 bool has_deleted_file = false;
2023 for (const auto* fmeta : files) {
2024 if (fmeta->fd.GetNumber() == 100) {
2025 has_deleted_file = true;
2026 break;
2027 }
2028 }
2029 ASSERT_FALSE(has_deleted_file);
2030 } else {
2031 ASSERT_TRUE(files.empty());
2032 }
2033 }
2034 }
2035
2036 } // namespace ROCKSDB_NAMESPACE
2037
main(int argc,char ** argv)2038 int main(int argc, char** argv) {
2039 ::testing::InitGoogleTest(&argc, argv);
2040 return RUN_ALL_TESTS();
2041 }
2042